[build] Build sourcekit-lsp for multiple arches and lipo them

Instead of building SourceKit-LSP using SwiftPM's multi-arch xcbuild backend, build it for only one arch at a time and then run `lipo` to merge the two resulting binaries.

This should allow us to share build products between building installing and testing and also eliminates other quirks resulting from the xcbuild backend.

Author: ahoppen

utils/swift_build_support/swift_build_support/products/indexstoredb.py

@@ -26,7 +26,6 @@
 from . import swiftsyntax
 from . import xctest
 from .. import shell
-from .. import targets
 
 
 class IndexStoreDB(product.Product):
@@ -50,14 +49,13 @@ def should_build(self, host_target):
         return True
 
     def build(self, host_target):
-        run_build_script_helper('build', host_target, self, self.args)
+        self.run_build_script_helper('build', host_target)
 
     def should_test(self, host_target):
         return self.args.test_indexstoredb
 
     def test(self, host_target):
-        run_build_script_helper('test', host_target, self, self.args,
-                                self.args.test_indexstoredb_sanitize_all)
+        self.run_build_script_helper('test', host_target)
 
     def should_install(self, host_target):
         return False
@@ -81,64 +79,32 @@ def get_dependencies(cls):
                 swiftpm.SwiftPM,
                 swiftsyntax.SwiftSyntax]
 
-
-def run_build_script_helper(action, host_target, product, args,
-                            sanitize_all=False, clean=False):
-    script_path = os.path.join(
-        product.source_dir, 'Utilities', 'build-script-helper.py')
-
-    install_destdir = product.host_install_destdir(host_target)
-    toolchain_path = product.native_toolchain_path(host_target)
-    is_release = product.is_release()
-    configuration = 'release' if is_release else 'debug'
-    helper_cmd = [
-        script_path,
-        action,
-        '--package-path', product.source_dir,
-        '--build-path', product.build_dir,
-        '--configuration', configuration,
-        '--toolchain', toolchain_path,
-        '--ninja-bin', product.toolchain.ninja,
-        '--multiroot-data-file', MULTIROOT_DATA_FILE_PATH,
-    ]
-    if args.verbose_build:
-        helper_cmd.append('--verbose')
-
-    if sanitize_all:
-        helper_cmd.append('--sanitize-all')
-    elif args.enable_asan:
-        helper_cmd.extend(['--sanitize', 'address'])
-    elif args.enable_ubsan:
-        helper_cmd.extend(['--sanitize', 'undefined'])
-    elif args.enable_tsan:
-        helper_cmd.extend(['--sanitize', 'thread'])
-
-    if clean:
-        helper_cmd.append('--clean')
-
-    # Pass Cross compile host info unless we're testing.
-    # It doesn't make sense to run tests of the cross compile host.
-    if product.has_cross_compile_hosts() and action != 'test':
-        if product.is_darwin_host(host_target):
-            if len(args.cross_compile_hosts) != 1:
-                raise RuntimeError("Cross-Compiling indexstoredb to multiple " +
-                                   "targets is not supported")
-            helper_cmd += ['--cross-compile-host', args.cross_compile_hosts[0]]
-        elif product.is_cross_compile_target(host_target):
-            helper_cmd.extend(['--cross-compile-host', host_target])
-            build_toolchain_path = install_destdir + args.install_prefix
-            resource_dir = '%s/lib/swift' % build_toolchain_path
-            helper_cmd += [
-                '--cross-compile-config',
-                targets.StdlibDeploymentTarget.get_target_for_name(host_target).platform
-                .swiftpm_config(args, output_dir=build_toolchain_path,
-                                swift_toolchain=toolchain_path,
-                                resource_path=resource_dir)
-            ]
-
-    if action == 'install' and product.product_name() == "sourcekitlsp":
-        helper_cmd.extend([
-            '--prefix', install_destdir + args.install_prefix
-        ])
-
-    shell.call(helper_cmd)
+    def run_build_script_helper(self, action, host_target):
+        script_path = os.path.join(
+            self.source_dir, 'Utilities', 'build-script-helper.py')
+
+        toolchain_path = self.native_toolchain_path(host_target)
+        configuration = 'release' if self.is_release() else 'debug'
+        helper_cmd = [
+            script_path,
+            action,
+            '--package-path', self.source_dir,
+            '--build-path', self.build_dir,
+            '--configuration', configuration,
+            '--toolchain', toolchain_path,
+            '--ninja-bin', self.toolchain.ninja,
+            '--multiroot-data-file', MULTIROOT_DATA_FILE_PATH,
+        ]
+        if self.args.verbose_build:
+            helper_cmd.append('--verbose')
+
+        if self.args.test_indexstoredb_sanitize_all:
+            helper_cmd.append('--sanitize-all')
+        elif self.args.enable_asan:
+            helper_cmd.extend(['--sanitize', 'address'])
+        elif self.args.enable_ubsan:
+            helper_cmd.extend(['--sanitize', 'undefined'])
+        elif self.args.enable_tsan:
+            helper_cmd.extend(['--sanitize', 'thread'])
+
+        shell.call(helper_cmd)

utils/swift_build_support/swift_build_support/products/sourcekitlsp.py

@@ -10,9 +10,12 @@
 #
 # ----------------------------------------------------------------------------
 
+import os
+
+from build_swift.build_swift.constants import MULTIROOT_DATA_FILE_PATH
+
 from . import cmark
 from . import foundation
-from . import indexstoredb
 from . import libcxx
 from . import libdispatch
 from . import llbuild
@@ -21,6 +24,8 @@
 from . import swift
 from . import swiftpm
 from . import xctest
+from .. import shell
+from .. import targets
 
 
 class SourceKitLSP(product.Product):
@@ -43,24 +48,35 @@ def is_swiftpm_unified_build_product(cls):
     def should_build(self, host_target):
         return True
 
+    def _for_each_host_target(self, base_target, body):
+        body(base_target)
+
+        # For Darwin host, 'build' is only called for the builder.
+        # Manually iterate the cross compile hosts.
+        if self.has_cross_compile_hosts() and self.is_darwin_host(base_target):
+            for target in self.args.cross_compile_hosts:
+                body(target)
+
     def build(self, host_target):
-        indexstoredb.run_build_script_helper(
-            'build', host_target, self, self.args)
+        self._for_each_host_target(
+            host_target,
+            lambda target: self.run_build_script_helper('build', host_target, target)
+        )
 
     def should_test(self, host_target):
         return self.args.test_sourcekitlsp
 
     def test(self, host_target):
-        indexstoredb.run_build_script_helper(
-            'test', host_target, self, self.args,
-            self.args.test_sourcekitlsp_sanitize_all)
+        self.run_build_script_helper('test', host_target, host_target)
 
     def should_install(self, host_target):
         return self.args.install_sourcekitlsp
 
     def install(self, host_target):
-        indexstoredb.run_build_script_helper(
-            'install', host_target, self, self.args)
+        self._for_each_host_target(
+            host_target,
+            lambda target: self.run_build_script_helper('install', host_target, target)
+        )
 
     @classmethod
     def get_dependencies(cls):
@@ -73,3 +89,54 @@ def get_dependencies(cls):
                 xctest.XCTest,
                 llbuild.LLBuild,
                 swiftpm.SwiftPM]
+
+    def run_build_script_helper(self, action, base_target, host_target):
+        # base_target is the machine that's driving the build.
+        # host_target is the target we are bulding for.
+        script_path = os.path.join(
+            self.source_dir, 'Utilities', 'build-script-helper.py')
+
+        install_destdir = self.host_install_destdir(host_target)
+        toolchain_path = self.native_toolchain_path(base_target)
+        configuration = 'release' if self.is_release() else 'debug'
+        helper_cmd = [
+            script_path,
+            action,
+            '--package-path', self.source_dir,
+            '--build-path', self.build_dir,
+            '--configuration', configuration,
+            '--toolchain', toolchain_path,
+            '--ninja-bin', self.toolchain.ninja,
+            '--multiroot-data-file', MULTIROOT_DATA_FILE_PATH,
+        ]
+        if self.args.verbose_build:
+            helper_cmd.append('--verbose')
+
+        if self.args.test_sourcekitlsp_sanitize_all:
+            helper_cmd.append('--sanitize-all')
+        elif self.args.enable_asan:
+            helper_cmd += ['--sanitize', 'address']
+        elif self.args.enable_ubsan:
+            helper_cmd += ['--sanitize', 'undefined']
+        elif self.args.enable_tsan:
+            helper_cmd += ['--sanitize', 'thread']
+
+        if self.has_cross_compile_hosts():
+            helper_cmd += ['--cross-compile-host', host_target]
+            if self.is_cross_compile_target(host_target) and \
+                    not self.is_darwin_host(host_target):
+                build_toolchain_path = install_destdir + self.args.install_prefix
+                resource_dir = '%s/lib/swift' % build_toolchain_path
+                helper_cmd += [
+                    '--cross-compile-config',
+                    targets.StdlibDeploymentTarget.get_target_for_name(host_target)
+                    .platform
+                    .swiftpm_config(self.args, output_dir=build_toolchain_path,
+                                    swift_toolchain=toolchain_path,
+                                    resource_path=resource_dir)
+                ]
+
+        if action == 'install':
+            helper_cmd += ['--prefix', install_destdir + self.args.install_prefix]
+
+        shell.call(helper_cmd)