[ORC][llvm-jitlink] Extend weak-linking emulation to real dylibs.

Commit 253e116 added support for emulating weak-linking against dylibs
that are (under the emulation) absent at runtime. This commit extends emulated
weak linking support to allow a real dylib to supply the interface (i.e.
-weak-lx / -weak_library can be pointed at a dylib, in which case they should
be read as "weak-link against this dylib, behavining as if it weren't actually
present at runtime").

Author: lhames

llvm/include/llvm/ExecutionEngine/Orc/GetDylibInterface.h

@@ -0,0 +1,40 @@
+//===---- GetDylibInterface.h - Get interface for real dylib ----*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// Get symbol interface from a real dynamic library or TAPI file. These
+// interfaces can be used to simulate weak linking (ld64 -weak-lx /
+// -weak_library) against a library that is absent at runtime.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_EXECUTIONENGINE_ORC_GETDYLIBINTERFACE_H
+#define LLVM_EXECUTIONENGINE_ORC_GETDYLIBINTERFACE_H
+
+#include "llvm/ExecutionEngine/Orc/Core.h"
+#include "llvm/Object/TapiUniversal.h"
+
+namespace llvm::orc {
+
+/// Returns a SymbolNameSet containing the exported symbols defined in the
+/// given dylib.
+Expected<SymbolNameSet> getDylibInterfaceFromDylib(ExecutionSession &ES,
+                                                   Twine Path);
+
+/// Returns a SymbolNameSet containing the exported symbols defined in the
+/// relevant slice of the TapiUniversal file.
+Expected<SymbolNameSet> getDylibInterfaceFromTapiFile(ExecutionSession &ES,
+                                                      Twine Path);
+
+/// Returns a SymbolNameSet containing the exported symbols defined in the
+/// relevant slice of the given file, which may be either a dylib or a tapi
+/// file.
+Expected<SymbolNameSet> getDylibInterface(ExecutionSession &ES, Twine Path);
+
+} // namespace llvm::orc
+
+#endif // LLVM_EXECUTIONENGINE_ORC_GETDYLIBINTERFACE_H

llvm/include/llvm/ExecutionEngine/Orc/GetTapiInterface.h

@@ -25,7 +25,7 @@ add_llvm_component_library(LLVMOrcJIT
   EPCIndirectionUtils.cpp
   ExecutionUtils.cpp
   ObjectFileInterface.cpp
-  GetTapiInterface.cpp
+  GetDylibInterface.cpp
   IndirectionUtils.cpp
   IRCompileLayer.cpp
   IRTransformLayer.cpp

llvm/lib/ExecutionEngine/Orc/CMakeLists.txt

@@ -0,0 +1,127 @@
+//===-------- GetDylibInterface.cpp - Get interface for real dylib --------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ExecutionEngine/Orc/GetDylibInterface.h"
+
+#include "llvm/BinaryFormat/Magic.h"
+#include "llvm/Object/MachO.h"
+#include "llvm/Object/MachOUniversal.h"
+
+#define DEBUG_TYPE "orc"
+
+namespace llvm::orc {
+
+Expected<SymbolNameSet> getDylibInterfaceFromDylib(ExecutionSession &ES,
+                                                   Twine Path) {
+  auto CPUType = MachO::getCPUType(ES.getTargetTriple());
+  if (!CPUType)
+    return CPUType.takeError();
+
+  auto CPUSubType = MachO::getCPUSubType(ES.getTargetTriple());
+  if (!CPUSubType)
+    return CPUSubType.takeError();
+
+  auto Buf = MemoryBuffer::getFile(Path);
+  if (!Buf)
+    return createFileError(Path, Buf.getError());
+
+  auto SymFile =
+      object::SymbolicFile::createSymbolicFile((*Buf)->getMemBufferRef());
+  if (!SymFile)
+    return SymFile.takeError();
+
+  std::unique_ptr<object::MachOObjectFile> MachOFile;
+  if (isa<object::MachOObjectFile>(**SymFile))
+    MachOFile.reset(dyn_cast<object::MachOObjectFile>(SymFile->release()));
+  else if (auto *MachOUni =
+               dyn_cast<object::MachOUniversalBinary>(SymFile->get())) {
+    for (auto &O : MachOUni->objects()) {
+      if (O.getCPUType() == *CPUType && O.getCPUSubType() == *CPUSubType) {
+        if (auto Obj = O.getAsObjectFile())
+          MachOFile = std::move(*Obj);
+        else
+          return Obj.takeError();
+        break;
+      }
+    }
+    if (!MachOFile)
+      return make_error<StringError>("MachO universal binary at " + Path +
+                                         " does not contain a slice for " +
+                                         ES.getTargetTriple().str(),
+                                     inconvertibleErrorCode());
+  } else
+    return make_error<StringError>("File at " + Path + " is not a MachO",
+                                   inconvertibleErrorCode());
+
+  if (MachOFile->getHeader().filetype != MachO::MH_DYLIB)
+    return make_error<StringError>("MachO at " + Path + " is not a dylib",
+                                   inconvertibleErrorCode());
+
+  SymbolNameSet Symbols;
+  for (auto &Sym : MachOFile->symbols()) {
+    if (auto Name = Sym.getName())
+      Symbols.insert(ES.intern(*Name));
+    else
+      return Name.takeError();
+  }
+
+  return std::move(Symbols);
+}
+
+Expected<SymbolNameSet> getDylibInterfaceFromTapiFile(ExecutionSession &ES,
+                                                      Twine Path) {
+  SymbolNameSet Symbols;
+
+  auto TapiFileBuffer = MemoryBuffer::getFile(Path);
+  if (!TapiFileBuffer)
+    return createFileError(Path, TapiFileBuffer.getError());
+
+  auto Tapi =
+      object::TapiUniversal::create((*TapiFileBuffer)->getMemBufferRef());
+  if (!Tapi)
+    return Tapi.takeError();
+
+  auto CPUType = MachO::getCPUType(ES.getTargetTriple());
+  if (!CPUType)
+    return CPUType.takeError();
+
+  auto CPUSubType = MachO::getCPUSubType(ES.getTargetTriple());
+  if (!CPUSubType)
+    return CPUSubType.takeError();
+
+  auto &IF = (*Tapi)->getInterfaceFile();
+  auto Interface =
+      IF.extract(MachO::getArchitectureFromCpuType(*CPUType, *CPUSubType));
+  if (!Interface)
+    return Interface.takeError();
+
+  for (auto *Sym : (*Interface)->exports())
+    Symbols.insert(ES.intern(Sym->getName()));
+
+  return Symbols;
+}
+
+Expected<SymbolNameSet> getDylibInterface(ExecutionSession &ES, Twine Path) {
+  file_magic Magic;
+  if (auto EC = identify_magic(Path, Magic))
+    return createFileError(Path, EC);
+
+  SymbolNameSet Symbols;
+  switch (Magic) {
+  case file_magic::macho_dynamically_linked_shared_lib:
+    return getDylibInterfaceFromDylib(ES, Path);
+  case file_magic::tapi_file:
+    return getDylibInterfaceFromTapiFile(ES, Path);
+  default:
+    return make_error<StringError>("Cannot get interface for " + Path +
+                                       " unrecognized file type",
+                                   inconvertibleErrorCode());
+  }
+}
+
+} // namespace llvm::orc

llvm/lib/ExecutionEngine/Orc/GetDylibInterface.cpp

@@ -29,7 +29,7 @@
 #include "llvm/ExecutionEngine/Orc/EPCDynamicLibrarySearchGenerator.h"
 #include "llvm/ExecutionEngine/Orc/EPCEHFrameRegistrar.h"
 #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h"
-#include "llvm/ExecutionEngine/Orc/GetTapiInterface.h"
+#include "llvm/ExecutionEngine/Orc/GetDylibInterface.h"
 #include "llvm/ExecutionEngine/Orc/IndirectionUtils.h"
 #include "llvm/ExecutionEngine/Orc/JITLinkRedirectableSymbolManager.h"
 #include "llvm/ExecutionEngine/Orc/JITLinkReentryTrampolines.h"
@@ -2117,17 +2117,8 @@ static SmallVector<StringRef, 5> getSearchPathsFromEnvVar(Session &S) {
 }
 
 static Expected<std::unique_ptr<DefinitionGenerator>>
-LoadLibraryWeak(Session &S, StringRef InterfacePath) {
-  auto TapiFileBuffer = getFile(InterfacePath);
-  if (!TapiFileBuffer)
-    return TapiFileBuffer.takeError();
-
-  auto Tapi =
-      object::TapiUniversal::create((*TapiFileBuffer)->getMemBufferRef());
-  if (!Tapi)
-    return Tapi.takeError();
-
-  auto Symbols = getInterfaceFromTapiFile(S.ES, **Tapi);
+LoadLibraryWeak(Session &S, StringRef Path) {
+  auto Symbols = getDylibInterface(S.ES, Path);
   if (!Symbols)
     return Symbols.takeError();
 
@@ -2227,7 +2218,7 @@ static Error addLibraries(Session &S,
   StringRef StandardExtensions[] = {".so", ".dylib", ".dll", ".a", ".lib"};
   StringRef DynLibExtensionsOnly[] = {".so", ".dylib", ".dll"};
   StringRef ArchiveExtensionsOnly[] = {".a", ".lib"};
-  StringRef InterfaceExtensionsOnly = {".tbd"};
+  StringRef WeakLinkExtensionsOnly[] = {".dylib", ".tbd"};
 
   // Add -lx arguments to LibraryLoads.
   for (auto LibItr = Libraries.begin(), LibEnd = Libraries.end();
@@ -2260,7 +2251,7 @@ static Error addLibraries(Session &S,
     LibraryLoad LL;
     LL.LibName = *LibWeakItr;
     LL.Position = LibrariesWeak.getPosition(LibWeakItr - LibrariesWeak.begin());
-    LL.CandidateExtensions = InterfaceExtensionsOnly;
+    LL.CandidateExtensions = WeakLinkExtensionsOnly;
     LL.Modifier = LibraryLoad::Weak;
     LibraryLoadQueue.push_back(std::move(LL));
   }
@@ -2403,8 +2394,15 @@ static Error addLibraries(Session &S,
         case file_magic::pecoff_executable:
         case file_magic::elf_shared_object:
         case file_magic::macho_dynamically_linked_shared_lib: {
-          if (auto Err = S.loadAndLinkDynamicLibrary(JD, LibPath.data()))
-            return Err;
+          if (LL.Modifier == LibraryLoad::Weak) {
+            if (auto G = LoadLibraryWeak(S, LibPath.data()))
+              JD.addGenerator(std::move(*G));
+            else
+              return G.takeError();
+          } else {
+            if (auto Err = S.loadAndLinkDynamicLibrary(JD, LibPath.data()))
+              return Err;
+          }
           break;
         }
         case file_magic::archive: