Merge pull request #71956 from benlangmuir/clang-prefix-map-consistent

[Caching] Use clang to prefix-map -fmodule-file-cache-key paths

Author: benlangmuir

include/swift/AST/ModuleDependencies.h

@@ -366,7 +366,11 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase {
 public:
   /// Destination output path
   const std::string pcmOutputPath;
-  
+
+  /// Same as \c pcmOutputPath, but possibly prefix-mapped using clang's prefix
+  /// mapper.
+  const std::string mappedPCMPath;
+
   /// The module map file used to generate the Clang module.
   const std::string moduleMapFile;
 
@@ -390,6 +394,7 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase {
   std::string CASClangIncludeTreeRootID;
 
   ClangModuleDependencyStorage(const std::string &pcmOutputPath,
+                               const std::string &mappedPCMPath,
                                const std::string &moduleMapFile,
                                const std::string &contextHash,
                                const std::vector<std::string> &buildCommandLine,
@@ -400,9 +405,10 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase {
                                const std::string &moduleCacheKey)
       : ModuleDependencyInfoStorageBase(ModuleDependencyKind::Clang,
                                         moduleCacheKey),
-        pcmOutputPath(pcmOutputPath), moduleMapFile(moduleMapFile),
-        contextHash(contextHash), buildCommandLine(buildCommandLine),
-        fileDependencies(fileDependencies), capturedPCMArgs(capturedPCMArgs),
+        pcmOutputPath(pcmOutputPath), mappedPCMPath(mappedPCMPath),
+        moduleMapFile(moduleMapFile), contextHash(contextHash),
+        buildCommandLine(buildCommandLine), fileDependencies(fileDependencies),
+        capturedPCMArgs(capturedPCMArgs),
         CASFileSystemRootID(CASFileSystemRootID),
         CASClangIncludeTreeRootID(clangIncludeTreeRoot) {}
 
@@ -526,20 +532,18 @@ class ModuleDependencyInfo {
   /// Describe the module dependencies for a Clang module that can be
   /// built from a module map and headers.
   static ModuleDependencyInfo forClangModule(
-      const std::string &pcmOutputPath,
-      const std::string &moduleMapFile,
-      const std::string &contextHash,
+      const std::string &pcmOutputPath, const std::string &mappedPCMPath,
+      const std::string &moduleMapFile, const std::string &contextHash,
       const std::vector<std::string> &nonPathCommandLine,
       const std::vector<std::string> &fileDependencies,
       const std::vector<std::string> &capturedPCMArgs,
       const std::string &CASFileSystemRootID,
       const std::string &clangIncludeTreeRoot,
       const std::string &moduleCacheKey) {
-    return ModuleDependencyInfo(
-        std::make_unique<ClangModuleDependencyStorage>(
-          pcmOutputPath, moduleMapFile, contextHash,
-          nonPathCommandLine, fileDependencies, capturedPCMArgs,
-          CASFileSystemRootID, clangIncludeTreeRoot,  moduleCacheKey));
+    return ModuleDependencyInfo(std::make_unique<ClangModuleDependencyStorage>(
+        pcmOutputPath, mappedPCMPath, moduleMapFile, contextHash,
+        nonPathCommandLine, fileDependencies, capturedPCMArgs,
+        CASFileSystemRootID, clangIncludeTreeRoot, moduleCacheKey));
   }
 
   /// Describe a placeholder dependency swift module.

include/swift/ClangImporter/ClangImporter.h

@@ -446,7 +446,9 @@ class ClangImporter final : public ClangModuleLoader {
   void verifyAllModules() override;
 
   using RemapPathCallback = llvm::function_ref<std::string(StringRef)>;
-  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1> bridgeClangModuleDependencies(
+  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
+  bridgeClangModuleDependencies(
+      clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
       clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
       StringRef moduleOutputPath, RemapPathCallback remapPath = nullptr);
 

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -39,7 +39,8 @@ using llvm::BCVBR;
 
 /// Every .moddepcache file begins with these 4 bytes, for easy identification.
 const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D','C'};
-const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 5; // optionalModuleImports
+const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR =
+    6; // mappedPCMPath
 /// Increment this on every change.
 const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 1;
 
@@ -182,6 +183,7 @@ using SwiftPlaceholderModuleDetailsLayout =
 using ClangModuleDetailsLayout =
     BCRecordLayout<CLANG_MODULE_DETAILS_NODE, // ID
                    FileIDField,               // pcmOutputPath
+                   FileIDField,               // mappedPCMPath
                    FileIDField,               // moduleMapPath
                    ContextHashIDField,        // contextHash
                    FlagIDArrayIDField,        // commandLine

lib/ClangImporter/ClangModuleDependencyScanner.cpp

@@ -127,9 +127,29 @@ static std::vector<std::string> getClangDepScanningInvocationArguments(
   return commandLineArgs;
 }
 
+static std::unique_ptr<llvm::PrefixMapper>
+getClangPrefixMapper(DependencyScanningTool &clangScanningTool,
+                     ModuleDeps &clangModuleDep,
+                     clang::CompilerInvocation &depsInvocation) {
+  std::unique_ptr<llvm::PrefixMapper> Mapper;
+  if (clangModuleDep.IncludeTreeID) {
+    Mapper = std::make_unique<llvm::PrefixMapper>();
+  } else if (clangModuleDep.CASFileSystemRootID) {
+    assert(clangScanningTool.getCachingFileSystem());
+    Mapper = std::make_unique<llvm::TreePathPrefixMapper>(
+        clangScanningTool.getCachingFileSystem());
+  }
+
+  if (Mapper)
+    DepscanPrefixMapping::configurePrefixMapper(depsInvocation, *Mapper);
+
+  return Mapper;
+}
+
 ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies(
-     clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
-     StringRef moduleOutputPath, RemapPathCallback callback) {
+    clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
+    clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
+    StringRef moduleOutputPath, RemapPathCallback callback) {
   const auto &ctx = Impl.SwiftContext;
   ModuleDependencyVector result;
 
@@ -206,6 +226,10 @@ ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies(
     (void)success;
     assert(success && "clang option from dep scanner round trip failed");
 
+    // Create a prefix mapper that matches clang's configuration.
+    auto Mapper =
+        getClangPrefixMapper(clangScanningTool, clangModuleDep, depsInvocation);
+
     // Clear the cache key for module. The module key is computed from clang
     // invocation, not swift invocation.
     depsInvocation.getFrontendOpts().ModuleCacheKeys.clear();
@@ -251,10 +275,14 @@ ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies(
       swiftArgs.push_back(IncludeTree);
     }
 
+    std::string mappedPCMPath = pcmPath;
+    if (Mapper)
+      Mapper->mapInPlace(mappedPCMPath);
+
     // Module-level dependencies.
     llvm::StringSet<> alreadyAddedModules;
     auto dependencies = ModuleDependencyInfo::forClangModule(
-        pcmPath, clangModuleDep.ClangModuleMapFile,
+        pcmPath, mappedPCMPath, clangModuleDep.ClangModuleMapFile,
         clangModuleDep.ID.ContextHash, swiftArgs, fileDeps, capturedPCMArgs,
         RootID, IncludeTree, /*module-cache-key*/ "");
     for (const auto &moduleName : clangModuleDep.ClangModuleDeps) {
@@ -414,7 +442,8 @@ ClangImporter::getModuleDependencies(Identifier moduleName,
     return {};
   }
 
-  return bridgeClangModuleDependencies(*clangModuleDependencies,
+  return bridgeClangModuleDependencies(clangScanningTool,
+                                       *clangModuleDependencies,
                                        moduleOutputPath, [&](StringRef path) {
                                          if (mapper)
                                            return mapper->mapToString(path);
@@ -479,8 +508,8 @@ bool ClangImporter::addBridgingHeaderDependencies(
 
   // Record module dependencies for each new module we found.
   auto bridgedDeps = bridgeClangModuleDependencies(
-      clangModuleDependencies->ModuleGraph, cache.getModuleOutputPath(),
-      [&cache](StringRef path) {
+      clangScanningTool, clangModuleDependencies->ModuleGraph,
+      cache.getModuleOutputPath(), [&cache](StringRef path) {
         return cache.getScanService().remapPath(path);
       });
   cache.recordDependencies(bridgedDeps);

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -557,19 +557,20 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
       if (!hasCurrentModule)
         llvm::report_fatal_error("Unexpected CLANG_MODULE_DETAILS_NODE record");
       cache.configureForContextHash(getContextHash());
-      unsigned pcmOutputPathID, moduleMapPathID, contextHashID, commandLineArrayID,
-               fileDependenciesArrayID, capturedPCMArgsArrayID, CASFileSystemRootID,
-               clangIncludeTreeRootID, moduleCacheKeyID;
-      ClangModuleDetailsLayout::readRecord(Scratch, pcmOutputPathID, moduleMapPathID,
-                                           contextHashID, commandLineArrayID,
-                                           fileDependenciesArrayID,
-                                           capturedPCMArgsArrayID,
-                                           CASFileSystemRootID,
-                                           clangIncludeTreeRootID,
-                                           moduleCacheKeyID);
+      unsigned pcmOutputPathID, mappedPCMPathID, moduleMapPathID, contextHashID,
+          commandLineArrayID, fileDependenciesArrayID, capturedPCMArgsArrayID,
+          CASFileSystemRootID, clangIncludeTreeRootID, moduleCacheKeyID;
+      ClangModuleDetailsLayout::readRecord(
+          Scratch, pcmOutputPathID, mappedPCMPathID, moduleMapPathID,
+          contextHashID, commandLineArrayID, fileDependenciesArrayID,
+          capturedPCMArgsArrayID, CASFileSystemRootID, clangIncludeTreeRootID,
+          moduleCacheKeyID);
       auto pcmOutputPath = getIdentifier(pcmOutputPathID);
       if (!pcmOutputPath)
         llvm::report_fatal_error("Bad pcm output path");
+      auto mappedPCMPath = getIdentifier(mappedPCMPathID);
+      if (!mappedPCMPath)
+        llvm::report_fatal_error("Bad mapped pcm path");
       auto moduleMapPath = getIdentifier(moduleMapPathID);
       if (!moduleMapPath)
         llvm::report_fatal_error("Bad module map path");
@@ -597,9 +598,9 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
 
       // Form the dependencies storage object
       auto moduleDep = ModuleDependencyInfo::forClangModule(
-          *pcmOutputPath, *moduleMapPath, *contextHash, *commandLineArgs,
-          *fileDependencies, *capturedPCMArgs, *rootFileSystemID,
-          *clangIncludeTreeRoot, *moduleCacheKey);
+          *pcmOutputPath, *mappedPCMPath, *moduleMapPath, *contextHash,
+          *commandLineArgs, *fileDependencies, *capturedPCMArgs,
+          *rootFileSystemID, *clangIncludeTreeRoot, *moduleCacheKey);
 
       // Add dependencies of this module
       for (const auto &moduleName : *currentModuleImports)
@@ -1036,6 +1037,7 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(
     ClangModuleDetailsLayout::emitRecord(
         Out, ScratchRecord, AbbrCodes[ClangModuleDetailsLayout::Code],
         getIdentifier(clangDeps->pcmOutputPath),
+        getIdentifier(clangDeps->mappedPCMPath),
         getIdentifier(clangDeps->moduleMapFile),
         getIdentifier(clangDeps->contextHash),
         getArrayID(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine),
@@ -1240,6 +1242,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
         auto clangDeps = dependencyInfo->getAsClangModule();
         assert(clangDeps);
         addIdentifier(clangDeps->pcmOutputPath);
+        addIdentifier(clangDeps->mappedPCMPath);
         addIdentifier(clangDeps->moduleMapFile);
         addIdentifier(clangDeps->contextHash);
         addStringArray(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine,

lib/DependencyScan/ScanDependencies.cpp

@@ -286,7 +286,7 @@ static llvm::Error resolveExplicitModuleInputs(
       if (!resolvingDepInfo.isClangModule()) {
         commandLine.push_back("-Xcc");
         commandLine.push_back("-fmodule-file=" + depModuleID.ModuleName + "=" +
-                              remapPath(clangDepDetails->pcmOutputPath));
+                              clangDepDetails->mappedPCMPath);
         if (!instance.getInvocation()
                  .getClangImporterOptions()
                  .UseClangIncludeTree) {
@@ -307,7 +307,7 @@ static llvm::Error resolveExplicitModuleInputs(
         appendXclang();
         commandLine.push_back("-fmodule-file-cache-key");
         appendXclang();
-        commandLine.push_back(remapPath(clangDepDetails->pcmOutputPath));
+        commandLine.push_back(clangDepDetails->mappedPCMPath);
         appendXclang();
         commandLine.push_back(clangDepDetails->moduleCacheKey);
       }
@@ -373,7 +373,7 @@ static llvm::Error resolveExplicitModuleInputs(
           newCommandLine.push_back("-Xcc");
           newCommandLine.push_back("-fmodule-file-cache-key");
           newCommandLine.push_back("-Xcc");
-          newCommandLine.push_back(remapPath(clangDep->pcmOutputPath));
+          newCommandLine.push_back(clangDep->mappedPCMPath);
           newCommandLine.push_back("-Xcc");
           newCommandLine.push_back(clangDep->moduleCacheKey);
         }

test/ScanDependencies/clang_module_output_symlink.swift

@@ -0,0 +1,39 @@
+// Test that after compiling a module to a path containing a symlink we still
+// get the same scanner output.
+
+// REQUIRES: OS=macosx
+
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+// RUN: mkdir %t/module-outputs
+// RUN: ln -s module-outputs %t/symlink
+
+// RUN: %target-swift-frontend -scan-dependencies %t/a.swift -o %t/deps.json -module-name A -emit-dependencies -emit-dependencies-path %t/deps.d -disable-implicit-concurrency-module-import -disable-implicit-string-processing-module-import -I %t -module-cache-path %t/symlink -verify -cache-compile-job -cas-path %t/cas
+// Check the contents of the JSON output
+// RUN: %validate-json %t/deps.json | %FileCheck %s
+
+// CHECK:      "-fmodule-file=C=[[PCM_PATH:.*symlink.*C-.*.pcm]]"
+// CHECK:      "-fmodule-file-cache-key"
+// CHECK-NEXT: "-Xcc"
+// CHECK-NEXT: "[[PCM_PATH]]"
+// CHECK-NEXT: "-Xcc"
+// CHECK-NEXT: "llvmcas://
+
+// Emit one of the modules, which will be in the symlinked path.
+// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps.json clang:C > %t/C.cmd
+// RUN: %swift_frontend_plain @%t/C.cmd
+
+// RUN: %target-swift-frontend -scan-dependencies %t/a.swift -o %t/deps2.json -module-name A -emit-dependencies -emit-dependencies-path %t/deps.d -disable-implicit-concurrency-module-import -disable-implicit-string-processing-module-import -I %t -module-cache-path %t/symlink -verify -cache-compile-job -cas-path %t/cas
+// RUN: diff -u %t/deps.json %t/deps2.json
+
+//--- module.modulemap
+module B { header "B.h" }
+module C { header "C.h" }
+
+//--- B.h
+#include "C.h"
+
+//--- C.h
+
+//--- a.swift
+import B