[Caching] Use clang to prefix-map -fmodule-file-cache-key paths

When prefix mapping paths that are used in clang, ensure we are
consistently using the same prefix mapper from clang. This prevents
mismatches that could cause modules to fail to load.

rdar://123324072

Author: benlangmuir

include/swift/AST/ModuleDependencies.h

@@ -366,7 +366,11 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase {
 public:
   /// Destination output path
   const std::string pcmOutputPath;
-  
+
+  /// Same as \c pcmOutputPath, but possibly prefix-mapped using clang's prefix
+  /// mapper.
+  const std::string mappedPCMPath;
+
   /// The module map file used to generate the Clang module.
   const std::string moduleMapFile;
 
@@ -390,6 +394,7 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase {
   std::string CASClangIncludeTreeRootID;
 
   ClangModuleDependencyStorage(const std::string &pcmOutputPath,
+                               const std::string &mappedPCMPath,
                                const std::string &moduleMapFile,
                                const std::string &contextHash,
                                const std::vector<std::string> &buildCommandLine,
@@ -400,9 +405,10 @@ class ClangModuleDependencyStorage : public ModuleDependencyInfoStorageBase {
                                const std::string &moduleCacheKey)
       : ModuleDependencyInfoStorageBase(ModuleDependencyKind::Clang,
                                         moduleCacheKey),
-        pcmOutputPath(pcmOutputPath), moduleMapFile(moduleMapFile),
-        contextHash(contextHash), buildCommandLine(buildCommandLine),
-        fileDependencies(fileDependencies), capturedPCMArgs(capturedPCMArgs),
+        pcmOutputPath(pcmOutputPath), mappedPCMPath(mappedPCMPath),
+        moduleMapFile(moduleMapFile), contextHash(contextHash),
+        buildCommandLine(buildCommandLine), fileDependencies(fileDependencies),
+        capturedPCMArgs(capturedPCMArgs),
         CASFileSystemRootID(CASFileSystemRootID),
         CASClangIncludeTreeRootID(clangIncludeTreeRoot) {}
 
@@ -526,20 +532,18 @@ class ModuleDependencyInfo {
   /// Describe the module dependencies for a Clang module that can be
   /// built from a module map and headers.
   static ModuleDependencyInfo forClangModule(
-      const std::string &pcmOutputPath,
-      const std::string &moduleMapFile,
-      const std::string &contextHash,
+      const std::string &pcmOutputPath, const std::string &mappedPCMPath,
+      const std::string &moduleMapFile, const std::string &contextHash,
       const std::vector<std::string> &nonPathCommandLine,
       const std::vector<std::string> &fileDependencies,
       const std::vector<std::string> &capturedPCMArgs,
       const std::string &CASFileSystemRootID,
       const std::string &clangIncludeTreeRoot,
       const std::string &moduleCacheKey) {
-    return ModuleDependencyInfo(
-        std::make_unique<ClangModuleDependencyStorage>(
-          pcmOutputPath, moduleMapFile, contextHash,
-          nonPathCommandLine, fileDependencies, capturedPCMArgs,
-          CASFileSystemRootID, clangIncludeTreeRoot,  moduleCacheKey));
+    return ModuleDependencyInfo(std::make_unique<ClangModuleDependencyStorage>(
+        pcmOutputPath, mappedPCMPath, moduleMapFile, contextHash,
+        nonPathCommandLine, fileDependencies, capturedPCMArgs,
+        CASFileSystemRootID, clangIncludeTreeRoot, moduleCacheKey));
   }
 
   /// Describe a placeholder dependency swift module.

include/swift/ClangImporter/ClangImporter.h

@@ -446,7 +446,9 @@ class ClangImporter final : public ClangModuleLoader {
   void verifyAllModules() override;
 
   using RemapPathCallback = llvm::function_ref<std::string(StringRef)>;
-  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1> bridgeClangModuleDependencies(
+  llvm::SmallVector<std::pair<ModuleDependencyID, ModuleDependencyInfo>, 1>
+  bridgeClangModuleDependencies(
+      clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
       clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
       StringRef moduleOutputPath, RemapPathCallback remapPath = nullptr);
 

include/swift/DependencyScan/SerializedModuleDependencyCacheFormat.h

@@ -39,7 +39,8 @@ using llvm::BCVBR;
 
 /// Every .moddepcache file begins with these 4 bytes, for easy identification.
 const unsigned char MODULE_DEPENDENCY_CACHE_FORMAT_SIGNATURE[] = {'I', 'M', 'D','C'};
-const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR = 5; // optionalModuleImports
+const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MAJOR =
+    6; // mappedPCMPath
 /// Increment this on every change.
 const unsigned MODULE_DEPENDENCY_CACHE_FORMAT_VERSION_MINOR = 1;
 
@@ -182,6 +183,7 @@ using SwiftPlaceholderModuleDetailsLayout =
 using ClangModuleDetailsLayout =
     BCRecordLayout<CLANG_MODULE_DETAILS_NODE, // ID
                    FileIDField,               // pcmOutputPath
+                   FileIDField,               // mappedPCMPath
                    FileIDField,               // moduleMapPath
                    ContextHashIDField,        // contextHash
                    FlagIDArrayIDField,        // commandLine

lib/ClangImporter/ClangModuleDependencyScanner.cpp

@@ -127,9 +127,29 @@ static std::vector<std::string> getClangDepScanningInvocationArguments(
   return commandLineArgs;
 }
 
+static std::unique_ptr<llvm::PrefixMapper>
+getClangPrefixMapper(DependencyScanningTool &clangScanningTool,
+                     ModuleDeps &clangModuleDep,
+                     clang::CompilerInvocation &depsInvocation) {
+  std::unique_ptr<llvm::PrefixMapper> Mapper;
+  if (clangModuleDep.IncludeTreeID) {
+    Mapper = std::make_unique<llvm::PrefixMapper>();
+  } else if (clangModuleDep.CASFileSystemRootID) {
+    assert(clangScanningTool.getCachingFileSystem());
+    Mapper = std::make_unique<llvm::TreePathPrefixMapper>(
+        clangScanningTool.getCachingFileSystem());
+  }
+
+  if (Mapper)
+    DepscanPrefixMapping::configurePrefixMapper(depsInvocation, *Mapper);
+
+  return Mapper;
+}
+
 ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies(
-     clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
-     StringRef moduleOutputPath, RemapPathCallback callback) {
+    clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
+    clang::tooling::dependencies::ModuleDepsGraph &clangDependencies,
+    StringRef moduleOutputPath, RemapPathCallback callback) {
   const auto &ctx = Impl.SwiftContext;
   ModuleDependencyVector result;
 
@@ -206,6 +226,10 @@ ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies(
     (void)success;
     assert(success && "clang option from dep scanner round trip failed");
 
+    // Create a prefix mapper that matches clang's configuration.
+    auto Mapper =
+        getClangPrefixMapper(clangScanningTool, clangModuleDep, depsInvocation);
+
     // Clear the cache key for module. The module key is computed from clang
     // invocation, not swift invocation.
     depsInvocation.getFrontendOpts().ModuleCacheKeys.clear();
@@ -251,10 +275,14 @@ ModuleDependencyVector ClangImporter::bridgeClangModuleDependencies(
       swiftArgs.push_back(IncludeTree);
     }
 
+    std::string mappedPCMPath = pcmPath;
+    if (Mapper)
+      Mapper->mapInPlace(mappedPCMPath);
+
     // Module-level dependencies.
     llvm::StringSet<> alreadyAddedModules;
     auto dependencies = ModuleDependencyInfo::forClangModule(
-        pcmPath, clangModuleDep.ClangModuleMapFile,
+        pcmPath, mappedPCMPath, clangModuleDep.ClangModuleMapFile,
         clangModuleDep.ID.ContextHash, swiftArgs, fileDeps, capturedPCMArgs,
         RootID, IncludeTree, /*module-cache-key*/ "");
     for (const auto &moduleName : clangModuleDep.ClangModuleDeps) {
@@ -414,7 +442,8 @@ ClangImporter::getModuleDependencies(Identifier moduleName,
     return {};
   }
 
-  return bridgeClangModuleDependencies(*clangModuleDependencies,
+  return bridgeClangModuleDependencies(clangScanningTool,
+                                       *clangModuleDependencies,
                                        moduleOutputPath, [&](StringRef path) {
                                          if (mapper)
                                            return mapper->mapToString(path);
@@ -479,8 +508,8 @@ bool ClangImporter::addBridgingHeaderDependencies(
 
   // Record module dependencies for each new module we found.
   auto bridgedDeps = bridgeClangModuleDependencies(
-      clangModuleDependencies->ModuleGraph, cache.getModuleOutputPath(),
-      [&cache](StringRef path) {
+      clangScanningTool, clangModuleDependencies->ModuleGraph,
+      cache.getModuleOutputPath(), [&cache](StringRef path) {
         return cache.getScanService().remapPath(path);
       });
   cache.recordDependencies(bridgedDeps);

lib/DependencyScan/ModuleDependencyCacheSerialization.cpp

@@ -557,19 +557,20 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
       if (!hasCurrentModule)
         llvm::report_fatal_error("Unexpected CLANG_MODULE_DETAILS_NODE record");
       cache.configureForContextHash(getContextHash());
-      unsigned pcmOutputPathID, moduleMapPathID, contextHashID, commandLineArrayID,
-               fileDependenciesArrayID, capturedPCMArgsArrayID, CASFileSystemRootID,
-               clangIncludeTreeRootID, moduleCacheKeyID;
-      ClangModuleDetailsLayout::readRecord(Scratch, pcmOutputPathID, moduleMapPathID,
-                                           contextHashID, commandLineArrayID,
-                                           fileDependenciesArrayID,
-                                           capturedPCMArgsArrayID,
-                                           CASFileSystemRootID,
-                                           clangIncludeTreeRootID,
-                                           moduleCacheKeyID);
+      unsigned pcmOutputPathID, mappedPCMPathID, moduleMapPathID, contextHashID,
+          commandLineArrayID, fileDependenciesArrayID, capturedPCMArgsArrayID,
+          CASFileSystemRootID, clangIncludeTreeRootID, moduleCacheKeyID;
+      ClangModuleDetailsLayout::readRecord(
+          Scratch, pcmOutputPathID, mappedPCMPathID, moduleMapPathID,
+          contextHashID, commandLineArrayID, fileDependenciesArrayID,
+          capturedPCMArgsArrayID, CASFileSystemRootID, clangIncludeTreeRootID,
+          moduleCacheKeyID);
       auto pcmOutputPath = getIdentifier(pcmOutputPathID);
       if (!pcmOutputPath)
         llvm::report_fatal_error("Bad pcm output path");
+      auto mappedPCMPath = getIdentifier(mappedPCMPathID);
+      if (!mappedPCMPath)
+        llvm::report_fatal_error("Bad mapped pcm path");
       auto moduleMapPath = getIdentifier(moduleMapPathID);
       if (!moduleMapPath)
         llvm::report_fatal_error("Bad module map path");
@@ -597,9 +598,9 @@ bool ModuleDependenciesCacheDeserializer::readGraph(SwiftDependencyScanningServi
 
       // Form the dependencies storage object
       auto moduleDep = ModuleDependencyInfo::forClangModule(
-          *pcmOutputPath, *moduleMapPath, *contextHash, *commandLineArgs,
-          *fileDependencies, *capturedPCMArgs, *rootFileSystemID,
-          *clangIncludeTreeRoot, *moduleCacheKey);
+          *pcmOutputPath, *mappedPCMPath, *moduleMapPath, *contextHash,
+          *commandLineArgs, *fileDependencies, *capturedPCMArgs,
+          *rootFileSystemID, *clangIncludeTreeRoot, *moduleCacheKey);
 
       // Add dependencies of this module
       for (const auto &moduleName : *currentModuleImports)
@@ -1036,6 +1037,7 @@ void ModuleDependenciesCacheSerializer::writeModuleInfo(
     ClangModuleDetailsLayout::emitRecord(
         Out, ScratchRecord, AbbrCodes[ClangModuleDetailsLayout::Code],
         getIdentifier(clangDeps->pcmOutputPath),
+        getIdentifier(clangDeps->mappedPCMPath),
         getIdentifier(clangDeps->moduleMapFile),
         getIdentifier(clangDeps->contextHash),
         getArrayID(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine),
@@ -1240,6 +1242,7 @@ void ModuleDependenciesCacheSerializer::collectStringsAndArrays(
         auto clangDeps = dependencyInfo->getAsClangModule();
         assert(clangDeps);
         addIdentifier(clangDeps->pcmOutputPath);
+        addIdentifier(clangDeps->mappedPCMPath);
         addIdentifier(clangDeps->moduleMapFile);
         addIdentifier(clangDeps->contextHash);
         addStringArray(moduleID, ModuleIdentifierArrayKind::NonPathCommandLine,

lib/DependencyScan/ScanDependencies.cpp

@@ -285,7 +285,7 @@ static llvm::Error resolveExplicitModuleInputs(
       if (!resolvingDepInfo.isClangModule()) {
         commandLine.push_back("-Xcc");
         commandLine.push_back("-fmodule-file=" + depModuleID.ModuleName + "=" +
-                              remapPath(clangDepDetails->pcmOutputPath));
+                              clangDepDetails->mappedPCMPath);
         if (!instance.getInvocation()
                  .getClangImporterOptions()
                  .UseClangIncludeTree) {
@@ -306,7 +306,7 @@ static llvm::Error resolveExplicitModuleInputs(
         appendXclang();
         commandLine.push_back("-fmodule-file-cache-key");
         appendXclang();
-        commandLine.push_back(remapPath(clangDepDetails->pcmOutputPath));
+        commandLine.push_back(clangDepDetails->mappedPCMPath);
         appendXclang();
         commandLine.push_back(clangDepDetails->moduleCacheKey);
       }
@@ -372,7 +372,7 @@ static llvm::Error resolveExplicitModuleInputs(
           newCommandLine.push_back("-Xcc");
           newCommandLine.push_back("-fmodule-file-cache-key");
           newCommandLine.push_back("-Xcc");
-          newCommandLine.push_back(remapPath(clangDep->pcmOutputPath));
+          newCommandLine.push_back(clangDep->mappedPCMPath);
           newCommandLine.push_back("-Xcc");
           newCommandLine.push_back(clangDep->moduleCacheKey);
         }

test/ScanDependencies/clang_module_output_symlink.swift

@@ -0,0 +1,39 @@
+// Test that after compiling a module to a path containing a symlink we still
+// get the same scanner output.
+
+// REQUIRES: OS=macosx
+
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+// RUN: mkdir %t/module-outputs
+// RUN: ln -s module-outputs %t/symlink
+
+// RUN: %target-swift-frontend -scan-dependencies %t/a.swift -o %t/deps.json -module-name A -emit-dependencies -emit-dependencies-path %t/deps.d -disable-implicit-concurrency-module-import -disable-implicit-string-processing-module-import -I %t -module-cache-path %t/symlink -verify -cache-compile-job -cas-path %t/cas
+// Check the contents of the JSON output
+// RUN: %validate-json %t/deps.json | %FileCheck %s
+
+// CHECK:      "-fmodule-file=C=[[PCM_PATH:.*symlink.*C-.*.pcm]]"
+// CHECK:      "-fmodule-file-cache-key"
+// CHECK-NEXT: "-Xcc"
+// CHECK-NEXT: "[[PCM_PATH]]"
+// CHECK-NEXT: "-Xcc"
+// CHECK-NEXT: "llvmcas://
+
+// Emit one of the modules, which will be in the symlinked path.
+// RUN: %{python} %S/../CAS/Inputs/BuildCommandExtractor.py %t/deps.json clang:C > %t/C.cmd
+// RUN: %swift_frontend_plain @%t/C.cmd
+
+// RUN: %target-swift-frontend -scan-dependencies %t/a.swift -o %t/deps2.json -module-name A -emit-dependencies -emit-dependencies-path %t/deps.d -disable-implicit-concurrency-module-import -disable-implicit-string-processing-module-import -I %t -module-cache-path %t/symlink -verify -cache-compile-job -cas-path %t/cas
+// RUN: diff -u %t/deps.json %t/deps2.json
+
+//--- module.modulemap
+module B { header "B.h" }
+module C { header "C.h" }
+
+//--- B.h
+#include "C.h"
+
+//--- C.h
+
+//--- a.swift
+import B