Merge pull request swiftlang#34073 from CodaFi/crossfire

Serialization for Incremental Swift Dependency Information

Author: CodaFi

include/swift/AST/Decl.h

@@ -578,7 +578,7 @@ class alignas(1 << DeclAlignInBits) Decl {
     HasAnyUnavailableValues : 1
   );
 
-  SWIFT_INLINE_BITFIELD(ModuleDecl, TypeDecl, 1+1+1+1+1+1+1+1+1,
+  SWIFT_INLINE_BITFIELD(ModuleDecl, TypeDecl, 1+1+1+1+1+1+1+1+1+1,
     /// If the module was or is being compiled with `-enable-testing`.
     TestingEnabled : 1,
 
@@ -607,7 +607,10 @@ class alignas(1 << DeclAlignInBits) Decl {
     IsNonSwiftModule : 1,
 
     /// Whether this module is the main module.
-    IsMainModule : 1
+    IsMainModule : 1,
+
+    /// Whether this module has incremental dependency information available.
+    HasIncrementalInfo : 1
   );
 
   SWIFT_INLINE_BITFIELD(PrecedenceGroupDecl, Decl, 1+2,

include/swift/AST/FineGrainedDependencies.h

@@ -59,11 +59,14 @@ namespace swift {
 class DependencyTracker;
 class DiagnosticEngine;
 class FrontendOptions;
+class ModuleDecl;
 class SourceFile;
 
 /// Use a new namespace to help keep the experimental code from clashing.
 namespace fine_grained_dependencies {
 
+class SourceFileDepGraph;
+
 using StringVec = std::vector<std::string>;
 
 template <typename Element> using ConstPtrVec = std::vector<const Element *>;
@@ -343,11 +346,11 @@ class BiIndexedTwoStageMap {
 // MARK: Start of fine-grained-dependency-specific code
 //==============================================================================
 
-/// The entry point into this system from the frontend:
-/// Write out the .swiftdeps file for a frontend compilation of a primary file.
-bool emitReferenceDependencies(DiagnosticEngine &diags, SourceFile *SF,
-                               const DependencyTracker &depTracker,
-                               StringRef outputPath, bool alsoEmitDotFile);
+bool withReferenceDependencies(
+    llvm::PointerUnion<ModuleDecl *, SourceFile *> MSF,
+    const DependencyTracker &depTracker, StringRef outputPath,
+    bool alsoEmitDotFile, llvm::function_ref<bool(SourceFileDepGraph &&)>);
+
 //==============================================================================
 // MARK: Enums
 //==============================================================================

include/swift/AST/FineGrainedDependencyFormat.h

@@ -124,7 +124,11 @@ bool readFineGrainedDependencyGraph(llvm::StringRef path,
 
 /// Tries to write the dependency graph to the given path name.
 /// Returns true if there was an error.
-bool writeFineGrainedDependencyGraph(DiagnosticEngine &diags, llvm::StringRef path,
+bool writeFineGrainedDependencyGraphToPath(DiagnosticEngine &diags,
+                                           llvm::StringRef path,
+                                           const SourceFileDepGraph &g);
+
+void writeFineGrainedDependencyGraph(llvm::BitstreamWriter &Out,
                                      const SourceFileDepGraph &g);
 
 } // namespace fine_grained_dependencies

include/swift/AST/Module.h

@@ -517,6 +517,12 @@ class ModuleDecl : public DeclContext, public TypeDecl {
     Bits.ModuleDecl.RawResilienceStrategy = unsigned(strategy);
   }
 
+  /// Returns true if this module was or is being compiled for testing.
+  bool hasIncrementalInfo() const { return Bits.ModuleDecl.HasIncrementalInfo; }
+  void setHasIncrementalInfo(bool enabled = true) {
+    Bits.ModuleDecl.HasIncrementalInfo = enabled;
+  }
+
   /// \returns true if this module is a system module; note that the StdLib is
   /// considered a system module.
   bool isSystemModule() const {

include/swift/Subsystems.h

@@ -75,6 +75,10 @@ namespace swift {
     class TypeConverter;
   }
 
+  namespace fine_grained_dependencies {
+    class SourceFileDepGraph;
+  }
+
   /// @{
 
   /// \returns true if the declaration should be verified.  This can return
@@ -174,8 +178,10 @@ namespace swift {
   using ModuleOrSourceFile = PointerUnion<ModuleDecl *, SourceFile *>;
 
   /// Serializes a module or single source file to the given output file.
-  void serialize(ModuleOrSourceFile DC, const SerializationOptions &options,
-                 const SILModule *M = nullptr);
+  void
+  serialize(ModuleOrSourceFile DC, const SerializationOptions &options,
+            const SILModule *M = nullptr,
+            const fine_grained_dependencies::SourceFileDepGraph *DG = nullptr);
 
   /// Serializes a module or single source file to the given output file and
   /// returns back the file's contents as a memory buffer.

lib/AST/FineGrainedDependencyFormat.cpp

@@ -289,16 +289,13 @@ class Serializer {
   unsigned LastIdentifierID = 0;
   std::vector<StringRef> IdentifiersToWrite;
 
-  SmallVector<char, 0> Buffer;
-  llvm::BitstreamWriter Out{Buffer};
+  llvm::BitstreamWriter &Out;
 
   /// A reusable buffer for emitting records.
   SmallVector<uint64_t, 64> ScratchRecord;
 
   std::array<unsigned, 256> AbbrCodes;
 
-  void writeFineGrainedDependencyGraph(const SourceFileDepGraph &g);
-
   void addIdentifier(StringRef str);
   unsigned getIdentifier(StringRef str);
 
@@ -321,8 +318,10 @@ class Serializer {
   void writeMetadata();
 
 public:
-  void writeFineGrainedDependencyGraph(llvm::raw_ostream &os,
-                                       const SourceFileDepGraph &g);
+  Serializer(llvm::BitstreamWriter &ExistingOut) : Out(ExistingOut) {}
+
+public:
+  void writeFineGrainedDependencyGraph(const SourceFileDepGraph &g);
 };
 
 } // end namespace
@@ -467,20 +466,22 @@ unsigned Serializer::getIdentifier(StringRef str) {
   return iter->second;
 }
 
-void Serializer::writeFineGrainedDependencyGraph(llvm::raw_ostream &os,
-                                                 const SourceFileDepGraph &g) {
-  writeFineGrainedDependencyGraph(g);
-  os.write(Buffer.data(), Buffer.size());
-  os.flush();
+void swift::fine_grained_dependencies::writeFineGrainedDependencyGraph(
+    llvm::BitstreamWriter &Out, const SourceFileDepGraph &g) {
+  Serializer serializer{Out};
+  serializer.writeFineGrainedDependencyGraph(g);
 }
 
-bool swift::fine_grained_dependencies::writeFineGrainedDependencyGraph(
+bool swift::fine_grained_dependencies::writeFineGrainedDependencyGraphToPath(
     DiagnosticEngine &diags, StringRef path,
     const SourceFileDepGraph &g) {
   PrettyStackTraceStringAction stackTrace("saving fine-grained dependency graph", path);
   return withOutputFile(diags, path, [&](llvm::raw_ostream &out) {
-    Serializer serializer;
-    serializer.writeFineGrainedDependencyGraph(out, g);
+    SmallVector<char, 0> Buffer;
+    llvm::BitstreamWriter Writer{Buffer};
+    writeFineGrainedDependencyGraph(Writer, g);
+    out.write(Buffer.data(), Buffer.size());
+    out.flush();
     return false;
   });
-}
+}

lib/AST/FrontendSourceFileDepGraphFactory.cpp

@@ -236,30 +236,20 @@ std::string DependencyKey::computeNameForProvidedEntity<
 // MARK: Entry point into frontend graph construction
 //==============================================================================
 
-bool fine_grained_dependencies::emitReferenceDependencies(
-    DiagnosticEngine &diags, SourceFile *const SF,
-    const DependencyTracker &depTracker,
-    StringRef outputPath,
-    const bool alsoEmitDotFile) {
-
-  // Before writing to the dependencies file path, preserve any previous file
-  // that may have been there. No error handling -- this is just a nicety, it
-  // doesn't matter if it fails.
-  llvm::sys::fs::rename(outputPath, outputPath + "~");
-
-  SourceFileDepGraph g = FrontendSourceFileDepGraphFactory(
-                             SF, outputPath, depTracker, alsoEmitDotFile)
-                              .construct();
-
-  bool hadError = writeFineGrainedDependencyGraph(diags, outputPath, g);
-
-  // If path is stdout, cannot read it back, so check for "-"
-  assert(outputPath == "-" || g.verifyReadsWhatIsWritten(outputPath));
-
-  if (alsoEmitDotFile)
-    g.emitDotFile(outputPath, diags);
-
-  return hadError;
+bool fine_grained_dependencies::withReferenceDependencies(
+    llvm::PointerUnion<ModuleDecl *, SourceFile *> MSF,
+    const DependencyTracker &depTracker, StringRef outputPath,
+    bool alsoEmitDotFile,
+    llvm::function_ref<bool(SourceFileDepGraph &&)> cont) {
+  if (MSF.dyn_cast<ModuleDecl *>()) {
+    llvm_unreachable("Cannot construct dependency graph for modules!");
+  } else {
+    auto *SF = MSF.get<SourceFile *>();
+    SourceFileDepGraph g = FrontendSourceFileDepGraphFactory(
+                               SF, outputPath, depTracker, alsoEmitDotFile)
+                               .construct();
+    return cont(std::move(g));
+  }
 }
 
 //==============================================================================

lib/AST/Module.cpp

@@ -484,6 +484,7 @@ ModuleDecl::ModuleDecl(Identifier name, ASTContext &ctx,
   Bits.ModuleDecl.IsSystemModule = 0;
   Bits.ModuleDecl.IsNonSwiftModule = 0;
   Bits.ModuleDecl.IsMainModule = 0;
+  Bits.ModuleDecl.HasIncrementalInfo = 0;
 }
 
 ArrayRef<ImplicitImport> ModuleDecl::getImplicitImports() const {

lib/AST/NameLookupRequests.cpp

@@ -300,16 +300,15 @@ void DirectLookupRequest::writeDependencySink(
 
 void LookupInModuleRequest::writeDependencySink(
     evaluator::DependencyCollector &reqTracker, QualifiedLookupResult l) const {
-  auto *module = std::get<0>(getStorage());
+  auto *DC = std::get<0>(getStorage());
   auto member = std::get<1>(getStorage());
-  auto *DC = std::get<4>(getStorage());
 
-  // Decline to record lookups outside our module.
-  if (!DC->getParentSourceFile() ||
-      module->getParentModule() != DC->getParentModule()) {
-    return;
+  // Decline to record lookups if the module in question has no incremental
+  // dependency information available.
+  auto *module = DC->getParentModule();
+  if (module->isMainModule() || module->hasIncrementalInfo()) {
+    reqTracker.addTopLevelName(member.getBaseName());
   }
-  reqTracker.addTopLevelName(member.getBaseName());
 }
 
 //----------------------------------------------------------------------------//
@@ -343,16 +342,14 @@ swift::extractNearestSourceLoc(const LookupConformanceDescriptor &desc) {
 
 void ModuleQualifiedLookupRequest::writeDependencySink(
     evaluator::DependencyCollector &reqTracker, QualifiedLookupResult l) const {
-  auto *DC = std::get<0>(getStorage());
   auto *module = std::get<1>(getStorage());
   auto member = std::get<2>(getStorage());
 
-  // Decline to record lookups outside our module.
-  if (!DC->getParentSourceFile() ||
-      module != DC->getModuleScopeContext()->getParentModule()) {
-    return;
+  // Decline to record lookups if the module in question has no incremental
+  // dependency information available.
+  if (module->isMainModule() || module->hasIncrementalInfo()) {
+    reqTracker.addTopLevelName(member.getBaseName());
   }
-  reqTracker.addTopLevelName(member.getBaseName());
 }
 
 //----------------------------------------------------------------------------//
@@ -370,16 +367,13 @@ void LookupConformanceInModuleRequest::writeDependencySink(
   if (!Adoptee)
     return;
 
-  auto source = reqTracker.getRecorder().getActiveDependencySourceOrNull();
-  if (source.isNull())
-    return;
-
-  // Decline to record conformances defined outside of the active module.
+  // Decline to record lookups if the module in question has no incremental
+  // dependency information available.
   auto *conformance = lookupResult.getConcrete();
-  if (source.get()->getParentModule() !=
-      conformance->getDeclContext()->getParentModule())
-    return;
-  reqTracker.addPotentialMember(Adoptee);
+  auto *module = conformance->getDeclContext()->getParentModule();
+  if (module->isMainModule() || module->hasIncrementalInfo()) {
+    reqTracker.addPotentialMember(Adoptee);
+  }
 }
 
 //----------------------------------------------------------------------------//

lib/FrontendTool/FrontendTool.cpp

@@ -30,6 +30,7 @@
 #include "swift/AST/DiagnosticsSema.h"
 #include "swift/AST/FileSystem.h"
 #include "swift/AST/FineGrainedDependencies.h"
+#include "swift/AST/FineGrainedDependencyFormat.h"
 #include "swift/AST/GenericSignatureBuilder.h"
 #include "swift/AST/IRGenOptions.h"
 #include "swift/AST/IRGenRequests.h"
@@ -1370,6 +1371,35 @@ static bool dumpAST(CompilerInstance &Instance) {
   return Instance.getASTContext().hadError();
 }
 
+static bool emitReferenceDependencies(CompilerInstance &Instance,
+                                      SourceFile *const SF,
+                                      StringRef outputPath) {
+  const auto alsoEmitDotFile = Instance.getInvocation()
+                                   .getLangOptions()
+                                   .EmitFineGrainedDependencySourcefileDotFiles;
+
+  // Before writing to the dependencies file path, preserve any previous file
+  // that may have been there. No error handling -- this is just a nicety, it
+  // doesn't matter if it fails.
+  llvm::sys::fs::rename(outputPath, outputPath + "~");
+
+  using SourceFileDepGraph = fine_grained_dependencies::SourceFileDepGraph;
+  return fine_grained_dependencies::withReferenceDependencies(
+      SF, *Instance.getDependencyTracker(), outputPath, alsoEmitDotFile,
+      [&](SourceFileDepGraph &&g) -> bool {
+        const bool hadError =
+            fine_grained_dependencies::writeFineGrainedDependencyGraphToPath(
+                Instance.getDiags(), outputPath, g);
+
+        // If path is stdout, cannot read it back, so check for "-"
+        assert(outputPath == "-" || g.verifyReadsWhatIsWritten(outputPath));
+
+        if (alsoEmitDotFile)
+          g.emitDotFile(outputPath, Instance.getDiags());
+        return hadError;
+      });
+}
+
 static void emitReferenceDependenciesForAllPrimaryInputsIfNeeded(
     CompilerInstance &Instance) {
   const auto &Invocation = Instance.getInvocation();
@@ -1384,13 +1414,11 @@ static void emitReferenceDependenciesForAllPrimaryInputsIfNeeded(
     const std::string &referenceDependenciesFilePath =
         Invocation.getReferenceDependenciesFilePathForPrimary(
             SF->getFilename());
-    if (!referenceDependenciesFilePath.empty()) {
-      const auto LangOpts = Invocation.getLangOptions();
-      (void)fine_grained_dependencies::emitReferenceDependencies(
-          Instance.getDiags(), SF, *Instance.getDependencyTracker(),
-          referenceDependenciesFilePath,
-          LangOpts.EmitFineGrainedDependencySourcefileDotFiles);
+    if (referenceDependenciesFilePath.empty()) {
+      continue;
     }
+
+    emitReferenceDependencies(Instance, SF, referenceDependenciesFilePath);
   }
 }
 static void

lib/Serialization/CMakeLists.txt

@@ -8,6 +8,7 @@ add_swift_host_library(swiftSerialization STATIC
   SerializedModuleLoader.cpp
   SerializedSILLoader.cpp
   SerializeDoc.cpp
+  SerializeIncremental.cpp
   SerializeSIL.cpp
 
   LLVM_LINK_COMPONENTS

lib/Serialization/ModuleFile.h

@@ -465,6 +465,9 @@ class ModuleFile
     return Core->Bits.IsImplicitDynamicEnabled;
   }
 
+  /// \c true if this module has incremental dependency information.
+  bool hasIncrementalInfo() const { return Core->hasIncrementalInfo(); }
+
   /// Associates this module file with the AST node representing it.
   ///
   /// Checks that the file is compatible with the AST module it's being loaded

lib/Serialization/ModuleFileSharedCore.cpp

@@ -1427,6 +1427,17 @@ ModuleFileSharedCore::ModuleFileSharedCore(
       break;
     }
 
+    case INCREMENTAL_INFORMATION_BLOCK_ID: {
+      HasIncrementalInfo = true;
+      // Skip incremental info if present. The Frontend currently doesn't do
+      // anything with this.
+      if (cursor.SkipBlock()) {
+        info.status = error(Status::Malformed);
+        return;
+      }
+      break;
+    }
+
     default:
       // Unknown top-level block, possibly for use by a future version of the
       // module format.