[C++20] [Modules] Embed all source files for C++20 Modules (#102444)

Close #72383

The implementation rationale is, I don't want to pass
`-fmodules-embed-all-files` all the time since we can't test it in lit
tests (we're using `clang_cc1`). So I tried to set it in FrontendActions
for modules.

Author: ChuanqiXu9

clang/include/clang/CodeGen/CodeGenAction.h

@@ -57,7 +57,7 @@ class CodeGenAction : public ASTFrontendAction {
   bool loadLinkModules(CompilerInstance &CI);
 
 protected:
-  bool BeginSourceFileAction(CompilerInstance &CI) override;
+  bool BeginInvocation(CompilerInstance &CI) override;
 
   /// Create a new code generation action.  If the optional \p _VMContext
   /// parameter is supplied, the action uses it without taking ownership,

clang/include/clang/Frontend/FrontendActions.h

@@ -152,11 +152,13 @@ class GenerateModuleFromModuleMapAction : public GenerateModuleAction {
   CreateOutputFile(CompilerInstance &CI, StringRef InFile) override;
 };
 
+bool BeginInvocationForModules(CompilerInstance &CI);
+
 /// Generates full BMI (which contains full information to generate the object
 /// files) for C++20 Named Modules.
 class GenerateModuleInterfaceAction : public GenerateModuleAction {
 protected:
-  bool BeginSourceFileAction(CompilerInstance &CI) override;
+  bool BeginInvocation(CompilerInstance &CI) override;
 
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
                                                  StringRef InFile) override;

clang/include/clang/Serialization/ModuleFile.h

@@ -88,13 +88,13 @@ class InputFile {
 
   InputFile(FileEntryRef File, bool isOverridden = false,
             bool isOutOfDate = false) {
-    assert(!(isOverridden && isOutOfDate) &&
-           "an overridden cannot be out-of-date");
     unsigned intVal = 0;
-    if (isOverridden)
-      intVal = Overridden;
-    else if (isOutOfDate)
+    // Make isOutOfDate with higher priority than isOverridden.
+    // It is possible if the recorded hash value mismatches.
+    if (isOutOfDate)
       intVal = OutOfDate;
+    else if (isOverridden)
+      intVal = Overridden;
     Val.setPointerAndInt(&File.getMapEntry(), intVal);
   }
 

clang/lib/CodeGen/CodeGenAction.cpp

@@ -969,9 +969,10 @@ CodeGenerator *CodeGenAction::getCodeGenerator() const {
   return BEConsumer->getCodeGenerator();
 }
 
-bool CodeGenAction::BeginSourceFileAction(CompilerInstance &CI) {
+bool CodeGenAction::BeginInvocation(CompilerInstance &CI) {
   if (CI.getFrontendOpts().GenReducedBMI)
-    CI.getLangOpts().setCompilingModule(LangOptions::CMK_ModuleInterface);
+    return BeginInvocationForModules(CI);
+
   return true;
 }
 

clang/lib/Frontend/FrontendActions.cpp

@@ -262,11 +262,20 @@ GenerateModuleFromModuleMapAction::CreateOutputFile(CompilerInstance &CI,
                                     /*ForceUseTemporary=*/true);
 }
 
-bool GenerateModuleInterfaceAction::BeginSourceFileAction(
-    CompilerInstance &CI) {
+bool clang::BeginInvocationForModules(CompilerInstance &CI) {
+  // Embed all module files for named modules.
+  // See https://github.com/llvm/llvm-project/issues/72383 for discussion.
+  CI.getFrontendOpts().ModulesEmbedAllFiles = true;
   CI.getLangOpts().setCompilingModule(LangOptions::CMK_ModuleInterface);
+  return true;
+}
 
-  return GenerateModuleAction::BeginSourceFileAction(CI);
+bool GenerateModuleInterfaceAction::BeginInvocation(
+    CompilerInstance &CI) {
+  if (!BeginInvocationForModules(CI))
+    return false;
+
+  return GenerateModuleAction::BeginInvocation(CI);
 }
 
 std::unique_ptr<ASTConsumer>

clang/test/Modules/no-local-decl-in-reduced-bmi.cppm

@@ -6,11 +6,11 @@
 //
 // RUN: %clang_cc1 -std=c++20 %t/a.cppm -emit-reduced-module-interface -o %t/a.pcm
 // RUN: llvm-bcanalyzer --dump --disable-histogram --show-binary-blobs %t/a.pcm > %t/a.dump
-// RUN: cat %t/a.dump | FileCheck %t/a.cppm
+// RUN: cat %t/a.dump | FileCheck %t/a.check
 //
 // RUN: %clang_cc1 -std=c++20 %t/b.cppm -emit-reduced-module-interface -o %t/b.pcm
 // RUN: llvm-bcanalyzer --dump --disable-histogram --show-binary-blobs %t/b.pcm > %t/b.dump
-// RUN: cat %t/b.dump | FileCheck %t/b.cppm
+// RUN: cat %t/b.dump | FileCheck %t/b.check
 
 //--- a.cppm
 export module a;
@@ -19,6 +19,9 @@ export int func() {
     return 43;
 }
 
+//--- a.check
+// Use a standalone check file since now we're going to embed all source files in the BMI
+// so we will check the `CHECK-NOT: <DECL_VAR` in the source file otherwise.
 // Test that the variable declaration is not recorded completely.
 // CHECK-NOT: <DECL_VAR
 
@@ -29,5 +32,6 @@ export inline int func() {
     return v;
 }
 
+//--- b.check
 // Check that we still records the declaration from inline functions.
 // CHECK: <DECL_VAR

clang/test/Modules/reduced-bmi-empty-module-purview-std.cppm

@@ -7,7 +7,7 @@
 //
 // RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm
 // RUN: llvm-bcanalyzer --dump --disable-histogram --show-binary-blobs %t/A.pcm > %t/A.dump
-// RUN: cat %t/A.dump | FileCheck %t/A.cppm
+// RUN: cat %t/A.dump | FileCheck %t/A.check
 
 //--- std.h
 namespace std {
@@ -22,6 +22,8 @@ module;
 #include "std.h"
 export module A;
 
+
+//--- A.check
 // CHECK-NOT: <DECL_NAMESPACE
 // CHECK-NOT: <DECL_CONTEXT_LEXICAL
 // CHECK-NOT: <DELAYED_NAMESPACE_LEXICAL_VISIBLE_RECORD

clang/test/Modules/reduced-bmi-empty-module-purview.cppm

@@ -9,12 +9,12 @@
 // RUN: %clang_cc1 -std=c++20 %t/A.cppm -emit-reduced-module-interface -o %t/A.pcm \
 // RUN:     -fmodule-file=M=%t/M.pcm
 // RUN: llvm-bcanalyzer --dump --disable-histogram --show-binary-blobs %t/A.pcm > %t/A.dump
-// RUN: cat %t/A.dump | FileCheck %t/A.cppm
+// RUN: cat %t/A.dump | FileCheck %t/A.check
 //
 // RUN: %clang_cc1 -std=c++20 %t/A1.cppm -emit-reduced-module-interface -o %t/A1.pcm \
 // RUN:     -fmodule-file=M=%t/M.pcm
 // RUN: llvm-bcanalyzer --dump --disable-histogram --show-binary-blobs %t/A1.pcm > %t/A1.dump
-// RUN: cat %t/A1.dump | FileCheck %t/A1.cppm
+// RUN: cat %t/A1.dump | FileCheck %t/A1.check
 
 //--- foo.h
 namespace ns {
@@ -82,6 +82,7 @@ module;
 export module A;
 import M;
 
+//--- A.check
 // CHECK-NOT: <DECL_CXX_RECORD
 // CHECK-NOT: <DECL_UPDATE_OFFSETS
 
@@ -91,6 +92,7 @@ import M;
 #include "foo.h"
 export module A;
 
+//--- A1.check
 // CHECK-NOT: <DECL_CXX_RECORD
 // CHECK-NOT: <DECL_UPDATE_OFFSETS
 

clang/test/Modules/unreached-static-entities.cppm

@@ -3,11 +3,13 @@
 //
 // RUN: rm -rf %t
 // RUN: mkdir -p %t
+// RUN: split-file %s %t
 //
-// RUN: %clang_cc1 -std=c++20 %s -emit-reduced-module-interface -o %t/S.pcm
+// RUN: %clang_cc1 -std=c++20 %t/S.cppm -emit-reduced-module-interface -o %t/S.pcm
 // RUN: llvm-bcanalyzer --dump --disable-histogram --show-binary-blobs %t/S.pcm > %t/S.dump
-// RUN: cat %t/S.dump | FileCheck %s
+// RUN: cat %t/S.dump | FileCheck %t/S.check
 
+//--- S.cppm
 export module S;
 static int static_func() {
     return 43;
@@ -17,6 +19,7 @@ export int func() {
     return static_func();
 }
 
+//--- S.check
 // CHECK: <DECL_FUNCTION
 // Checks that we won't see a second function
 // CHECK-NOT: <DECL_FUNCTION