Repository: llvm/llvm-project
Pull request #135471: [Modules] Fix an identifier hiding a function-like macro definition.
Author: login: vsapsai
Labels: name: clang
description: Clang issues not falling into any other category
name: clang:modules
description: C++20 modules and Clang Header Modules
Created: 2025-04-12T03:39:02Z | Merged: 2025-04-16T17:14:05Z

We emit a macro definition only in a module defining it. But it means that if another module has an identifier with the same name as the macro, the users of such module won't be able to use the macro anymore.

Fix by storing that an identifier has a macro definition that's not in a current module (`MacroDirectivesOffset == 0`). This way `IdentifierLookupVisitor` knows not to stop at the first module with an identifier but to keep checking included modules for the actual macro definition.

Fixes issue #32040.

rdar://30258278


Diff:
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 02c31dff620ec..58d3d33b31644 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1131,7 +1131,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
   bool HasRevertedTokenIDToIdentifier = readBit(Bits);
   bool Poisoned = readBit(Bits);
   bool ExtensionToken = readBit(Bits);
-  bool HadMacroDefinition = readBit(Bits);
+  bool HasMacroDefinition = readBit(Bits);
 
   assert(Bits == 0 && "Extra bits in the identifier?");
   DataLen -= sizeof(uint16_t) * 2;
@@ -1151,14 +1151,17 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
          "Incorrect C++ operator keyword flag");
   (void)CPlusPlusOperatorKeyword;
 
-  // If this identifier is a macro, deserialize the macro
-  // definition.
-  if (HadMacroDefinition) {
+  // If this identifier has a macro definition, deserialize it or notify the
+  // visitor the actual definition is in a different module.
+  if (HasMacroDefinition) {
     uint32_t MacroDirectivesOffset =
         endian::readNext<uint32_t, llvm::endianness::little>(d);
     DataLen -= 4;
 
-    Reader.addPendingMacro(II, &F, MacroDirectivesOffset);
+    if (MacroDirectivesOffset)
+      Reader.addPendingMacro(II, &F, MacroDirectivesOffset);
+    else
+      hasMacroDefinitionInDependencies = true;
   }
 
   Reader.SetIdentifierInfo(ID, II);
@@ -2419,6 +2422,10 @@ namespace {
       // declarations it needs.
       ++NumIdentifierLookupHits;
       Found = *Pos;
+      if (Trait.hasMoreInformationInDependencies()) {
+        // Look for the identifier in extra modules as they contain more info.
+        return false;
+      }
       return true;
     }
 
diff --git a/clang/lib/Serialization/ASTReaderInternals.h b/clang/lib/Serialization/ASTReaderInternals.h
index 353e0a53cad9b..4a7794889b039 100644
--- a/clang/lib/Serialization/ASTReaderInternals.h
+++ b/clang/lib/Serialization/ASTReaderInternals.h
@@ -286,6 +286,8 @@ class ASTIdentifierLookupTrait : public ASTIdentifierLookupTraitBase {
   // identifier that was constructed before the AST file was read.
   IdentifierInfo *KnownII;
 
+  bool hasMacroDefinitionInDependencies = false;
+
 public:
   using data_type = IdentifierInfo *;
 
@@ -300,6 +302,10 @@ class ASTIdentifierLookupTrait : public ASTIdentifierLookupTraitBase {
   IdentifierID ReadIdentifierID(const unsigned char *d);
 
   ASTReader &getReader() const { return Reader; }
+
+  bool hasMoreInformationInDependencies() const {
+    return hasMacroDefinitionInDependencies;
+  }
 };
 
 /// The on-disk hash table used to contain information about
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index a48c05061626a..05a3b8f08029f 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -3792,7 +3792,10 @@ bool IsInterestingIdentifier(const IdentifierInfo *II, uint64_t MacroOffset,
       II->getNotableIdentifierID() != tok::NotableIdentifierKind::not_notable ||
       II->getBuiltinID() != Builtin::ID::NotBuiltin ||
       II->getObjCKeywordID() != tok::ObjCKeywordKind::objc_not_keyword;
-  if (MacroOffset || II->isPoisoned() || (!IsModule && IsInteresting) ||
+  if (MacroOffset ||
+      (II->hasMacroDefinition() &&
+       II->hasFETokenInfoChangedSinceDeserialization()) ||
+      II->isPoisoned() || (!IsModule && IsInteresting) ||
       II->hasRevertedTokenIDToIdentifier() ||
       (NeedDecls && II->getFETokenInfo()))
     return true;
@@ -3871,7 +3874,8 @@ class ASTIdentifierTableTrait {
     if (isInterestingIdentifier(II, MacroOffset)) {
       DataLen += 2; // 2 bytes for builtin ID
       DataLen += 2; // 2 bytes for flags
-      if (MacroOffset)
+      if (MacroOffset || (II->hasMacroDefinition() &&
+                          II->hasFETokenInfoChangedSinceDeserialization()))
         DataLen += 4; // MacroDirectives offset.
 
       if (NeedDecls && IdResolver)
@@ -3902,15 +3906,17 @@ class ASTIdentifierTableTrait {
     assert((Bits & 0xffff) == Bits && "ObjCOrBuiltinID too big for ASTReader.");
     LE.write<uint16_t>(Bits);
     Bits = 0;
-    bool HadMacroDefinition = MacroOffset != 0;
-    Bits = (Bits << 1) | unsigned(HadMacroDefinition);
+    bool HasMacroDefinition =
+        (MacroOffset != 0) || (II->hasMacroDefinition() &&
+                               II->hasFETokenInfoChangedSinceDeserialization());
+    Bits = (Bits << 1) | unsigned(HasMacroDefinition);
     Bits = (Bits << 1) | unsigned(II->isExtensionToken());
     Bits = (Bits << 1) | unsigned(II->isPoisoned());
     Bits = (Bits << 1) | unsigned(II->hasRevertedTokenIDToIdentifier());
     Bits = (Bits << 1) | unsigned(II->isCPlusPlusOperatorKeyword());
     LE.write<uint16_t>(Bits);
 
-    if (HadMacroDefinition)
+    if (HasMacroDefinition)
       LE.write<uint32_t>(MacroOffset);
 
     if (NeedDecls && IdResolver) {
diff --git a/clang/test/Modules/macro-identifier-hiding.c b/clang/test/Modules/macro-identifier-hiding.c
new file mode 100644
index 0000000000000..4cd7cf0500322
--- /dev/null
+++ b/clang/test/Modules/macro-identifier-hiding.c
@@ -0,0 +1,29 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/modules.cache \
+// RUN:            -fsyntax-only %t/test.c -verify
+// Test again with the populated module cache.
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/modules.cache \
+// RUN:            -fsyntax-only %t/test.c -verify
+
+// Test that an identifier with the same name as a macro doesn't hide this
+// macro from the includers.
+
+//--- macro-definition.h
+#define __P(protos) ()
+#define __Q(protos) ()
+
+//--- macro-transitive.h
+#include "macro-definition.h"
+void test(int __P) {} // not "interesting" identifier
+struct __Q {};        // "interesting" identifier
+
+//--- module.modulemap
+module MacroDefinition { header "macro-definition.h" export * }
+module MacroTransitive { header "macro-transitive.h" export * }
+
+//--- test.c
+// expected-no-diagnostics
+#include "macro-transitive.h"
+void foo __P(());
+void bar __Q(());


Reviews:
user: login: vgvassilev
body: The patch looks reasonable to me, however, I'd wait for somebody with more experience in the macro logic in modules.
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-14T17:50:17Z
user: login: jansvoboda11
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-14T17:58:31Z
user: login: vsapsai
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-14T21:16:16Z
user: login: vsapsai
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-14T22:05:39Z
user: login: jansvoboda11
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-14T22:26:56Z
user: login: jansvoboda11
body: LGTM. The behavior is a bit subtle (always was), but it's good to have this edge-case fixed.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-14T22:30:01Z
user: login: jansvoboda11
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-14T22:30:43Z
user: login: vsapsai
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-15T21:15:42Z

Review comments:
diff_hunk: @@ -3871,7 +3874,8 @@ class ASTIdentifierTableTrait {
     if (isInterestingIdentifier(II, MacroOffset)) {
       DataLen += 2; // 2 bytes for builtin ID
       DataLen += 2; // 2 bytes for flags
-      if (MacroOffset)
+      if (MacroOffset || (II->hasMacroDefinition() &&
+                          II->hasFETokenInfoChangedSinceDeserialization()))
path: clang/lib/Serialization/ASTWriter.cpp
user: login: jansvoboda11
body: The way this condition works makes me think of this situation:

```
//--- macro-definition.h
#define __P(protos) ()
#define __Q(protos) ()

//--- macro-transitive.h
void test(int __P) {} // not "interesting" identifier
struct __Q {};        // "interesting" identifier
#include "macro-definition.h" // NOTE: Import of the macros module
                              // comes after local declarations.

//--- module.modulemap
module MacroDefinition { header "macro-definition.h" export * }
module MacroTransitive { header "macro-transitive.h" export * }

//--- test.c
// expected-no-diagnostics
#include "macro-transitive.h"
void foo __P(());
void bar __Q(());
```

When writing the PCM for the MacroTransitive module, I'd expect `II->hasFETokenInfoChangedSinceDeserialization()` to be false and `ASTWriter` therefore skipping serializing the  fact that `__P` and `__Q` have macros attached to them. How is this working out in this case? This modified test passes, just curious why.
created_at: 2025-04-14T17:58:25Z
author_association: CONTRIBUTOR
path: clang/test/Modules/shadow-macro.c
user: login: vsapsai
body: By the way, looked at the other tests in Modules directory and think that the test should start with "macro-" for consistency. Probably something like "macro-identifier-hiding.c" as we have "macro-hiding.cpp" already.
created_at: 2025-04-14T21:16:16Z
author_association: COLLABORATOR
diff_hunk: @@ -3871,7 +3874,8 @@ class ASTIdentifierTableTrait {
     if (isInterestingIdentifier(II, MacroOffset)) {
       DataLen += 2; // 2 bytes for builtin ID
       DataLen += 2; // 2 bytes for flags
-      if (MacroOffset)
+      if (MacroOffset || (II->hasMacroDefinition() &&
+                          II->hasFETokenInfoChangedSinceDeserialization()))
path: clang/lib/Serialization/ASTWriter.cpp
user: login: vsapsai
body: This is a really good question, thanks for noticing and asking it.

The suggested change worked correctly even before my change. In my debugging function parameter "__P" isn't emitted in MacroTransitive because that's not an "interesting" identifier. So during the module traversal it doesn't prevent us from visiting MacroDefinition and discovering the macro.

Struct "__Q" does get serialized in MacroTransitive but it has non-zero MacroOffset, so we serialize the macro definition in MacroTransitive. And MacroOffset is non-zero because we've added "__Q" to `IdentMacroDirectivesOffsetMap` and that's because in the snippet
```c++
    for (auto &Id : PP.getIdentifierTable())
      if (Id.second->hadMacroDefinition() &&
          (!Id.second->isFromAST() ||
          Id.second->hasChangedSinceDeserialization()))
        MacroIdentifiers.push_back(Id.second);
```
`hasChangedSinceDeserialization()` returns true for "__Q". Which I believe is triggered by ASTReader in
```c++
static void markIdentifierFromAST(ASTReader &Reader, IdentifierInfo &II,
                                  bool IsModule) {
  if (!II.isFromAST()) {
    II.setIsFromAST();
    if (isInterestingIdentifier(Reader, II, IsModule))
      II.setChangedSinceDeserialization();
  }
}
```
created_at: 2025-04-14T22:05:39Z
author_association: COLLABORATOR
diff_hunk: @@ -3871,7 +3874,8 @@ class ASTIdentifierTableTrait {
     if (isInterestingIdentifier(II, MacroOffset)) {
       DataLen += 2; // 2 bytes for builtin ID
       DataLen += 2; // 2 bytes for flags
-      if (MacroOffset)
+      if (MacroOffset || (II->hasMacroDefinition() &&
+                          II->hasFETokenInfoChangedSinceDeserialization()))
path: clang/lib/Serialization/ASTWriter.cpp
user: login: jansvoboda11
body: Ah, I see that `ASTReader` does this for each `ModuleFile`:
```c++
    for (auto Offset : F.PreloadIdentifierOffsets) {
      // ...
      IdentifierInfo *II;
      if (!PP.getLangOpts().CPlusPlus) {
        // Identifiers present in both the module file and the importing
        // instance are marked out-of-date so that they can be deserialized
        // on next use via ASTReader::updateOutOfDateIdentifier().
        // Identifiers present in the module file but not in the importing
        // instance are ignored for now, preventing growth of the identifier
        // table. They will be deserialized on first use via ASTReader::get().
        auto It = PP.getIdentifierTable().find(Key);
        if (It == PP.getIdentifierTable().end())
          continue;
        II = It->second;
      } else {
        // With C++ modules, not many identifiers are considered interesting.
        // All identifiers in the module file can be placed into the identifier
        // table of the importing instance and marked as out-of-date. This makes
        // ASTReader::get() a no-op, and deserialization will take place on
        // first/next use via ASTReader::updateOutOfDateIdentifier().
        II = &PP.getIdentifierTable().getOwn(Key);
      }
      // ...
      markIdentifierFromAST(*this, *II, /*IsModule=*/true);
      //...
    }
```
and "__Q" already is in the table, so that works out. Makes sense, thanks for walking me through it.
created_at: 2025-04-14T22:26:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2419,6 +2422,9 @@ namespace {
       // declarations it needs.
       ++NumIdentifierLookupHits;
       Found = *Pos;
+      if (Trait.hasMoreInformationInDependencies())
+        // Look for the identifier in extra modules as they contain more info.
+        return false;
path: clang/lib/Serialization/ASTReader.cpp
user: login: jansvoboda11
body: Nit: I'd wrap this in braces since it spans two lines.
created_at: 2025-04-14T22:30:43Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2419,6 +2422,9 @@ namespace {
       // declarations it needs.
       ++NumIdentifierLookupHits;
       Found = *Pos;
+      if (Trait.hasMoreInformationInDependencies())
+        // Look for the identifier in extra modules as they contain more info.
+        return false;
path: clang/lib/Serialization/ASTReader.cpp
user: login: vsapsai
body: Done.
created_at: 2025-04-15T21:15:41Z
author_association: COLLABORATOR

PR comments:
user: login: llvmbot
created_at: 2025-04-12T03:39:37Z
body: 
<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-clang

Author: Volodymyr Sapsai (vsapsai)

<details>
<summary>Changes</summary>

We emit a macro definition only in a module defining it. But it means that if another module has an identifier with the same name as the macro, the users of such module won't be able to use the macro anymore.

Fix by storing that an identifier has a macro definition that's not in a current module (`MacroDirectivesOffset == 0`). This way `IdentifierLookupVisitor` knows not to stop at the first module with an identifier but to keep checking included modules for the actual macro definition.

Fixes issue #<!-- -->32040.

rdar://30258278

---
Full diff: https://github.com/llvm/llvm-project/pull/135471.diff


4 Files Affected:

- (modified) clang/lib/Serialization/ASTReader.cpp (+11-5) 
- (modified) clang/lib/Serialization/ASTReaderInternals.h (+6) 
- (modified) clang/lib/Serialization/ASTWriter.cpp (+11-5) 
- (added) clang/test/Modules/shadow-macro.c (+29) 


``````````diff
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 02c31dff620ec..e8a3c148f9f61 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1131,7 +1131,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
   bool HasRevertedTokenIDToIdentifier = readBit(Bits);
   bool Poisoned = readBit(Bits);
   bool ExtensionToken = readBit(Bits);
-  bool HadMacroDefinition = readBit(Bits);
+  bool HasMacroDefinition = readBit(Bits);
 
   assert(Bits == 0 && "Extra bits in the identifier?");
   DataLen -= sizeof(uint16_t) * 2;
@@ -1151,14 +1151,17 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
          "Incorrect C++ operator keyword flag");
   (void)CPlusPlusOperatorKeyword;
 
-  // If this identifier is a macro, deserialize the macro
-  // definition.
-  if (HadMacroDefinition) {
+  // If this identifier has a macro definition, deserialize it or notify the
+  // visitor the actual definition is in a different module.
+  if (HasMacroDefinition) {
     uint32_t MacroDirectivesOffset =
         endian::readNext<uint32_t, llvm::endianness::little>(d);
     DataLen -= 4;
 
-    Reader.addPendingMacro(II, &F, MacroDirectivesOffset);
+    if (MacroDirectivesOffset)
+      Reader.addPendingMacro(II, &F, MacroDirectivesOffset);
+    else
+      hasMacroDefinitionInDependencies = true;
   }
 
   Reader.SetIdentifierInfo(ID, II);
@@ -2419,6 +2422,9 @@ namespace {
       // declarations it needs.
       ++NumIdentifierLookupHits;
       Found = *Pos;
+      if (Trait.hasMoreInformationInDependencies())
+        // Look for the identifier in extra modules as they contain more info.
+        return false;
       return true;
     }
 
diff --git a/clang/lib/Serialization/ASTReaderInternals.h b/clang/lib/Serialization/ASTReaderInternals.h
index 353e0a53cad9b..4a7794889b039 100644
--- a/clang/lib/Serialization/ASTReaderInternals.h
+++ b/clang/lib/Serialization/ASTReaderInternals.h
@@ -286,6 +286,8 @@ class ASTIdentifierLookupTrait : public ASTIdentifierLookupTraitBase {
   // identifier that was constructed before the AST file was read.
   IdentifierInfo *KnownII;
 
+  bool hasMacroDefinitionInDependencies = false;
+
 public:
   using data_type = IdentifierInfo *;
 
@@ -300,6 +302,10 @@ class ASTIdentifierLookupTrait : public ASTIdentifierLookupTraitBase {
   IdentifierID ReadIdentifierID(const unsigned char *d);
 
   ASTReader &getReader() const { return Reader; }
+
+  bool hasMoreInformationInDependencies() const {
+    return hasMacroDefinitionInDependencies;
+  }
 };
 
 /// The on-disk hash table used to contain information about
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index a48c05061626a..05a3b8f08029f 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -3792,7 +3792,10 @@ bool IsInterestingIdentifier(const IdentifierInfo *II, uint64_t MacroOffset,
       II->getNotableIdentifierID() != tok::NotableIdentifierKind::not_notable ||
       II->getBuiltinID() != Builtin::ID::NotBuiltin ||
       II->getObjCKeywordID() != tok::ObjCKeywordKind::objc_not_keyword;
-  if (MacroOffset || II->isPoisoned() || (!IsModule && IsInteresting) ||
+  if (MacroOffset ||
+      (II->hasMacroDefinition() &&
+       II->hasFETokenInfoChangedSinceDeserialization()) ||
+      II->isPoisoned() || (!IsModule && IsInteresting) ||
       II->hasRevertedTokenIDToIdentifier() ||
       (NeedDecls && II->getFETokenInfo()))
     return true;
@@ -3871,7 +3874,8 @@ class ASTIdentifierTableTrait {
     if (isInterestingIdentifier(II, MacroOffset)) {
       DataLen += 2; // 2 bytes for builtin ID
       DataLen += 2; // 2 bytes for flags
-      if (MacroOffset)
+      if (MacroOffset || (II->hasMacroDefinition() &&
+                          II->hasFETokenInfoChangedSinceDeserialization()))
         DataLen += 4; // MacroDirectives offset.
 
       if (NeedDecls && IdResolver)
@@ -3902,15 +3906,17 @@ class ASTIdentifierTableTrait {
     assert((Bits & 0xffff) == Bits && "ObjCOrBuiltinID too big for ASTReader.");
     LE.write<uint16_t>(Bits);
     Bits = 0;
-    bool HadMacroDefinition = MacroOffset != 0;
-    Bits = (Bits << 1) | unsigned(HadMacroDefinition);
+    bool HasMacroDefinition =
+        (MacroOffset != 0) || (II->hasMacroDefinition() &&
+                               II->hasFETokenInfoChangedSinceDeserialization());
+    Bits = (Bits << 1) | unsigned(HasMacroDefinition);
     Bits = (Bits << 1) | unsigned(II->isExtensionToken());
     Bits = (Bits << 1) | unsigned(II->isPoisoned());
     Bits = (Bits << 1) | unsigned(II->hasRevertedTokenIDToIdentifier());
     Bits = (Bits << 1) | unsigned(II->isCPlusPlusOperatorKeyword());
     LE.write<uint16_t>(Bits);
 
-    if (HadMacroDefinition)
+    if (HasMacroDefinition)
       LE.write<uint32_t>(MacroOffset);
 
     if (NeedDecls && IdResolver) {
diff --git a/clang/test/Modules/shadow-macro.c b/clang/test/Modules/shadow-macro.c
new file mode 100644
index 0000000000000..4cd7cf0500322
--- /dev/null
+++ b/clang/test/Modules/shadow-macro.c
@@ -0,0 +1,29 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/modules.cache \
+// RUN:            -fsyntax-only %t/test.c -verify
+// Test again with the populated module cache.
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/modules.cache \
+// RUN:            -fsyntax-only %t/test.c -verify
+
+// Test that an identifier with the same name as a macro doesn't hide this
+// macro from the includers.
+
+//--- macro-definition.h
+#define __P(protos) ()
+#define __Q(protos) ()
+
+//--- macro-transitive.h
+#include "macro-definition.h"
+void test(int __P) {} // not "interesting" identifier
+struct __Q {};        // "interesting" identifier
+
+//--- module.modulemap
+module MacroDefinition { header "macro-definition.h" export * }
+module MacroTransitive { header "macro-transitive.h" export * }
+
+//--- test.c
+// expected-no-diagnostics
+#include "macro-transitive.h"
+void foo __P(());
+void bar __Q(());

``````````

</details>

author_association: MEMBER
user: login: llvmbot
created_at: 2025-04-12T03:39:38Z
body: 
<!--LLVM PR SUMMARY COMMENT-->

@llvm/pr-subscribers-clang-modules

Author: Volodymyr Sapsai (vsapsai)

<details>
<summary>Changes</summary>

We emit a macro definition only in a module defining it. But it means that if another module has an identifier with the same name as the macro, the users of such module won't be able to use the macro anymore.

Fix by storing that an identifier has a macro definition that's not in a current module (`MacroDirectivesOffset == 0`). This way `IdentifierLookupVisitor` knows not to stop at the first module with an identifier but to keep checking included modules for the actual macro definition.

Fixes issue #<!-- -->32040.

rdar://30258278

---
Full diff: https://github.com/llvm/llvm-project/pull/135471.diff


4 Files Affected:

- (modified) clang/lib/Serialization/ASTReader.cpp (+11-5) 
- (modified) clang/lib/Serialization/ASTReaderInternals.h (+6) 
- (modified) clang/lib/Serialization/ASTWriter.cpp (+11-5) 
- (added) clang/test/Modules/shadow-macro.c (+29) 


``````````diff
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 02c31dff620ec..e8a3c148f9f61 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -1131,7 +1131,7 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
   bool HasRevertedTokenIDToIdentifier = readBit(Bits);
   bool Poisoned = readBit(Bits);
   bool ExtensionToken = readBit(Bits);
-  bool HadMacroDefinition = readBit(Bits);
+  bool HasMacroDefinition = readBit(Bits);
 
   assert(Bits == 0 && "Extra bits in the identifier?");
   DataLen -= sizeof(uint16_t) * 2;
@@ -1151,14 +1151,17 @@ IdentifierInfo *ASTIdentifierLookupTrait::ReadData(const internal_key_type& k,
          "Incorrect C++ operator keyword flag");
   (void)CPlusPlusOperatorKeyword;
 
-  // If this identifier is a macro, deserialize the macro
-  // definition.
-  if (HadMacroDefinition) {
+  // If this identifier has a macro definition, deserialize it or notify the
+  // visitor the actual definition is in a different module.
+  if (HasMacroDefinition) {
     uint32_t MacroDirectivesOffset =
         endian::readNext<uint32_t, llvm::endianness::little>(d);
     DataLen -= 4;
 
-    Reader.addPendingMacro(II, &F, MacroDirectivesOffset);
+    if (MacroDirectivesOffset)
+      Reader.addPendingMacro(II, &F, MacroDirectivesOffset);
+    else
+      hasMacroDefinitionInDependencies = true;
   }
 
   Reader.SetIdentifierInfo(ID, II);
@@ -2419,6 +2422,9 @@ namespace {
       // declarations it needs.
       ++NumIdentifierLookupHits;
       Found = *Pos;
+      if (Trait.hasMoreInformationInDependencies())
+        // Look for the identifier in extra modules as they contain more info.
+        return false;
       return true;
     }
 
diff --git a/clang/lib/Serialization/ASTReaderInternals.h b/clang/lib/Serialization/ASTReaderInternals.h
index 353e0a53cad9b..4a7794889b039 100644
--- a/clang/lib/Serialization/ASTReaderInternals.h
+++ b/clang/lib/Serialization/ASTReaderInternals.h
@@ -286,6 +286,8 @@ class ASTIdentifierLookupTrait : public ASTIdentifierLookupTraitBase {
   // identifier that was constructed before the AST file was read.
   IdentifierInfo *KnownII;
 
+  bool hasMacroDefinitionInDependencies = false;
+
 public:
   using data_type = IdentifierInfo *;
 
@@ -300,6 +302,10 @@ class ASTIdentifierLookupTrait : public ASTIdentifierLookupTraitBase {
   IdentifierID ReadIdentifierID(const unsigned char *d);
 
   ASTReader &getReader() const { return Reader; }
+
+  bool hasMoreInformationInDependencies() const {
+    return hasMacroDefinitionInDependencies;
+  }
 };
 
 /// The on-disk hash table used to contain information about
diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp
index a48c05061626a..05a3b8f08029f 100644
--- a/clang/lib/Serialization/ASTWriter.cpp
+++ b/clang/lib/Serialization/ASTWriter.cpp
@@ -3792,7 +3792,10 @@ bool IsInterestingIdentifier(const IdentifierInfo *II, uint64_t MacroOffset,
       II->getNotableIdentifierID() != tok::NotableIdentifierKind::not_notable ||
       II->getBuiltinID() != Builtin::ID::NotBuiltin ||
       II->getObjCKeywordID() != tok::ObjCKeywordKind::objc_not_keyword;
-  if (MacroOffset || II->isPoisoned() || (!IsModule && IsInteresting) ||
+  if (MacroOffset ||
+      (II->hasMacroDefinition() &&
+       II->hasFETokenInfoChangedSinceDeserialization()) ||
+      II->isPoisoned() || (!IsModule && IsInteresting) ||
       II->hasRevertedTokenIDToIdentifier() ||
       (NeedDecls && II->getFETokenInfo()))
     return true;
@@ -3871,7 +3874,8 @@ class ASTIdentifierTableTrait {
     if (isInterestingIdentifier(II, MacroOffset)) {
       DataLen += 2; // 2 bytes for builtin ID
       DataLen += 2; // 2 bytes for flags
-      if (MacroOffset)
+      if (MacroOffset || (II->hasMacroDefinition() &&
+                          II->hasFETokenInfoChangedSinceDeserialization()))
         DataLen += 4; // MacroDirectives offset.
 
       if (NeedDecls && IdResolver)
@@ -3902,15 +3906,17 @@ class ASTIdentifierTableTrait {
     assert((Bits & 0xffff) == Bits && "ObjCOrBuiltinID too big for ASTReader.");
     LE.write<uint16_t>(Bits);
     Bits = 0;
-    bool HadMacroDefinition = MacroOffset != 0;
-    Bits = (Bits << 1) | unsigned(HadMacroDefinition);
+    bool HasMacroDefinition =
+        (MacroOffset != 0) || (II->hasMacroDefinition() &&
+                               II->hasFETokenInfoChangedSinceDeserialization());
+    Bits = (Bits << 1) | unsigned(HasMacroDefinition);
     Bits = (Bits << 1) | unsigned(II->isExtensionToken());
     Bits = (Bits << 1) | unsigned(II->isPoisoned());
     Bits = (Bits << 1) | unsigned(II->hasRevertedTokenIDToIdentifier());
     Bits = (Bits << 1) | unsigned(II->isCPlusPlusOperatorKeyword());
     LE.write<uint16_t>(Bits);
 
-    if (HadMacroDefinition)
+    if (HasMacroDefinition)
       LE.write<uint32_t>(MacroOffset);
 
     if (NeedDecls && IdResolver) {
diff --git a/clang/test/Modules/shadow-macro.c b/clang/test/Modules/shadow-macro.c
new file mode 100644
index 0000000000000..4cd7cf0500322
--- /dev/null
+++ b/clang/test/Modules/shadow-macro.c
@@ -0,0 +1,29 @@
+// RUN: rm -rf %t
+// RUN: split-file %s %t
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/modules.cache \
+// RUN:            -fsyntax-only %t/test.c -verify
+// Test again with the populated module cache.
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps -fmodules-cache-path=%t/modules.cache \
+// RUN:            -fsyntax-only %t/test.c -verify
+
+// Test that an identifier with the same name as a macro doesn't hide this
+// macro from the includers.
+
+//--- macro-definition.h
+#define __P(protos) ()
+#define __Q(protos) ()
+
+//--- macro-transitive.h
+#include "macro-definition.h"
+void test(int __P) {} // not "interesting" identifier
+struct __Q {};        // "interesting" identifier
+
+//--- module.modulemap
+module MacroDefinition { header "macro-definition.h" export * }
+module MacroTransitive { header "macro-transitive.h" export * }
+
+//--- test.c
+// expected-no-diagnostics
+#include "macro-transitive.h"
+void foo __P(());
+void bar __Q(());

``````````

</details>

author_association: MEMBER
user: login: vsapsai
created_at: 2025-04-12T03:40:44Z
body: cc @bcardosolopes @vgvassilev as participants of the original bug discussion.
author_association: COLLABORATOR
user: login: vgvassilev
created_at: 2025-04-12T05:22:03Z
body: Thanks! Does the pr fix https://bugs.llvm.org//show_bug.cgi?id=32670
author_association: CONTRIBUTOR
user: login: vsapsai
created_at: 2025-04-14T17:12:33Z
body: > Thanks! Does the pr fix https://bugs.llvm.org//show_bug.cgi?id=32670

Nope. The test case in #32017 is still failing. I believe it is a different issue as we don't have any identifiers with the same names as macros. And seems like the behavior wasn't affected by d79514e24b4da111b6b08535f975ae8f4d81c26d (aka r259901) while #32040 was exposed by that change.
author_association: COLLABORATOR