Add invisible open delimiters to TokenDescription.

Invisible delimiters pretty-print as empty strings, and changing that
can break some proc macros. But error messages saying "expected
identifer, found ``" are bad. So this commit adds support for invisible
open delimiters in `TokenDescription` so they print as "invisible open
delimiter" in error messages, instead of "``".

It's not used meaningfully yet, but will be needed to get rid of interpolated
tokens.

Author: nnethercote

compiler/rustc_parse/messages.ftl

@@ -204,6 +204,9 @@ parse_expected_identifier = expected identifier
 
 parse_expected_identifier_found_doc_comment = expected identifier, found doc comment
 parse_expected_identifier_found_doc_comment_str = expected identifier, found doc comment `{$token}`
+parse_expected_identifier_found_invisible_open_delim = expected identifier, found invisible open delimiter
+# Deliberately doesn't print `$token`, which is empty.
+parse_expected_identifier_found_invisible_open_delim_str = expected identifier, found invisible open delimiter
 parse_expected_identifier_found_keyword = expected identifier, found keyword
 parse_expected_identifier_found_keyword_str = expected identifier, found keyword `{$token}`
 parse_expected_identifier_found_reserved_identifier = expected identifier, found reserved identifier
@@ -216,6 +219,8 @@ parse_expected_mut_or_const_in_raw_pointer_type = expected `mut` or `const` keyw
     .suggestion = add `mut` or `const` here
 
 parse_expected_semi_found_doc_comment_str = expected `;`, found doc comment `{$token}`
+# Deliberately doesn't print `$token`, which is empty.
+parse_expected_semi_found_invisible_open_delim_str = expected `;`, found invisible open delimiter
 parse_expected_semi_found_keyword_str = expected `;`, found keyword `{$token}`
 parse_expected_semi_found_reserved_identifier_str = expected `;`, found reserved identifier `{$token}`
 parse_expected_semi_found_reserved_keyword_str = expected `;`, found reserved keyword `{$token}`
@@ -797,6 +802,8 @@ parse_unexpected_token_after_not_default = use `!` to perform logical negation o
 parse_unexpected_token_after_not_logical = use `!` to perform logical negation
 parse_unexpected_token_after_struct_name = expected `where`, `{"{"}`, `(`, or `;` after struct name
 parse_unexpected_token_after_struct_name_found_doc_comment = expected `where`, `{"{"}`, `(`, or `;` after struct name, found doc comment `{$token}`
+# Deliberately doesn't print `$token`, which is empty.
+parse_unexpected_token_after_struct_name_found_invisible_open_delim = expected `where`, `{"{"}`, `(`, or `;` after struct name, found invisible open delim
 parse_unexpected_token_after_struct_name_found_keyword = expected `where`, `{"{"}`, `(`, or `;` after struct name, found keyword `{$token}`
 parse_unexpected_token_after_struct_name_found_other = expected `where`, `{"{"}`, `(`, or `;` after struct name, found `{$token}`
 

compiler/rustc_parse/src/errors.rs

@@ -948,6 +948,8 @@ pub(crate) enum ExpectedIdentifierFound {
     #[label(parse_expected_identifier_found_doc_comment)]
     DocComment(#[primary_span] Span),
     #[label(parse_expected_identifier)]
+    InvisibleOpenDelim(#[primary_span] Span),
+    #[label(parse_expected_identifier)]
     Other(#[primary_span] Span),
 }
 
@@ -960,6 +962,9 @@ impl ExpectedIdentifierFound {
             Some(TokenDescription::Keyword) => ExpectedIdentifierFound::Keyword,
             Some(TokenDescription::ReservedKeyword) => ExpectedIdentifierFound::ReservedKeyword,
             Some(TokenDescription::DocComment) => ExpectedIdentifierFound::DocComment,
+            Some(TokenDescription::InvisibleOpenDelim) => {
+                ExpectedIdentifierFound::InvisibleOpenDelim
+            }
             None => ExpectedIdentifierFound::Other,
         })(span)
     }
@@ -992,6 +997,9 @@ impl<'a, G: EmissionGuarantee> IntoDiagnostic<'a, G> for ExpectedIdentifier {
             Some(TokenDescription::DocComment) => {
                 fluent::parse_expected_identifier_found_doc_comment_str
             }
+            Some(TokenDescription::InvisibleOpenDelim) => {
+                fluent::parse_expected_identifier_found_invisible_open_delim_str
+            }
             None => fluent::parse_expected_identifier_found_str,
         });
         diag.set_span(self.span);
@@ -1047,6 +1055,9 @@ impl<'a, G: EmissionGuarantee> IntoDiagnostic<'a, G> for ExpectedSemi {
                 fluent::parse_expected_semi_found_reserved_keyword_str
             }
             Some(TokenDescription::DocComment) => fluent::parse_expected_semi_found_doc_comment_str,
+            Some(TokenDescription::InvisibleOpenDelim) => {
+                fluent::parse_expected_semi_found_invisible_open_delim_str
+            }
             None => fluent::parse_expected_semi_found_str,
         });
         diag.set_span(self.span);
@@ -1671,6 +1682,13 @@ pub(crate) enum UnexpectedTokenAfterStructName {
         span: Span,
         token: Token,
     },
+    #[diag(parse_unexpected_token_after_struct_name_found_invisible_open_delim)]
+    InvisibleOpenDelim {
+        #[primary_span]
+        #[label(parse_unexpected_token_after_struct_name)]
+        span: Span,
+        token: Token,
+    },
     #[diag(parse_unexpected_token_after_struct_name_found_other)]
     Other {
         #[primary_span]
@@ -1687,6 +1705,7 @@ impl UnexpectedTokenAfterStructName {
             Some(TokenDescription::Keyword) => Self::Keyword { span, token },
             Some(TokenDescription::ReservedKeyword) => Self::ReservedKeyword { span, token },
             Some(TokenDescription::DocComment) => Self::DocComment { span, token },
+            Some(TokenDescription::InvisibleOpenDelim) => Self::InvisibleOpenDelim { span, token },
             None => Self::Other { span, token },
         }
     }

compiler/rustc_parse/src/parser/mod.rs

@@ -346,6 +346,15 @@ pub enum TokenDescription {
     Keyword,
     ReservedKeyword,
     DocComment,
+
+    // Invisible delimiters aren't pretty-printed. But in error messages we
+    // want to print something, otherwise we get confusing things in messages
+    // like "expected `(`, found ``". It's better to say "expected `(`, found
+    // invisible open delimiter".
+    //
+    // There has been no need for an `InvisibleCloseDelim` entry yet, but one
+    // could be added if necessary.
+    InvisibleOpenDelim,
 }
 
 impl TokenDescription {
@@ -355,22 +364,26 @@ impl TokenDescription {
             _ if token.is_used_keyword() => Some(TokenDescription::Keyword),
             _ if token.is_unused_keyword() => Some(TokenDescription::ReservedKeyword),
             token::DocComment(..) => Some(TokenDescription::DocComment),
+            token::OpenDelim(Delimiter::Invisible(_)) => Some(TokenDescription::InvisibleOpenDelim),
             _ => None,
         }
     }
 }
 
 pub(super) fn token_descr(token: &Token) -> String {
-    let name = pprust::token_to_string(token).to_string();
+    use TokenDescription::*;
 
-    let kind = TokenDescription::from_token(token).map(|kind| match kind {
-        TokenDescription::ReservedIdentifier => "reserved identifier",
-        TokenDescription::Keyword => "keyword",
-        TokenDescription::ReservedKeyword => "reserved keyword",
-        TokenDescription::DocComment => "doc comment",
-    });
+    let s = pprust::token_to_string(token).to_string();
 
-    if let Some(kind) = kind { format!("{kind} `{name}`") } else { format!("`{name}`") }
+    match TokenDescription::from_token(token) {
+        Some(ReservedIdentifier) => format!("reserved identifier `{s}`"),
+        Some(Keyword) => format!("keyword `{s}`"),
+        Some(ReservedKeyword) => format!("reserved keyword `{s}`"),
+        Some(DocComment) => format!("doc comment `{s}`"),
+        // Deliberately doesn't print `s`, which is empty.
+        Some(InvisibleOpenDelim) => "invisible open delimiter".to_string(),
+        None => format!("`{s}`"),
+    }
 }
 
 impl<'a> Parser<'a> {