Repository: carbon-language/carbon-lang
File: toolchain/lex/lex.cpp (C++)

Code before:
  auto LexFileEnd(llvm::StringRef source_text, ssize_t position) -> void;

  // Perform final checking and cleanup that should be done once we have
  // finished lexing the whole file, and before we consider the tokenized buffer
  // to be complete.
  auto Finalize() -> void;

  auto DiagnoseAndFixMismatchedBrackets() -> void;

  // The main entry point for dispatching through the lexer's table. This method
  // should always fully consume the source text.
  auto Lex() && -> TokenizedBuffer;

  // Checks for an ends a `DumpSemIRRange` that's missing an explicit end
  // marker.
  auto EndDumpSemIRRangeIfIncomplete(const char* diag_loc) -> void;

  auto has_dump_sem_ir_ranges() -> bool {
    return buffer_.has_dump_sem_ir_ranges();
  }

 private:
  class ErrorRecoveryBuffer;

  // Handles `//@dump-sem-ir-file`.
  auto DumpSemIRFile() -> void;

  // Handles `//@dump-sem-ir-begin` for a `DumpSemIRRange`.
  auto BeginDumpSemIRRange(const char* diag_loc) -> void;

  // Handles `//@dump-sem-ir-end` for a `DumpSemIRRange`.
  auto EndDumpSemIRRange(const char* diag_loc) -> void;

  TokenizedBuffer buffer_;

  LineIndex line_index_ = LineIndex::None;

  // Tracks whether the lexer has encountered whitespace that will be leading
  // whitespace for the next lexed token. Reset after each token lexed.
  bool has_leading_space_ = false;

  llvm::SmallVector<TokenIndex> open_groups_;
  bool has_mismatched_brackets_ = false;

  Diagnostics::ErrorTrackingConsumer consumer_;

  TokenizedBuffer::SourcePointerDiagnosticEmitter emitter_;

  TokenizedBuffer::TokenDiagnosticEmitter token_emitter_;
};


Review comment (suggestion):
Suggest renaming to `HandleDumpSemIRFile` or similar, since this function doesn't dump anything itself.

Code after:
  auto LexFileEnd(llvm::StringRef source_text, ssize_t position) -> void;

  // Perform final checking and cleanup that should be done once we have
  // finished lexing the whole file, and before we consider the tokenized buffer
  // to be complete.
  auto Finalize() -> void;

  auto DiagnoseAndFixMismatchedBrackets() -> void;

  // The main entry point for dispatching through the lexer's table. This method
  // should always fully consume the source text.
  auto Lex() && -> TokenizedBuffer;

  // Checks for an ends a `DumpSemIRRange` that's missing an explicit end
  // marker.
  auto EndDumpSemIRRangeIfIncomplete(const char* diag_loc) -> void;

  auto has_dump_sem_ir_ranges() -> bool {
    return buffer_.has_dump_sem_ir_ranges();
  }

 private:
  class ErrorRecoveryBuffer;

  // Handles `//@dump-sem-ir-begin` for a `DumpSemIRRange`.
  auto BeginDumpSemIRRange(const char* diag_loc) -> void;

  // Handles `//@dump-sem-ir-end` for a `DumpSemIRRange`.
  auto EndDumpSemIRRange(const char* diag_loc) -> void;

  TokenizedBuffer buffer_;

  LineIndex line_index_ = LineIndex::None;

  // Tracks whether the lexer has encountered whitespace that will be leading
  // whitespace for the next lexed token. Reset after each token lexed.
  bool has_leading_space_ = false;

  llvm::SmallVector<TokenIndex> open_groups_;
  bool has_mismatched_brackets_ = false;

  Diagnostics::ErrorTrackingConsumer consumer_;

  TokenizedBuffer::SourcePointerDiagnosticEmitter emitter_;

  TokenizedBuffer::TokenDiagnosticEmitter token_emitter_;
};

#if CARBON_USE_SIMD
namespace {
#if __ARM_NEON