Repository: swc-project/swc
File: crates/swc_ecma_parser/src/lexer/token.rs (Rust)

Code before:
        buffer.expect_string_token_value()
    }

    #[inline(always)]
    pub fn take_num<I: Tokens>(self, buffer: &mut Buffer<I>) -> (f64, Atom) {
        buffer.expect_number_token_value()
    }

    #[inline(always)]
    pub fn take_bigint<I: Tokens>(self, buffer: &mut Buffer<I>) -> (Box<BigInt>, Atom) {
        buffer.expect_bigint_token_value()
    }

    #[inline]
    pub fn take_word<I: Tokens>(self, buffer: &Buffer<I>) -> Atom {
        if self == Token::Ident {
            let value = buffer.get_token_value();
            let Some(TokenValue::Word(word)) = value else {
                unreachable!("{:#?}", value)
            };
            return word.clone();
        }

        let span = buffer.cur.span;
        let atom = Atom::new(buffer.iter.read_string(span));
        atom
    }

    #[inline(always)]
    pub fn take_known_ident(self) -> Atom {
        self.as_known_ident_atom().unwrap()
    }

    #[inline(always)]
    pub fn take_unknown_ident_ref<'b, I: Tokens>(&'b self, buffer: &'b Buffer<I>) -> &'b Atom {
        buffer.expect_word_token_value_ref()
    }

    #[inline(always)]
    pub fn take_template<I: Tokens>(self, buffer: &mut Buffer<I>) -> (LexResult<Wtf8Atom>, Atom) {
        buffer.expect_template_token_value()
    }

    #[inline(always)]
    pub fn jsx_text(value: Wtf8Atom, raw: Atom, lexer: &mut Lexer) -> Self {
        lexer.set_token_value(Some(TokenValue::Str { value, raw }));
        Token::JSXText
    }

    #[inline(always)]
    pub fn take_jsx_text<I: Tokens>(self, buffer: &mut Buffer<I>) -> (Atom, Atom) {

Review comment (bug):
The return type was changed from `Option<Atom>` to `Atom`, removing the ability to signal when a token is not a word. The old implementation would return `None` for tokens like `Token::Null`, `Token::True`, `Token::False`, or tokens that are neither identifiers, known identifiers, nor keywords. The new implementation will attempt to slice the source for any non-Ident token, which may produce incorrect results for non-word tokens. Consider adding validation that the token `is_word()` before proceeding, or document that callers must verify this precondition.

Code after:
        buffer.expect_string_token_value()
    }

    #[inline(always)]
    pub fn take_num<I: Tokens>(self, buffer: &mut Buffer<I>) -> (f64, Atom) {
        buffer.expect_number_token_value()
    }

    #[inline(always)]
    pub fn take_bigint<I: Tokens>(self, buffer: &mut Buffer<I>) -> (Box<BigInt>, Atom) {
        buffer.expect_bigint_token_value()
    }

    #[inline]
    pub fn take_word<I: Tokens>(self, buffer: &Buffer<I>) -> Atom {
        if self == Token::Ident {
            let value = buffer.get_token_value();
            let Some(TokenValue::Word(word)) = value else {
                unreachable!("{:#?}", value)
            };
            return word.clone();
        }

        let span = buffer.cur.span;
        let atom = Atom::new(buffer.iter.read_string(span));
        atom
    }

    #[inline(always)]
    pub fn take_known_ident<I: Tokens>(self, buffer: &Buffer<I>) -> Atom {
        let span = buffer.cur.span;
        let atom = Atom::new(buffer.iter.read_string(span));
        atom
    }

    #[inline(always)]
    pub fn take_unknown_ident_ref<'b, I: Tokens>(&'b self, buffer: &'b Buffer<I>) -> &'b Atom {
        buffer.expect_word_token_value_ref()
    }

    #[inline(always)]
    pub fn take_template<I: Tokens>(self, buffer: &mut Buffer<I>) -> (LexResult<Wtf8Atom>, Atom) {
        buffer.expect_template_token_value()
    }

    #[inline(always)]
    pub fn jsx_text(value: Wtf8Atom, raw: Atom, lexer: &mut Lexer) -> Self {
        lexer.set_token_value(Some(TokenValue::Str { value, raw }));
        Token::JSXText
    }