Repository: typst/typst
File: crates/typst-syntax/src/lexer.rs (Rust)

Code before:
            let dedent = if i == 0 && !skipped { 0 } else { dedent };
            let offset: usize = line.chars().take(dedent).map(char::len_utf8).sum();
            self.s.eat_newline();
            self.s.advance(offset);
            self.push_raw(SyntaxKind::RawTrimmed);
            self.s.advance(line.len() - offset);
            self.push_raw(SyntaxKind::Text);
        }

        // Add final trimmed.
        if self.s.cursor() < end - backticks {
            self.s.jump(end - backticks);
            self.push_raw(SyntaxKind::RawTrimmed);
        }
        self.s.jump(end);
    }

    fn inline_raw(&mut self, start: usize, end: usize, backticks: usize) {
        self.s.jump(start + backticks);
        let inner = self.s.to(end - backticks);
        for line in split_newlines(inner).into_iter() {
            self.s.eat_newline();
            self.push_raw(SyntaxKind::RawTrimmed);
            self.s.advance(line.len());
            self.push_raw(SyntaxKind::Text);
        }
        self.s.jump(end);
    }

    /// Push the current cursor as a breakpoint that marks the end of a segment.
    fn push_raw(&mut self, kind: SyntaxKind) {
        let end = self.s.cursor();
        self.raw.push((kind, end));
    }

    fn link(&mut self) -> SyntaxKind {
        let (link, balanced) = link_prefix(self.s.after());
        self.s.advance(link.len());

        if !balanced {
            return self.error(
                "automatic links cannot contain unbalanced brackets, \
                 use the `link` function instead",
            );
        }

        SyntaxKind::Link
    }

    fn numbering(&mut self, start: usize) -> SyntaxKind {
        self.s.eat_while(char::is_ascii_digit);

Review comment (performance):
I think lexing directly with the scanner rather than relying on `split_newlines` would be pretty simple and more efficient here.

Code after:
            let dedent = if i == 0 && !skipped { 0 } else { dedent };
            let offset: usize = line.chars().take(dedent).map(char::len_utf8).sum();
            self.s.eat_newline();
            self.s.advance(offset);
            self.push_raw(SyntaxKind::RawTrimmed);
            self.s.advance(line.len() - offset);
            self.push_raw(SyntaxKind::Text);
        }

        // Add final trimmed.
        if self.s.cursor() < end - backticks {
            self.s.jump(end - backticks);
            self.push_raw(SyntaxKind::RawTrimmed);
        }
        self.s.jump(end);
    }

    fn inline_raw(&mut self, start: usize, end: usize, backticks: usize) {
        self.s.jump(start + backticks);

        while self.s.cursor() < end - backticks {
            if self.s.at(is_newline) {
                self.push_raw(SyntaxKind::Text);
                self.s.eat_newline();
                self.push_raw(SyntaxKind::RawTrimmed);
                continue;
            }
            self.s.eat();
        }
        self.push_raw(SyntaxKind::Text);

        self.s.jump(end);
    }

    /// Push the current cursor that marks the end of a raw segment of
    /// the given `kind`.
    fn push_raw(&mut self, kind: SyntaxKind) {
        let end = self.s.cursor();
        self.raw.push((kind, end));
    }

    fn link(&mut self) -> SyntaxKind {
        let (link, balanced) = link_prefix(self.s.after());
        self.s.advance(link.len());

        if !balanced {
            return self.error(
                "automatic links cannot contain unbalanced brackets, \
                 use the `link` function instead",
            );
        }