Repository: helix-editor/helix
File: helix-term/src/commands.rs (Rust)

Code before:
        .and_then(|config| config.comment_tokens.as_ref())
        .map(|pfxs| &pfxs[..])
        .unwrap_or(&[]);
    // Sort by length to handle Rust's /// vs //
    let mut comment_prefixes = Vec::from(comment_prefixes);
    comment_prefixes.sort_unstable_by_key(|x| std::cmp::Reverse(x.len()));

    let mut changes = Vec::new();

    for selection in doc.selection(view.id) {
        let (start, mut end) = selection.line_range(slice);
        if start == end {
            end = (end + 1).min(text.len_lines() - 1);
        }
        let lines = start..end;

        changes.reserve(lines.len());

        for line in lines {
            let start = line_end_char_index(&slice, line);
            let mut end = text.line_to_char(line + 1);
            end = skip_while(slice, end, |ch| matches!(ch, ' ' | '\t')).unwrap_or(end);
            let slice_from_end = slice.slice(end..);
            for pfx in comment_prefixes.iter() {
                if slice_from_end.starts_with(pfx) {
                    end += pfx.len();
                    break;
                }
            }
            end = skip_while(slice, end, |ch| matches!(ch, ' ' | '\t')).unwrap_or(end);

            let separator = if end == line_end_char_index(&slice, line + 1) {
                // the joining line contains only space-characters => don't include a whitespace when joining
                None
            } else {
                Some(Tendril::from(" "))
            };
            changes.push((start, end, separator));
        }
    }

    // nothing to do, bail out early to avoid crashes later
    if changes.is_empty() {
        return;
    }

    changes.sort_unstable_by_key(|(from, _to, _text)| *from);
    changes.dedup();

    // select inserted spaces
    let transaction = if select_space {

Review comment (suggestion):
This is mixing up byte and character indexing: `end` is a character index into the rope and the token's `len()` is the number of bytes in its UTF-8 representation. It won't make a difference in the common case (comment token is ASCII) but if you try a comment token that takes more than one byte in UTF-8 `end` will be greater than it should. Instead here we have to use `token.chars().count()`.

Code after:
        .and_then(|config| config.comment_tokens.as_deref())
        .unwrap_or(&[]);
    // Sort by length to handle Rust's /// vs //
    let mut comment_tokens: Vec<&str> = comment_tokens.iter().map(|x| x.as_str()).collect();
    comment_tokens.sort_unstable_by_key(|x| std::cmp::Reverse(x.len()));

    let mut changes = Vec::new();

    for selection in doc.selection(view.id) {
        let (start, mut end) = selection.line_range(slice);
        if start == end {
            end = (end + 1).min(text.len_lines() - 1);
        }
        let lines = start..end;

        changes.reserve(lines.len());

        let first_line_idx = slice.line_to_char(start);
        let first_line_idx = skip_while(slice, first_line_idx, |ch| matches!(ch, ' ' | 't'))
            .unwrap_or(first_line_idx);
        let first_line = slice.slice(first_line_idx..);
        let mut current_comment_token = comment_tokens
            .iter()
            .find(|token| first_line.starts_with(token));

        for line in lines {
            let start = line_end_char_index(&slice, line);
            let mut end = text.line_to_char(line + 1);
            end = skip_while(slice, end, |ch| matches!(ch, ' ' | '\t')).unwrap_or(end);
            let slice_from_end = slice.slice(end..);
            if let Some(token) = comment_tokens
                .iter()
                .find(|token| slice_from_end.starts_with(token))
            {
                if Some(token) == current_comment_token {
                    end += token.chars().count();
                    end = skip_while(slice, end, |ch| matches!(ch, ' ' | '\t')).unwrap_or(end);
                } else {
                    // update current token, but don't delete this one.
                    current_comment_token = Some(token);
                }
            }

            let separator = if end == line_end_char_index(&slice, line + 1) {
                // the joining line contains only space-characters => don't include a whitespace when joining
                None
            } else {
                Some(Tendril::from(" "))
            };
            changes.push((start, end, separator));
        }