Repository: TabbyML/tabby
File: crates/tabby-index/src/code/index.rs (Rust)

Code before:
}

async fn add_changed_documents(
    repository: &CodeRepository,
    commit: &str,
    embedding: Arc<dyn Embedding>,
    files: Vec<DirEntry>,
) -> usize {
    let concurrency = std::cmp::max(std::thread::available_parallelism().unwrap().get() * 4, 64);
    let builder = Arc::new(create_code_builder(Some(embedding)));
    let index = Arc::new(Indexer::new(corpus::CODE));
    let cloned_index = index.clone();

    let mut count_docs = 0;
    stream! {
        for file in files {
            let Some(key) = CodeIntelligence::compute_source_file_id(file.path()) else {
                continue;
            };

            let id = SourceCode::to_index_id(&repository.source_id, &key).id;

            // Skip if already indexed and has no failed chunks,
            // when skip, we should check if the document needs to be backfilled.
            if !require_updates(cloned_index.clone(), &id) {
                backfill_commit_if_needed(
                    builder.clone(),
                    cloned_index.clone(),
                    &id,
                    repository,
                    commit,
                    file.path()).await.unwrap_or_else(|e| {
                        warn!("Failed to backfill commit for {id}: {e}");
                    }
                );
                continue;
            }

            let Some(code) = CodeIntelligence::compute_source_file(repository, commit, file.path()) else {
                continue;
            };

            if !is_valid_file(&code) {
                continue;
            }

            let (_, s) = builder.build(code).await;
            // must delete before adding, otherwise the some fields like failed_chunks_count will remain
            cloned_index.delete(&id);
            for await task in s {
                yield task;

Review comment (suggestion):
```suggestion
                backfill_commit_in_doc_if_needed(
```

Code after:
}

async fn add_changed_documents(
    repository: &CodeRepository,
    commit: &str,
    embedding: Arc<dyn Embedding>,
    files: Vec<DirEntry>,
) -> usize {
    let concurrency = std::cmp::max(std::thread::available_parallelism().unwrap().get() * 4, 64);
    let builder = Arc::new(create_code_builder(Some(embedding)));
    let index = Arc::new(Indexer::new(corpus::CODE));
    let cloned_index = index.clone();

    let mut count_docs = 0;
    stream! {
        for file in files {
            let Some(key) = CodeIntelligence::compute_source_file_id(file.path()) else {
                continue;
            };

            let id = SourceCode::to_index_id(&repository.source_id, &key).id;

            // Skip if already indexed and has no failed chunks,
            // when skip, we should check if the document needs to be backfilled.
            if !require_updates(cloned_index.clone(), &id) {
                backfill_commit_in_doc_if_needed(
                    builder.clone(),
                    cloned_index.clone(),
                    &id,
                    repository,
                    commit,
                    file.path()).await.unwrap_or_else(|e| {
                        warn!("Failed to backfill commit for {id}: {e}");
                    }
                );
                continue;
            }

            let Some(code) = CodeIntelligence::compute_source_file(repository, commit, file.path()) else {
                continue;
            };

            if !is_valid_file(&code) {
                continue;
            }

            let (_, s) = builder.build(code).await;
            // must delete before adding, otherwise the some fields like failed_chunks_count will remain
            cloned_index.delete(&id);
            for await task in s {
                yield task;