Repository: TabbyML/tabby
File: crates/tabby-index/src/code/index.rs (Rust)

Code before:

    true
}

// v0.23.0 add the commit field to the code document.
async fn backfill_commit_if_needed(indexer: Arc<Indexer>, id: &str, commit: &str) -> Result<()> {
    if indexer.has_attribute_field(id, code::fields::ATTRIBUTE_COMMIT) {
        return Ok(());
    }

    let doc = indexer.get_doc(id).await?;
    indexer.delete_doc(id);
    indexer.add(create_document_with_commit(&doc, commit));

    Ok(())
}

fn is_valid_file(file: &SourceCode) -> bool {
    file.max_line_length <= MAX_LINE_LENGTH_THRESHOLD
        && file.avg_line_length <= AVG_LINE_LENGTH_THRESHOLD
        && file.alphanum_fraction >= MIN_ALPHA_NUM_FRACTION
        && file.num_lines <= MAX_NUMBER_OF_LINES
        && file.number_fraction <= MAX_NUMBER_FRACTION
}

fn create_document_with_commit(doc: &TantivyDocument, commit: &str) -> TantivyDocument {
    let schema = tabby_common::index::IndexSchema::instance();
    doc! {
        schema.field_id => get_text(doc, schema.field_id),
        schema.field_source_id => get_text(doc, schema.field_source_id).to_string(),
        schema.field_corpus => get_text(doc, schema.field_corpus).to_string(),
        schema.field_attributes => json!({
            code::fields::ATTRIBUTE_COMMIT: commit,
        }),
        schema.field_updated_at => get_text(doc, schema.field_updated_at).to_string(),,
        schema.field_failed_chunks_count => get_json_number_field(doc, schema.field_failed_chunks_count, code::fields::FAILED_CHUNKS_COUNT) as usize,
    }
}

fn get_text(doc: &TantivyDocument, field: schema::Field) -> &str {
    doc.get_first(field).unwrap().as_str().unwrap()
}

fn get_json_number_field(doc: &TantivyDocument, field: schema::Field, name: &str) -> i64 {
    doc.get_first(field)
        .unwrap()
        .as_object()
        .unwrap()
        .find(|(k, _)| *k == name)
        .unwrap()
        .1

Review comment (refactor):
can this method being somewhat refactored into `TantivyDocBuilder`? e.g `TantivyDocBuilder::backfill_attribute`

Code after:
    };

    count_docs
}

fn require_updates(indexer: Arc<Indexer>, id: &str) -> bool {
    if indexer.is_indexed(id) && !indexer.has_failed_chunks(id) {
        return false;
    };

    true
}

// v0.23.0 add the commit field to the code document.
async fn backfill_commit_in_doc_if_needed(
    builder: Arc<TantivyDocBuilder<SourceCode>>,
    indexer: Arc<Indexer>,
    id: &str,
    repository: &CodeRepository,
    commit: &str,
    path: &Path,
) -> Result<()> {
    if indexer.has_attribute_field(id, code::fields::ATTRIBUTE_COMMIT) {
        return Ok(());
    }

    let code = CodeIntelligence::compute_source_file(repository, commit, path)
        .ok_or_else(|| anyhow::anyhow!("Failed to compute source file"))?;
    if !is_valid_file(&code) {
        anyhow::bail!("Invalid file");
    }

    let origin = indexer.get_doc(id).await?;
    indexer.delete_doc(id);
    indexer
        .add(builder.backfill_doc_attributes(&origin, &code).await)
        .await;

    Ok(())
}

fn is_valid_file(file: &SourceCode) -> bool {
    file.max_line_length <= MAX_LINE_LENGTH_THRESHOLD
        && file.avg_line_length <= AVG_LINE_LENGTH_THRESHOLD
        && file.alphanum_fraction >= MIN_ALPHA_NUM_FRACTION
        && file.num_lines <= MAX_NUMBER_OF_LINES
        && file.number_fraction <= MAX_NUMBER_FRACTION
}

#[cfg(test)]
mod tests {