Repository: TabbyML/tabby
File: crates/tabby-crawler/src/lib.rs (Rust)

Code before:

    Some(CrawledDocument::new(
        data.request.endpoint,
        md.trim().to_owned(),
        metadata.into(),
    ))
}

pub async fn crawl_pipeline(
    start_url: &str,
    prefix_url: &str,
) -> anyhow::Result<impl Stream<Item = CrawledDocument>> {
    Ok(crawl_url(start_url, prefix_url)
        .await?
        .filter_map(move |data| async move { to_document(data) }))
}

/// Attempts to fetch `llms-full.txt` or `llms.txt` from the given base URL.
/// If `llms-full.txt` is available, it returns that document; otherwise, it falls back to `llms.txt`.
/// Returns a `CrawledDocument` if successful.
pub async fn crawler_llms(start_url: &str) -> anyhow::Result<CrawledDocument> {
    // Remove trailing slash from the base URL if present.
    let base_url = start_url.trim_end_matches('/');

    // Construct the URL for llms-full.txt.
    let llms_full_url = format!("{}/llms-full.txt", base_url);
    // Try fetching llms-full.txt first.
    match reqwest::get(&llms_full_url).await {
        Ok(resp) if resp.status().is_success() => {
            let body = resp.text().await?;
            debug!("Successfully fetched llms-full.txt: {}", llms_full_url);
            return Ok(CrawledDocument::new(
                llms_full_url,
                body,
                CrawledMetadata {
                    title: start_url.to_owned().into(),
                    description: start_url.to_owned().into(),
                },
            ));
        }
        Ok(resp) => {
            warn!(
                "Failed to fetch llms-full.txt, status code: {}. Trying llms.txt...",
                resp.status()
            );
        }
        Err(err) => {
            warn!("Error fetching llms-full.txt: {}. Trying llms.txt...", err);
        }
    }


Review comment (security):
can we parse the markdown to generate multiple documents?

```
# Cloudflare Aegis

URL: https://developers.cloudflare.com/aegis/

import { CardGrid, Description, GlossaryTooltip, LinkTitleCard, Plan, RelatedProduct } from "~/components"

<Description>

Leverage dedicated IPs to improve your origin security and implement Zero Trust.
</Description>

<Plan type="enterprise" />

Cloudflare Aegis provides dedicated egress IPs (from Cloudflare to your origin) for your layer 7 [WAF](/waf/) and <GlossaryTooltip term="content delivery network (CDN)">CDN</GlossaryTooltip> services, as well as [Spectrum](/spectrum/). The egress IPs are reserved exclusively for your account so that you can increase your origin security by only allowing traffic from a small list of IP addresses. Both [BYOIP](/byoip) and Cloudflare-leased IPs are supported by Cloudflare Aegis.
```

e.g title / url

Code after:

    Some(CrawledDocument::new(
        data.request.endpoint,
        md.trim().to_owned(),
        metadata.into(),
    ))
}

pub async fn crawl_pipeline(
    start_url: &str,
    prefix_url: &str,
) -> anyhow::Result<impl Stream<Item = CrawledDocument>> {
    Ok(crawl_url(start_url, prefix_url)
        .await?
        .filter_map(move |data| async move { to_document(data) }))
}

/// Attempts to fetch `llms-full.txt` from the given base URL,
/// then splits its markdown content into multiple sections based on H1 headings.
/// Each section becomes a separate `CrawledDocument`.
/// Returns a vector of `CrawledDocument`s if successful.
pub async fn crawler_llms(start_url: &str) -> anyhow::Result<Vec<CrawledDocument>> {
    // Remove trailing slash from the base URL if present.
    let base_url = start_url.trim_end_matches('/');

    let llms_full_url = format!("{}/llms-full.txt", base_url);
    let resp = reqwest::get(&llms_full_url).await?;
    if !resp.status().is_success() {
        anyhow::bail!("Unable to fetch llms-full.txt from {}", base_url);
    }
    let body = resp.text().await?;
    debug!("Successfully fetched llms-full.txt: {}", llms_full_url);

    // Split the fetched markdown content into sections.
    let docs = llms_txt_parser::split_llms_content(&body, start_url);
    if docs.is_empty() {
        anyhow::bail!("No sections found in llms-full.txt from {}", base_url);
    }

    Ok(docs)
}

#[cfg(test)]
mod tests {

    use tracing_test::traced_test;

    use super::*;

    #[tokio::test]
    #[traced_test]