Repository: TabbyML/tabby
File: crates/http-api-bindings/src/embedding/llama.rs (Rust)

Code before:
impl LlamaCppEngine {
    pub fn create(api_endpoint: &str, api_key: Option<String>) -> Box<dyn Embedding> {
        let client = create_reqwest_client(api_endpoint);

        Box::new(Self {
            client,
            api_endpoint: format!("{}/embedding", api_endpoint),
            api_key,
        })
    }
}

#[derive(Serialize)]
struct EmbeddingRequest {
    content: String,
}

#[derive(Deserialize)]
struct EmbeddingResponse {
    embedding: Vec<Vec<f32>>,
}

#[async_trait]
impl Embedding for LlamaCppEngine {
    async fn embed(&self, prompt: &str) -> anyhow::Result<Vec<f32>> {
        // Some initial requests to llama.cpp are experiencing issues
        // would failed with `Connection reset by peer` or `Broken pipe`
        //
        // This serves as a temporary solution to attempt the request up to three times.
        //
        // Track issue: https://github.com/ggerganov/llama.cpp/issues/11411
        let strategy = ExponentialBackoff::from_millis(100).map(jitter).take(3);
        let response = RetryIf::spawn(
            strategy,
            || {
                let request = EmbeddingRequest {
                    content: prompt.to_owned(),
                };
                let mut request = self.client.post(&self.api_endpoint).json(&request);
                if let Some(api_key) = &self.api_key {
                    request = request.bearer_auth(api_key);
                }

                async move {
                    request
                        .send()
                        .instrument(embedding_info_span!("llamacpp"))
                        .await
                }
            },
            |e: &reqwest::Error| {

Review comment (suggestion):
```suggestion
        // Some initial requests to llama.cpp are experiencing issues
```

Please refine the docs here to reflect the fact that not only "initial request", but from time to time whenever embedding server being vacant for a while.

Code after:
impl LlamaCppEngine {
    pub fn create(api_endpoint: &str, api_key: Option<String>) -> Box<dyn Embedding> {
        let client = create_reqwest_client(api_endpoint);

        Box::new(Self {
            client,
            api_endpoint: format!("{}/embedding", api_endpoint),
            api_key,
        })
    }
}

#[derive(Serialize)]
struct EmbeddingRequest {
    content: String,
}

#[derive(Deserialize)]
struct EmbeddingResponse {
    embedding: Vec<Vec<f32>>,
}

#[async_trait]
impl Embedding for LlamaCppEngine {
    async fn embed(&self, prompt: &str) -> anyhow::Result<Vec<f32>> {
        // Occasionally, when the embedding server has been idle for a period,
        // some of the concurrent initial requests to llama.cpp encounter problems,
        // resulting in failures with `Connection reset by peer` or `Broken pipe`.
        //
        // This serves as a temporary solution to attempt the request up to three times.
        //
        // Track issue: https://github.com/ggerganov/llama.cpp/issues/11411
        let strategy = ExponentialBackoff::from_millis(100).map(jitter).take(3);
        let response = RetryIf::spawn(
            strategy,
            || {
                let request = EmbeddingRequest {
                    content: prompt.to_owned(),
                };
                let mut request = self.client.post(&self.api_endpoint).json(&request);
                if let Some(api_key) = &self.api_key {
                    request = request.bearer_auth(api_key);
                }

                async move {
                    request
                        .send()
                        .instrument(embedding_info_span!("llamacpp"))
                        .await
                }
            },