Repository: qdrant/qdrant
File: src/common/inference/service.rs (Rust)

Code before:
    }
}

pub struct InferenceService {
    pub(crate) config: InferenceConfig,
    pub(crate) client: Client,
}

static INFERENCE_SERVICE: RwLock<Option<Arc<InferenceService>>> = RwLock::new(None);

impl InferenceService {
    pub fn new(config: Option<InferenceConfig>) -> Self {
        let config = config.unwrap_or_default();
        let InferenceConfig {
            address: _,
            timeout,
            token: _,
        } = &config;

        let client_builder = Client::builder();

        let client_builder = if let Some(timeout_secs) = timeout {
            client_builder.timeout(Duration::from_secs(*timeout_secs))
        } else {
            client_builder
        };

        Self {
            config,
            client: client_builder
                .build()
                .expect("Invalid timeout value for HTTP client"),
        }
    }

    pub fn init_global(config: Option<InferenceConfig>) -> Result<(), StorageError> {
        let mut inference_service = INFERENCE_SERVICE.write();

        let service = Self::new(config);

        if !service.is_address_valid() {
            return Err(StorageError::service_error(
                "Cannot initialize InferenceService: address is required but not provided or empty in config",
            ));
        }

        *inference_service = Some(Arc::new(service));
        Ok(())
    }

    pub fn get_global() -> Option<Arc<InferenceService>> {

Review comment (suggestion):
Disclaimer: I only skimmed the PR.

Am I seeing correctly that it is possible for us to create a client that has no timeout at all? If so, I strongly vote against that. We should always provide _some_ timeout.

Relying in a timeout in the system we talk to (inference proxy) is not good enough.

Code after:
    }
}

pub struct InferenceService {
    pub(crate) config: InferenceConfig,
    pub(crate) client: Client,
}

static INFERENCE_SERVICE: RwLock<Option<Arc<InferenceService>>> = RwLock::new(None);

/// We assume that the inference provider will handle timeouts itself, if
/// not provided by the user or configured. But we need ensurance, that we don't
/// wait forever for a response.
static DEFAULT_INFERENCE_TIMEOUT_SECS: u64 = 10 * 60; // 10 minutes

impl InferenceService {
    pub fn new(config: Option<InferenceConfig>) -> Self {
        let config = config.unwrap_or_default();
        let InferenceConfig {
            address: _,
            timeout,
            token: _,
        } = &config;

        let timeout = timeout.unwrap_or(DEFAULT_INFERENCE_TIMEOUT_SECS);
        let client_builder = Client::builder().timeout(Duration::from_secs(timeout));

        Self {
            config,
            client: client_builder
                .build()
                .expect("Invalid timeout value for HTTP client"),
        }
    }

    pub fn init_global(config: Option<InferenceConfig>) -> Result<(), StorageError> {
        let mut inference_service = INFERENCE_SERVICE.write();

        let service = Self::new(config);

        if !service.is_address_valid() {
            return Err(StorageError::service_error(
                "Cannot initialize InferenceService: address is required but not provided or empty in config",
            ));
        }

        *inference_service = Some(Arc::new(service));
        Ok(())
    }

    pub fn get_global() -> Option<Arc<InferenceService>> {