Repository: TabbyML/tabby
File: crates/llama-cpp-server/src/supervisor.rs (Rust)

Code before:
                                    "ERROR",
                                    tabby_common::terminal::HeaderFormat::BoldRed,
                                    &solution_lines,
                                );
                                msg.print();
                            }
                            eprintln!(
                                "llama-server <{}> encountered a fatal error. Exiting service. Please check the above logs and suggested solutions for details.",
                                name
                            );
                            std::process::exit(1);
                        }
                        _ => {
                            warn!(
                                "llama-server <{}> exited with status code {}, retrying...",
                                name, status_code
                            );
                        }
                    }
                    tokio::time::sleep(Duration::from_secs(1)).await;
                }
            }
        });

        Self { name, handle, port }
    }

    pub fn port(&self) -> u16 {
        self.port
    }

    pub async fn start(&self) {
        debug!("Waiting for llama-server <{}> to start...", self.name);
        let client = reqwest::Client::builder().no_proxy().build().unwrap();
        loop {
            let Ok(resp) = client
                .get(api_endpoint(self.port) + "/health")
                .timeout(Duration::from_secs(1))
                .send()
                .await
            else {
                debug!("llama-server <{}> not ready yet, retrying...", self.name);
                tokio::time::sleep(Duration::from_secs(1)).await;
                continue;
            };

            if resp.status().is_success() {
                debug!("llama-server <{}> started successfully", self.name);
                return;
            }
        }

Review comment (suggestion):
how about printing the first or some of the retried error messages, so we can gather unknown error messages when users experience them or if they are caught in a retry loop.

WDYT @wsxiaoys 

Code after:
                                eprintln!("{}", line);
                                if let Some(solution) = analyze_error_message(&line) {
                                    let solution_lines: Vec<_> = solution.split('\n').collect();
                                    let msg = tabby_common::terminal::InfoMessage::new(
                                        "ERROR",
                                        tabby_common::terminal::HeaderFormat::BoldRed,
                                        &solution_lines,
                                    );
                                    msg.print();
                                    break;
                                }
                            }
                            eprintln!(
                                "llama-server <{}> encountered a fatal error. Exiting service. Please check the above logs and suggested solutions for details.",
                                name
                            );
                            std::process::exit(1);
                        }
                        _ => {
                            if first_retry {
                                for line in &error_lines {
                                    eprintln!("{}", line);
                                }
                                first_retry = false;
                            }
                            warn!(
                                "llama-server <{}> exited with status code {}, retrying...",
                                name, status_code
                            );
                        }
                    }
                    tokio::time::sleep(std::time::Duration::from_secs(1)).await;
                }
            }
        });

        Self { name, handle, port }
    }

    pub fn port(&self) -> u16 {
        self.port
    }

    pub async fn start(&self) {
        debug!("Waiting for llama-server <{}> to start...", self.name);
        let client = reqwest::Client::builder().no_proxy().build().unwrap();
        loop {
            let Ok(resp) = client
                .get(api_endpoint(self.port) + "/health")
                .timeout(Duration::from_secs(1))
                .send()