Repository: jmorganca/ollama (go)

Issue:
Windows install runs into errors
after manual install instructions and
```go build```
I receive the following errors:
```
# github.com/jmorganca/ollama/readline
readline\readline.go:199:12: undefined: syscall.Kill
readline\readline.go:199:28: undefined: syscall.SIGSTOP
```
After messing with readline.go I just manage to get a .exe file that doesn't work

@deadcoder0904 or anyone who compiled on windows and got a working executable, could you describe how you did it?

Pull request description:
os specific ctrl-z
Add OS specific readline functions. Windows does not support these suspend system calls, so make ctrl-z a no-op on windows. This fixes development windows native builds.

resolves #1414

Patch:
diff --git a/README.md b/README.md
index a8669fc5..d202d73e 100644
--- a/README.md
+++ b/README.md
@@ -205,7 +205,8 @@ Finally, in a separate shell, run a model:
 ## REST API
 
 Ollama has a REST API for running and managing models.
-For example, to generate text from a model:
+
+### Generate a response
 
 ```
 curl http://localhost:11434/api/generate -d '{
@@ -214,7 +215,7 @@ curl http://localhost:11434/api/generate -d '{
 }'
 ```
 
-Or send a chat message (coming in 0.1.14):
+### Chat with a model
 
 ```
 curl http://localhost:11434/api/chat -d '{
diff --git a/api/types.go b/api/types.go
index 5048d5b6..3fbc5829 100644
--- a/api/types.go
+++ b/api/types.go
@@ -203,22 +203,12 @@ type GenerateResponse struct {
 	CreatedAt time.Time `json:"created_at"`
 	Response  string    `json:"response"`
 
-	ModelConfiguration ModelConfiguration `json:"model_configuration"`
-
 	Done    bool  `json:"done"`
 	Context []int `json:"context,omitempty"`
 
 	Metrics
 }
 
-type ModelConfiguration struct {
-	ModelFormat   string   `json:"model_format"`
-	ModelFamily   string   `json:"model_family"`
-	ModelFamilies []string `json:"model_families"`
-	ModelType     string   `json:"model_type"`
-	FileType      string   `json:"file_type"`
-}
-
 func (m *Metrics) Summary() {
 	if m.TotalDuration > 0 {
 		fmt.Fprintf(os.Stderr, "total duration:       %v\n", m.TotalDuration)
diff --git a/cmd/cmd.go b/cmd/cmd.go
index 2d31a121..d0b295a6 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -714,11 +714,11 @@ func generateInteractive(cmd *cobra.Command, opts generateOptions) error {
 			case MultilineSystem:
 				opts.System = prompt
 				prompt = ""
-				fmt.Println("Set system template.\n")
+				fmt.Println("Set system template.")
 			case MultilineTemplate:
 				opts.Template = prompt
 				prompt = ""
-				fmt.Println("Set model template.\n")
+				fmt.Println("Set model template.")
 			}
 			multiline = MultilineNone
 		case strings.HasPrefix(line, `"""`) && len(prompt) == 0:
@@ -793,9 +793,9 @@ func generateInteractive(cmd *cobra.Command, opts generateOptions) error {
 						if found {
 							opts.System = prompt
 							if args[1] == "system" {
-								fmt.Println("Set system template.\n")
+								fmt.Println("Set system template.")
 							} else {
-								fmt.Println("Set prompt template.\n")
+								fmt.Println("Set prompt template.")
 							}
 							prompt = ""
 						} else {
@@ -809,7 +809,7 @@ func generateInteractive(cmd *cobra.Command, opts generateOptions) error {
 						}
 					} else {
 						opts.System = line
-						fmt.Println("Set system template.\n")
+						fmt.Println("Set system template.")
 					}
 				default:
 					fmt.Printf("Unknown command '/set %s'. Type /? for help\n", args[1])
diff --git a/docs/api.md b/docs/api.md
index 846e92b4..5910ccf2 100644
--- a/docs/api.md
+++ b/docs/api.md
@@ -3,7 +3,7 @@
 ## Endpoints
 
 - [Generate a completion](#generate-a-completion)
-- [Send Chat Messages](#send-chat-messages)
+- [Generate a chat completion](#generate-a-chat-completion)
 - [Create a Model](#create-a-model)
 - [List Local Models](#list-local-models)
 - [Show Model Information](#show-model-information)
@@ -58,7 +58,7 @@ Enable JSON mode by setting the `format` parameter to `json`. This will structur
 
 ### Examples
 
-#### Request (Prompt)
+#### Request
 
 ```shell
 curl http://localhost:11434/api/generate -d '{
@@ -253,7 +253,7 @@ curl http://localhost:11434/api/generate -d '{
     "penalize_newline": true,
     "stop": ["\n", "user:"],
     "numa": false,
-    "num_ctx": 4,
+    "num_ctx": 1024,
     "num_batch": 2,
     "num_gqa": 1,
     "num_gpu": 1,
@@ -268,7 +268,7 @@ curl http://localhost:11434/api/generate -d '{
     "rope_frequency_base": 1.1,
     "rope_frequency_scale": 0.8,
     "num_thread": 8
-    }
+  }
 }'
 ```
 
@@ -291,7 +291,7 @@ curl http://localhost:11434/api/generate -d '{
 }
 ```
 
-## Send Chat Messages (coming in 0.1.14)
+## Generate a chat completion
 
 ```shell
 POST /api/chat
diff --git a/examples/typescript-simplechat/client.ts b/examples/typescript-simplechat/client.ts
new file mode 100644
index 00000000..3e571ab6
--- /dev/null
+++ b/examples/typescript-simplechat/client.ts
@@ -0,0 +1,77 @@
+import * as readline from "readline";
+
+const model = "llama2";
+type Message = {
+  role: "assistant" | "user" | "system";
+  content: string;
+}
+const messages: Message[] = [{
+  role: "system",
+  content: "You are a helpful AI agent."
+}]
+
+const rl = readline.createInterface({
+  input: process.stdin,
+  output: process.stdout
+})
+
+async function chat(messages: Message[]): Promise<Message> {
+  const body = {
+    model: model,
+    messages: messages
+  }
+
+  const response = await fetch("http://localhost:11434/api/chat", {
+    method: "POST",
+    body: JSON.stringify(body)
+  })
+
+  const reader = response.body?.getReader()
+  if (!reader) {
+    throw new Error("Failed to read response body")
+  }
+  let content = ""
+  while (true) {
+    const { done, value } = await reader.read()
+    if (done) {
+      break;
+    }
+    const rawjson = new TextDecoder().decode(value);
+    const json = JSON.parse(rawjson)
+
+    if (json.done === false) {
+      process.stdout.write(json.message.content);
+      content += json.message.content
+    }
+
+  }
+  return { role: "assistant", content: content };
+}
+
+async function askQuestion(): Promise<void> {
+  return new Promise<void>((resolve) => {
+    rl.question("\n\nAsk a question: (press enter alone to quit)\n\n", async (user_input) => {
+      if (user_input.trim() === "") {
+        rl.close();
+        console.log("Thankyou. Goodbye.\n")
+        console.log("=======\nHere is the message history that was used in this conversation.\n=======\n")
+        messages.forEach(message => {
+          console.log(message)
+        })
+        resolve();
+      } else {
+        console.log();
+        messages.push({ role: "user", content: user_input });
+        messages.push(await chat(messages));
+        await askQuestion(); // Ask the next question
+      }
+    });
+  });
+}
+
+async function main() {
+  await askQuestion();
+
+}
+
+main();
\ No newline at end of file
diff --git a/examples/typescript-simplechat/package.json b/examples/typescript-simplechat/package.json
new file mode 100644
index 00000000..4ee1647d
--- /dev/null
+++ b/examples/typescript-simplechat/package.json
@@ -0,0 +1,1 @@
+{ "dependencies": { "@types/node": "^20.10.4", "prompt-sync": "^4.2.0", "readline": "^1.3.0" } }
\ No newline at end of file
diff --git a/examples/typescript-simplechat/readme.md b/examples/typescript-simplechat/readme.md
new file mode 100644
index 00000000..ccd4aaf6
--- /dev/null
+++ b/examples/typescript-simplechat/readme.md
@@ -0,0 +1,39 @@
+# Simple Chat Example
+
+The **chat** endpoint is one of two ways to generate text from an LLM with Ollama. At a high level you provide the endpoint an array of message objects with a role and content specified. Then with each output and prompt, you add more messages, which builds up the history.
+
+## Run the Example
+
+There are a few ways to run this, just like any Typescript code:
+
+1. Compile with `tsc` and then run it with `node client.js`.
+2. Install `tsx` and run it with `tsx client.ts`.
+3. Install `bun` and run it with `bun client.ts`.
+
+## Review the Code
+
+You can see in the **chat** function that is actually calling the endpoint is simply done with:
+
+```typescript
+const body = {
+  model: model,
+  messages: messages
+}
+
+const response = await fetch("http://localhost:11434/api/chat", {
+  method: "POST",
+  body: JSON.stringify(body)
+})
+```
+
+With the **generate** endpoint, you need to provide a `prompt`. But with **chat**, you provide `messages`. And the resulting stream of responses includes a `message` object with a `content` field.
+
+The final JSON object doesn't provide the full content, so you will need to build the content yourself. In this example, **chat** takes the full array of messages and outputs the resulting message from this call of the chat endpoint.
+
+In the **askQuestion** function, we collect `user_input` and add it as a message to our messages and that is passed to the chat function. When the LLM is done responding the output is added as another message to the messages array.
+
+At the end, you will see a printout of all the messages.
+
+## Next Steps
+
+In this example, all generations are kept. You might want to experiment with summarizing everything older than 10 conversations to enable longer history with less context being used.
diff --git a/llm/ggml.go b/llm/ggml.go
index c3e71b88..8d421e03 100644
--- a/llm/ggml.go
+++ b/llm/ggml.go
@@ -93,6 +93,8 @@ func (c *containerGGML) Name() string {
 }
 
 func (c *containerGGML) Decode(ro *readSeekOffset) (model, error) {
+	// file contents aren't decoded
+	ro.Seek(0, io.SeekEnd)
 	return nil, nil
 }
 
@@ -115,6 +117,10 @@ func (c *containerGGMF) Decode(ro *readSeekOffset) (model, error) {
 	}
 
 	c.version = version
+
+	// remaining file contents aren't decoded
+	ro.Seek(0, io.SeekEnd)
+
 	return nil, nil
 }
 
@@ -141,6 +147,10 @@ func (c *containerGGJT) Decode(ro *readSeekOffset) (model, error) {
 	// different model types may have different layouts for hyperparameters
 	var llama llamaModel
 	binary.Read(ro, binary.LittleEndian, &llama.hyperparameters)
+
+	// remaining file contents aren't decoded
+	ro.Seek(0, io.SeekEnd)
+
 	return &llama, nil
 }
 
@@ -163,6 +173,10 @@ func (c *containerLORA) Decode(ro *readSeekOffset) (model, error) {
 	}
 
 	c.version = version
+
+	// remaining file contents aren't decoded
+	ro.Seek(0, io.SeekEnd)
+
 	return nil, nil
 }
 
diff --git a/llm/llama.go b/llm/llama.go
index b2f1571f..6ccac861 100644
--- a/llm/llama.go
+++ b/llm/llama.go
@@ -59,6 +59,7 @@ ws ::= ([ \t\n] ws)?
 var llamaCppEmbed embed.FS
 
 type ModelRunner struct {
+	Type        string // "gguf" or "ggml"
 	Path        string // path to the model runner executable
 	Accelerated bool
 }
@@ -72,25 +73,25 @@ func chooseRunners(workDir, runnerType string) []ModelRunner {
 	switch runtime.GOOS {
 	case "darwin":
 		if runtime.GOARCH == "arm64" {
-			runners = []ModelRunner{{Path: path.Join(buildPath, "metal", "bin", "ollama-runner")}}
+			runners = []ModelRunner{{Type: runnerType, Path: path.Join(buildPath, "metal", "bin", "ollama-runner")}}
 		} else {
-			runners = []ModelRunner{{Path: path.Join(buildPath, "cpu", "bin", "ollama-runner")}}
+			runners = []ModelRunner{{Type: runnerType, Path: path.Join(buildPath, "cpu", "bin", "ollama-runner")}}
 		}
 	case "linux":
 		runners = []ModelRunner{
-			{Path: path.Join(buildPath, "cuda", "bin", "ollama-runner"), Accelerated: true},
-			{Path: path.Join(buildPath, "cpu", "bin", "ollama-runner")},
+			{Type: runnerType, Path: path.Join(buildPath, "cuda", "bin", "ollama-runner"), Accelerated: true},
+			{Type: runnerType, Path: path.Join(buildPath, "cpu", "bin", "ollama-runner")},
 		}
 	case "windows":
 		// TODO: select windows GPU runner here when available
 		runners = []ModelRunner{
-			{Path: path.Join(buildPath, "cuda", "bin", "Release", "ollama-runner.exe"), Accelerated: true},
-			{Path: path.Join(buildPath, "cpu", "bin", "Release", "ollama-runner.exe")},
+			{Type: runnerType, Path: path.Join(buildPath, "cuda", "bin", "Release", "ollama-runner.exe"), Accelerated: true},
+			{Type: runnerType, Path: path.Join(buildPath, "cpu", "bin", "Release", "ollama-runner.exe")},
 		}
 	default:
 		log.Printf("unknown OS, running on CPU: %s", runtime.GOOS)
 		runners = []ModelRunner{
-			{Path: path.Join(buildPath, "cpu", "bin", "ollama-runner")},
+			{Type: runnerType, Path: path.Join(buildPath, "cpu", "bin", "ollama-runner")},
 		}
 	}
 
@@ -148,6 +149,7 @@ func chooseRunners(workDir, runnerType string) []ModelRunner {
 	for _, r := range runners {
 		// clean the ModelRunner paths so that they match the OS we are running on
 		localRunnersByPriority = append(localRunnersByPriority, ModelRunner{
+			Type:        r.Type,
 			Path:        filepath.Clean(path.Join(workDir, r.Path)),
 			Accelerated: r.Accelerated,
 		})
@@ -402,11 +404,17 @@ func newLlama(model string, adapters, projectors []string, runners []ModelRunner
 		}
 
 		port := rand.Intn(65535-49152) + 49152 // get a random port in the ephemeral range
+		params := append(params, "--port", strconv.Itoa(port))
+
+		if runner.Type == "gguf" {
+			params = append(params, "--parallel", "2")
+		}
+
 		ctx, cancel := context.WithCancel(context.Background())
 		cmd := exec.CommandContext(
 			ctx,
 			runner.Path,
-			append(params, "--port", strconv.Itoa(port))...,
+			params...,
 		)
 
 		var libraryPaths []string
@@ -537,7 +545,6 @@ type prediction struct {
 const maxBufferSize = 512 * format.KiloByte
 
 type PredictOpts struct {
-	Model            string
 	Prompt           string
 	Format           string
 	CheckpointStart  time.Time
@@ -545,7 +552,6 @@ type PredictOpts struct {
 }
 
 type PredictResult struct {
-	Model              string
 	CreatedAt          time.Time
 	TotalDuration      time.Duration
 	LoadDuration       time.Duration
@@ -631,34 +637,35 @@ func (llm *llama) Predict(ctx context.Context, predict PredictOpts, fn func(Pred
 				continue
 			}
 
-			if evt, ok := bytes.CutPrefix(line, []byte("data: ")); ok {
-				var p prediction
-				if err := json.Unmarshal(evt, &p); err != nil {
-					return fmt.Errorf("error unmarshaling llm prediction response: %v", err)
-				}
+			evt, ok := bytes.CutPrefix(line, []byte("data: "))
+			if !ok {
+				return fmt.Errorf("error parsing llm response stream: %s", line)
+			}
 
-				if p.Content != "" {
-					fn(PredictResult{
-						Model:     predict.Model,
-						CreatedAt: time.Now().UTC(),
-						Content:   p.Content,
-					})
-				}
+			var p prediction
+			if err := json.Unmarshal(evt, &p); err != nil {
+				return fmt.Errorf("error unmarshaling llm prediction response: %v", err)
+			}
 
-				if p.Stop {
-					fn(PredictResult{
-						Model:         predict.Model,
-						CreatedAt:     time.Now().UTC(),
-						TotalDuration: time.Since(predict.CheckpointStart),
-
-						Done:               true,
-						PromptEvalCount:    p.Timings.PromptN,
-						PromptEvalDuration: parseDurationMs(p.Timings.PromptMS),
-						EvalCount:          p.Timings.PredictedN,
-						EvalDuration:       parseDurationMs(p.Timings.PredictedMS),
-					})
-					return nil
-				}
+			if p.Content != "" {
+				fn(PredictResult{
+					CreatedAt: time.Now().UTC(),
+					Content:   p.Content,
+				})
+			}
+
+			if p.Stop {
+				fn(PredictResult{
+					CreatedAt:     time.Now().UTC(),
+					TotalDuration: time.Since(predict.CheckpointStart),
+
+					Done:               true,
+					PromptEvalCount:    p.Timings.PromptN,
+					PromptEvalDuration: parseDurationMs(p.Timings.PromptMS),
+					EvalCount:          p.Timings.PredictedN,
+					EvalDuration:       parseDurationMs(p.Timings.PredictedMS),
+				})
+				return nil
 			}
 		}
 	}
diff --git a/readline/readline.go b/readline/readline.go
index 781ac8a3..c4c73520 100644
--- a/readline/readline.go
+++ b/readline/readline.go
@@ -192,14 +192,7 @@ func (i *Instance) Readline() (string, error) {
 		case CharCtrlW:
 			buf.DeleteWord()
 		case CharCtrlZ:
-			if err := UnsetRawMode(fd, termios); err != nil {
-				return "", err
-			}
-
-			syscall.Kill(0, syscall.SIGSTOP)
-
-			// on resume...
-			return "", nil
+			return handleCharCtrlZ(fd, termios)
 		case CharEnter:
 			output := buf.String()
 			if output != "" {
diff --git a/readline/readline_unix.go b/readline/readline_unix.go
new file mode 100644
index 00000000..a1e34d27
--- /dev/null
+++ b/readline/readline_unix.go
@@ -0,0 +1,18 @@
+//go:build !windows
+
+package readline
+
+import (
+	"syscall"
+)
+
+func handleCharCtrlZ(fd int, termios *Termios) (string, error) {
+	if err := UnsetRawMode(fd, termios); err != nil {
+		return "", err
+	}
+
+	syscall.Kill(0, syscall.SIGSTOP)
+
+	// on resume...
+	return "", nil
+}
diff --git a/readline/readline_windows.go b/readline/readline_windows.go
new file mode 100644
index 00000000..c8178903
--- /dev/null
+++ b/readline/readline_windows.go
@@ -0,0 +1,6 @@
+package readline
+
+func handleCharCtrlZ(fd int, state *State) (string, error) {
+	// not supported
+	return "", nil
+}
diff --git a/server/images.go b/server/images.go
index 68b2770a..097cbe2f 100644
--- a/server/images.go
+++ b/server/images.go
@@ -64,6 +64,11 @@ func (m *Model) Prompt(p PromptVars) (string, error) {
 		return "", err
 	}
 
+	if p.System == "" {
+		// use the default system prompt for this model if one is not specified
+		p.System = m.System
+	}
+
 	vars := map[string]any{
 		"System":   p.System,
 		"Prompt":   p.Prompt,
@@ -98,16 +103,16 @@ func (m *Model) ChatPrompt(msgs []api.Message) (string, error) {
 	}
 
 	for _, msg := range msgs {
-		switch msg.Role {
+		switch strings.ToLower(msg.Role) {
 		case "system":
-			if currentVars.Prompt != "" || currentVars.System != "" {
+			if currentVars.System != "" {
 				if err := writePrompt(); err != nil {
 					return "", err
 				}
 			}
 			currentVars.System = msg.Content
 		case "user":
-			if currentVars.Prompt != "" || currentVars.System != "" {
+			if currentVars.Prompt != "" {
 				if err := writePrompt(); err != nil {
 					return "", err
 				}
@@ -141,12 +146,16 @@ type ManifestV2 struct {
 }
 
 type ConfigV2 struct {
+	ModelFormat   string   `json:"model_format"`
+	ModelFamily   string   `json:"model_family"`
+	ModelFamilies []string `json:"model_families"`
+	ModelType     string   `json:"model_type"`
+	FileType      string   `json:"file_type"`
+
 	// required by spec
 	Architecture string `json:"architecture"`
 	OS           string `json:"os"`
 	RootFS       RootFS `json:"rootfs"`
-
-	api.ModelConfiguration
 }
 
 func (c *ConfigV2) SetModelFormat(format string) {
diff --git a/server/routes.go b/server/routes.go
index ea3c0692..9716c84e 100644
--- a/server/routes.go
+++ b/server/routes.go
@@ -199,10 +199,9 @@ func GenerateHandler(c *gin.Context) {
 	// an empty request loads the model
 	if req.Prompt == "" && req.Template == "" && req.System == "" {
 		c.JSON(http.StatusOK, api.GenerateResponse{
-			CreatedAt:          time.Now().UTC(),
-			Model:              req.Model,
-			ModelConfiguration: model.Config.ModelConfiguration,
-			Done:               true})
+			CreatedAt: time.Now().UTC(),
+			Model:     req.Model,
+			Done:      true})
 		return
 	}
 
@@ -261,11 +260,10 @@ func GenerateHandler(c *gin.Context) {
 			}
 
 			resp := api.GenerateResponse{
-				Model:              r.Model,
-				ModelConfiguration: model.Config.ModelConfiguration,
-				CreatedAt:          r.CreatedAt,
-				Done:               r.Done,
-				Response:           r.Content,
+				Model:     req.Model,
+				CreatedAt: r.CreatedAt,
+				Done:      r.Done,
+				Response:  r.Content,
 				Metrics: api.Metrics{
 					TotalDuration:      r.TotalDuration,
 					LoadDuration:       r.LoadDuration,
@@ -277,7 +275,7 @@ func GenerateHandler(c *gin.Context) {
 			}
 
 			if r.Done && !req.Raw {
-				embd, err := loaded.runner.Encode(c.Request.Context(), req.Prompt+generated.String())
+				embd, err := loaded.runner.Encode(c.Request.Context(), prompt+generated.String())
 				if err != nil {
 					ch <- gin.H{"error": err.Error()}
 					return
@@ -290,7 +288,6 @@ func GenerateHandler(c *gin.Context) {
 
 		// Start prediction
 		predictReq := llm.PredictOpts{
-			Model:            model.Name,
 			Prompt:           prompt,
 			Format:           req.Format,
 			CheckpointStart:  checkpointStart,
@@ -302,19 +299,30 @@ func GenerateHandler(c *gin.Context) {
 	}()
 
 	if req.Stream != nil && !*req.Stream {
-		// Wait for the channel to close
-		var r api.GenerateResponse
+		// Accumulate responses into the final response
+		var final api.GenerateResponse
 		var sb strings.Builder
 		for resp := range ch {
-			var ok bool
-			if r, ok = resp.(api.GenerateResponse); !ok {
-				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			switch r := resp.(type) {
+			case api.GenerateResponse:
+				sb.WriteString(r.Response)
+				final = r
+			case gin.H:
+				if errorMsg, ok := r["error"].(string); ok {
+					c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
+					return
+				} else {
+					c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in response"})
+					return
+				}
+			default:
+				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error"})
 				return
 			}
-			sb.WriteString(r.Response)
 		}
-		r.Response = sb.String()
-		c.JSON(http.StatusOK, r)
+
+		final.Response = sb.String()
+		c.JSON(http.StatusOK, final)
 		return
 	}
 
@@ -852,7 +860,7 @@ func Serve(ln net.Listener, allowOrigins []string) error {
 	if runtime.GOOS == "linux" {
 		// check compatibility to log warnings
 		if _, err := llm.CheckVRAM(); err != nil {
-			log.Printf(err.Error())
+			log.Print(err.Error())
 		}
 	}
 
@@ -976,7 +984,7 @@ func ChatHandler(c *gin.Context) {
 			loaded.expireTimer.Reset(sessionDuration)
 
 			resp := api.ChatResponse{
-				Model:     r.Model,
+				Model:     req.Model,
 				CreatedAt: r.CreatedAt,
 				Done:      r.Done,
 				Metrics: api.Metrics{
@@ -998,7 +1006,6 @@ func ChatHandler(c *gin.Context) {
 
 		// Start prediction
 		predictReq := llm.PredictOpts{
-			Model:            model.Name,
 			Prompt:           prompt,
 			Format:           req.Format,
 			CheckpointStart:  checkpointStart,
@@ -1010,21 +1017,33 @@ func ChatHandler(c *gin.Context) {
 	}()
 
 	if req.Stream != nil && !*req.Stream {
-		// Wait for the channel to close
-		var r api.ChatResponse
+		// Accumulate responses into the final response
+		var final api.ChatResponse
 		var sb strings.Builder
 		for resp := range ch {
-			var ok bool
-			if r, ok = resp.(api.ChatResponse); !ok {
-				c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()})
+			switch r := resp.(type) {
+			case api.ChatResponse:
+				if r.Message != nil {
+					sb.WriteString(r.Message.Content)
+				}
+
+				final = r
+			case gin.H:
+				if errorMsg, ok := r["error"].(string); ok {
+					c.JSON(http.StatusInternalServerError, gin.H{"error": errorMsg})
+					return
+				} else {
+					c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error format in response"})
+					return
+				}
+			default:
+				c.JSON(http.StatusInternalServerError, gin.H{"error": "unexpected error"})
 				return
 			}
-			if r.Message != nil {
-				sb.WriteString(r.Message.Content)
-			}
 		}
-		r.Message = &api.Message{Role: "assistant", Content: sb.String()}
-		c.JSON(http.StatusOK, r)
+
+		final.Message = &api.Message{Role: "assistant", Content: sb.String()}
+		c.JSON(http.StatusOK, final)
 		return
 	}
 


Test patch:
diff --git a/server/images_test.go b/server/images_test.go
index 85e8d4bd..28cb39a3 100644
--- a/server/images_test.go
+++ b/server/images_test.go
@@ -1,21 +1,98 @@
 package server
 
 import (
+	"strings"
 	"testing"
+
+	"github.com/jmorganca/ollama/api"
 )
 
-func TestModelPrompt(t *testing.T) {
-	m := Model{
-		Template: "a{{ .Prompt }}b",
-	}
-	s, err := m.Prompt(PromptVars{
-		Prompt: "<h1>",
-	})
-	if err != nil {
-		t.Fatal(err)
+func TestChat(t *testing.T) {
+	tests := []struct {
+		name     string
+		template string
+		msgs     []api.Message
+		want     string
+		wantErr  string
+	}{
+		{
+			name:     "Single Message",
+			template: "
 {{ .System }} {{ .Prompt }} 
",
+			msgs: []api.Message{
+				{
+					Role:    "system",
+					Content: "You are a Wizard.",
+				},
+				{
+					Role:    "user",
+					Content: "What are the potion ingredients?",
+				},
+			},
+			want: "
 You are a Wizard. What are the potion ingredients? 
",
+		},
+		{
+			name:     "Message History",
+			template: "
 {{ .System }} {{ .Prompt }} 
",
+			msgs: []api.Message{
+				{
+					Role:    "system",
+					Content: "You are a Wizard.",
+				},
+				{
+					Role:    "user",
+					Content: "What are the potion ingredients?",
+				},
+				{
+					Role:    "assistant",
+					Content: "sugar",
+				},
+				{
+					Role:    "user",
+					Content: "Anything else?",
+				},
+			},
+			want: "
 You are a Wizard. What are the potion ingredients? 
sugar
  Anything else? 
",
+		},
+		{
+			name:     "Assistant Only",
+			template: "
 {{ .System }} {{ .Prompt }} 
",
+			msgs: []api.Message{
+				{
+					Role:    "assistant",
+					Content: "everything nice",
+				},
+			},
+			want: "
   
everything nice",
+		},
+		{
+			name: "Invalid Role",
+			msgs: []api.Message{
+				{
+					Role:    "not-a-role",
+					Content: "howdy",
+				},
+			},
+			wantErr: "invalid role: not-a-role",
+		},
 	}
-	want := "a<h1>b"
-	if s != want {
-		t.Errorf("got %q, want %q", s, want)
+
+	for _, tt := range tests {
+		m := Model{
+			Template: tt.template,
+		}
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := m.ChatPrompt(tt.msgs)
+			if tt.wantErr != "" {
+				if err == nil {
+					t.Errorf("ChatPrompt() expected error, got nil")
+				}
+				if !strings.Contains(err.Error(), tt.wantErr) {
+					t.Errorf("ChatPrompt() error = %v, wantErr %v", err, tt.wantErr)
+				}
+			}
+			if got != tt.want {
+				t.Errorf("ChatPrompt() got = %v, want %v", got, tt.want)
+			}
+		})
 	}
 }