Repository: jmorganca/ollama (go)

Issue:
Please don’t clutter the user home directory
Currently Ollama is writing into a directory in the user home, `~/.ollama`.

However, the user home directory should be under the *user’s* control, and applications may only write into it if explicitly instructed to do so by the user. Settings etc. belong in OS-specific locations; in particular, under `~/Library` on macOS, under `%AppData%` or `%LocalAppData%` on Windows and, by convention, in directories following the [XDG basedir specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html) on Linux.

Pull request description:
allow for a configurable ollama model storage directory
- set `OLLAMA_MODELS` in the environment that ollama is running in to change where models are stored
- update docs

```bash
$ OLLAMA_MODELS=/Users/bruce/ollama_models ollama serve
# store models and public keys in /Users/bruce/ollama_models
```

Resolves #228

I'll hold off on merging this until #847 is in to avoid causing that PR pain.

Patch:
diff --git a/README.md b/README.md
index 44ee7052..84996372 100644
--- a/README.md
+++ b/README.md
@@ -29,7 +29,8 @@ curl https://ollama.ai/install.sh | sh
 
 ### Docker
 
-See the official [Docker image](https://hub.docker.com/r/ollama/ollama).
+The official [Ollama Docker image `ollama/ollama`](https://hub.docker.com/r/ollama/ollama)
+is available on Docker Hub.
 
 ## Quickstart
 
@@ -178,8 +179,7 @@ ollama list
 Install `cmake` and `go`:
 
 ```
-brew install cmake
-brew install go
+brew install cmake go
 ```
 
 Then generate dependencies and build:
@@ -203,9 +203,8 @@ Finally, in a separate shell, run a model:
 
 ## REST API
 
-See the [API documentation](docs/api.md) for all endpoints.
-
-Ollama has an API for running and managing models. For example to generate text from a model:
+Ollama has a REST API for running and managing models.
+For example, to generate text from a model:
 
 ```
 curl -X POST http://localhost:11434/api/generate -d '{
@@ -214,6 +213,8 @@ curl -X POST http://localhost:11434/api/generate -d '{
 }'
 ```
 
+See the [API documentation](./docs/api.md) for all endpoints.
+
 ## Community Integrations
 
 - [LangChain](https://python.langchain.com/docs/integrations/llms/ollama) and [LangChain.js](https://js.langchain.com/docs/modules/model_io/models/llms/integrations/ollama) with [example](https://js.langchain.com/docs/use_cases/question_answering/local_retrieval_qa)
diff --git a/api/client.go b/api/client.go
index 3cf55a25..c89ff38f 100644
--- a/api/client.go
+++ b/api/client.go
@@ -18,10 +18,6 @@ import (
 	"github.com/jmorganca/ollama/version"
 )
 
-const DefaultHost = "127.0.0.1:11434"
-
-var envHost = os.Getenv("OLLAMA_HOST")
-
 type Client struct {
 	base *url.URL
 	http http.Client
@@ -44,14 +40,24 @@ func checkError(resp *http.Response, body []byte) error {
 }
 
 func ClientFromEnvironment() (*Client, error) {
+	defaultPort := "11434"
+
 	scheme, hostport, ok := strings.Cut(os.Getenv("OLLAMA_HOST"), "://")
-	if !ok {
+	switch {
+	case !ok:
 		scheme, hostport = "http", os.Getenv("OLLAMA_HOST")
+	case scheme == "http":
+		defaultPort = "80"
+	case scheme == "https":
+		defaultPort = "443"
 	}
 
+	// trim trailing slashes
+	hostport = strings.TrimRight(hostport, "/")
+
 	host, port, err := net.SplitHostPort(hostport)
 	if err != nil {
-		host, port = "127.0.0.1", "11434"
+		host, port = "127.0.0.1", defaultPort
 		if ip := net.ParseIP(strings.Trim(hostport, "[]")); ip != nil {
 			host = ip.String()
 		} else if hostport != "" {
diff --git a/cmd/cmd.go b/cmd/cmd.go
index 16383d60..b05c5bae 100644
--- a/cmd/cmd.go
+++ b/cmd/cmd.go
@@ -553,6 +553,9 @@ func generateInteractive(cmd *cobra.Command, model string) error {
 		wordWrap = false
 	}
 
+	fmt.Print(readline.StartBracketedPaste)
+	defer fmt.Printf(readline.EndBracketedPaste)
+
 	var multiLineBuffer string
 
 	for {
diff --git a/docs/faq.md b/docs/faq.md
index 2c7711d5..a1d11fc5 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -60,3 +60,7 @@ systemctl restart ollama
 
 - macOS: Raw model data is stored under `~/.ollama/models`.
 - Linux: Raw model data is stored under `/usr/share/ollama/.ollama/models`
+
+### How can I change where Ollama stores models?
+
+To modify where models are stored, you can use the `OLLAMA_MODELS` environment variable. Note that on Linux this means defining `OLLAMA_MODELS` in a drop-in `/etc/systemd/system/ollama.service.d` service file, reloading systemd, and restarting the ollama service.
diff --git a/llm/llama.go b/llm/llama.go
index 61288b52..b257f3b8 100644
--- a/llm/llama.go
+++ b/llm/llama.go
@@ -243,12 +243,15 @@ func NumGPU(numLayer, fileSizeBytes int64, opts api.Options) int {
 			return 0
 		}
 
-		// Calculate bytes per layer
-		// TODO: this is a rough heuristic, better would be to calculate this based on number of layers and context size
+		/*
+		 Calculate bytes per layer, this will roughly be the size of the model file divided by the number of layers.
+		 We can store the model weights and the kv cache in vram,
+		 to enable kv chache vram storage add two additional layers to the number of layers retrieved from the model file.
+		*/
 		bytesPerLayer := fileSizeBytes / numLayer
 
-		// max number of layers we can fit in VRAM, subtract 8% to prevent consuming all available VRAM and running out of memory
-		layers := int(freeBytes/bytesPerLayer) * 92 / 100
+		// 75% of the absolute max number of layers we can fit in available VRAM, off-loading too many layers to the GPU can cause OOM errors
+		layers := int(freeBytes/bytesPerLayer) * 3 / 4
 		log.Printf("%d MB VRAM available, loading up to %d GPU layers", freeBytes/(1024*1024), layers)
 
 		return layers
diff --git a/readline/readline.go b/readline/readline.go
index 43ab400f..8a74727e 100644
--- a/readline/readline.go
+++ b/readline/readline.go
@@ -65,8 +65,7 @@ func (i *Instance) Readline() (string, error) {
 	var esc bool
 	var escex bool
 	var metaDel bool
-	var bracketedPaste bool
-	var ignoreEnter bool
+	var pasteMode PasteMode
 
 	var currentLineBuf []rune
 
@@ -111,7 +110,16 @@ func (i *Instance) Readline() (string, error) {
 			case KeyRight:
 				buf.MoveRight()
 			case CharBracketedPaste:
-				bracketedPaste = true
+				var code string
+				for cnt := 0; cnt < 3; cnt++ {
+					r = i.Terminal.ReadRune()
+					code += string(r)
+				}
+				if code == CharBracketedPasteStart {
+					pasteMode = PasteModeStart
+				} else if code == CharBracketedPasteEnd {
+					pasteMode = PasteModeEnd
+				}
 			case KeyDel:
 				if buf.Size() > 0 {
 					buf.Delete()
@@ -141,10 +149,6 @@ func (i *Instance) Readline() (string, error) {
 		}
 
 		switch r {
-		case CharBracketedPasteStart:
-			if bracketedPaste {
-				ignoreEnter = true
-			}
 		case CharEsc:
 			esc = true
 		case CharInterrupt:
@@ -179,16 +183,19 @@ func (i *Instance) Readline() (string, error) {
 		case CharCtrlW:
 			buf.DeleteWord()
 		case CharEnter:
-			if !ignoreEnter {
-				output := buf.String()
-				if output != "" {
-					i.History.Add([]rune(output))
-				}
-				buf.MoveToEnd()
-				fmt.Println()
-				return output, nil
+			output := buf.String()
+			if output != "" {
+				i.History.Add([]rune(output))
+			}
+			buf.MoveToEnd()
+			fmt.Println()
+			switch pasteMode {
+			case PasteModeStart:
+				output = `"""` + output
+			case PasteModeEnd:
+				output = output + `"""`
 			}
-			fallthrough
+			return output, nil
 		default:
 			if metaDel {
 				metaDel = false
diff --git a/readline/term_bsd.go b/readline/term_bsd.go
index 98672075..6dcf42fc 100644
--- a/readline/term_bsd.go
+++ b/readline/term_bsd.go
@@ -1,4 +1,4 @@
-// go build darwin dragonfly freebsd netbsd openbsd
+//go:build darwin || freebsd || netbsd || openbsd
 package readline
 
 import (
diff --git a/readline/term_linux.go b/readline/term_linux.go
index ed3634e2..9028f52e 100644
--- a/readline/term_linux.go
+++ b/readline/term_linux.go
@@ -1,3 +1,4 @@
+//go:build linux || solaris
 package readline
 
 import (
diff --git a/readline/types.go b/readline/types.go
index 32b77007..f4b3d0c4 100644
--- a/readline/types.go
+++ b/readline/types.go
@@ -72,6 +72,14 @@ const (
 
 const (
 	CharBracketedPaste      = 50
-	CharBracketedPasteStart = 0
-	CharBracketedPasteEnd   = 1
+	CharBracketedPasteStart = "00~"
+	CharBracketedPasteEnd   = "01~"
+)
+
+type PasteMode int
+
+const (
+	PastModeOff = iota
+	PasteModeStart
+	PasteModeEnd
 )
diff --git a/server/download.go b/server/download.go
index acfa8e52..fa559f0c 100644
--- a/server/download.go
+++ b/server/download.go
@@ -15,6 +15,7 @@ import (
 	"strings"
 	"sync"
 	"sync/atomic"
+	"syscall"
 	"time"
 
 	"golang.org/x/sync/errgroup"
@@ -158,7 +159,8 @@ func (b *blobDownload) run(ctx context.Context, requestURL *url.URL, opts *Regis
 				w := io.NewOffsetWriter(file, part.StartsAt())
 				err := b.downloadChunk(inner, requestURL, w, part, opts)
 				switch {
-				case errors.Is(err, context.Canceled):
+				case errors.Is(err, context.Canceled), errors.Is(err, syscall.ENOSPC):
+					// return immediately if the context is canceled or the device is out of space
 					return err
 				case err != nil:
 					log.Printf("%s part %d attempt %d failed: %v, retrying", b.Digest[7:19], i, try, err)
diff --git a/server/images.go b/server/images.go
index 5ae8d008..e249f8f9 100644
--- a/server/images.go
+++ b/server/images.go
@@ -131,7 +131,7 @@ func (m *ManifestV2) GetTotalSize() (total int64) {
 }
 
 func GetManifest(mp ModelPath) (*ManifestV2, string, error) {
-	fp, err := mp.GetManifestPath(false)
+	fp, err := mp.GetManifestPath()
 	if err != nil {
 		return nil, "", err
 	}
@@ -595,10 +595,13 @@ func CreateManifest(name string, cfg *LayerReader, layers []*Layer) error {
 		return err
 	}
 
-	fp, err := mp.GetManifestPath(true)
+	fp, err := mp.GetManifestPath()
 	if err != nil {
 		return err
 	}
+	if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil {
+		return err
+	}
 	return os.WriteFile(fp, manifestJSON, 0o644)
 }
 
@@ -710,16 +713,19 @@ func CreateLayer(f io.ReadSeeker) (*LayerReader, error) {
 
 func CopyModel(src, dest string) error {
 	srcModelPath := ParseModelPath(src)
-	srcPath, err := srcModelPath.GetManifestPath(false)
+	srcPath, err := srcModelPath.GetManifestPath()
 	if err != nil {
 		return err
 	}
 
 	destModelPath := ParseModelPath(dest)
-	destPath, err := destModelPath.GetManifestPath(true)
+	destPath, err := destModelPath.GetManifestPath()
 	if err != nil {
 		return err
 	}
+	if err := os.MkdirAll(filepath.Dir(destPath), 0o755); err != nil {
+		return err
+	}
 
 	// copy the file
 	input, err := os.ReadFile(srcPath)
@@ -882,7 +888,7 @@ func DeleteModel(name string) error {
 		return err
 	}
 
-	fp, err := mp.GetManifestPath(false)
+	fp, err := mp.GetManifestPath()
 	if err != nil {
 		return err
 	}
@@ -1121,10 +1127,13 @@ func PullModel(ctx context.Context, name string, regOpts *RegistryOptions, fn fu
 		return err
 	}
 
-	fp, err := mp.GetManifestPath(true)
+	fp, err := mp.GetManifestPath()
 	if err != nil {
 		return err
 	}
+	if err := os.MkdirAll(filepath.Dir(fp), 0o755); err != nil {
+		return err
+	}
 
 	err = os.WriteFile(fp, manifestJSON, 0o644)
 	if err != nil {
diff --git a/server/modelpath.go b/server/modelpath.go
index c6798a02..13b26eeb 100644
--- a/server/modelpath.go
+++ b/server/modelpath.go
@@ -85,20 +85,27 @@ func (mp ModelPath) GetShortTagname() string {
 	return fmt.Sprintf("%s/%s/%s:%s", mp.Registry, mp.Namespace, mp.Repository, mp.Tag)
 }
 
-func (mp ModelPath) GetManifestPath(createDir bool) (string, error) {
+// modelsDir returns the value of the OLLAMA_MODELS environment variable or the user's home directory if OLLAMA_MODELS is not set.
+// The models directory is where Ollama stores its model files and manifests.
+func modelsDir() (string, error) {
+	if models, exists := os.LookupEnv("OLLAMA_MODELS"); exists {
+		return models, nil
+	}
 	home, err := os.UserHomeDir()
 	if err != nil {
 		return "", err
 	}
+	return filepath.Join(home, ".ollama", "models"), nil
+}
 
-	path := filepath.Join(home, ".ollama", "models", "manifests", mp.Registry, mp.Namespace, mp.Repository, mp.Tag)
-	if createDir {
-		if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
-			return "", err
-		}
+// GetManifestPath returns the path to the manifest file for the given model path, it is up to the caller to create the directory if it does not exist.
+func (mp ModelPath) GetManifestPath() (string, error) {
+	dir, err := modelsDir()
+	if err != nil {
+		return "", err
 	}
 
-	return path, nil
+	return filepath.Join(dir, "manifests", mp.Registry, mp.Namespace, mp.Repository, mp.Tag), nil
 }
 
 func (mp ModelPath) BaseURL() *url.URL {
@@ -109,12 +116,12 @@ func (mp ModelPath) BaseURL() *url.URL {
 }
 
 func GetManifestPath() (string, error) {
-	home, err := os.UserHomeDir()
+	dir, err := modelsDir()
 	if err != nil {
 		return "", err
 	}
 
-	path := filepath.Join(home, ".ollama", "models", "manifests")
+	path := filepath.Join(dir, "manifests")
 	if err := os.MkdirAll(path, 0o755); err != nil {
 		return "", err
 	}
@@ -123,7 +130,7 @@ func GetManifestPath() (string, error) {
 }
 
 func GetBlobsPath(digest string) (string, error) {
-	home, err := os.UserHomeDir()
+	dir, err := modelsDir()
 	if err != nil {
 		return "", err
 	}
@@ -132,7 +139,7 @@ func GetBlobsPath(digest string) (string, error) {
 		digest = strings.ReplaceAll(digest, ":", "-")
 	}
 
-	path := filepath.Join(home, ".ollama", "models", "blobs", digest)
+	path := filepath.Join(dir, "blobs", digest)
 	dirPath := filepath.Dir(path)
 	if digest == "" {
 		dirPath = path


Test patch:
diff --git a/api/client_test.go b/api/client_test.go
new file mode 100644
index 00000000..0eafedca
--- /dev/null
+++ b/api/client_test.go
@@ -0,0 +1,43 @@
+package api
+
+import "testing"
+
+func TestClientFromEnvironment(t *testing.T) {
+	type testCase struct {
+		value  string
+		expect string
+		err    error
+	}
+
+	testCases := map[string]*testCase{
+		"empty":                      {value: "", expect: "http://127.0.0.1:11434"},
+		"only address":               {value: "1.2.3.4", expect: "http://1.2.3.4:11434"},
+		"only port":                  {value: ":1234", expect: "http://:1234"},
+		"address and port":           {value: "1.2.3.4:1234", expect: "http://1.2.3.4:1234"},
+		"scheme http and address":    {value: "http://1.2.3.4", expect: "http://1.2.3.4:80"},
+		"scheme https and address":   {value: "https://1.2.3.4", expect: "https://1.2.3.4:443"},
+		"scheme, address, and port":  {value: "https://1.2.3.4:1234", expect: "https://1.2.3.4:1234"},
+		"hostname":                   {value: "example.com", expect: "http://example.com:11434"},
+		"hostname and port":          {value: "example.com:1234", expect: "http://example.com:1234"},
+		"scheme http and hostname":   {value: "http://example.com", expect: "http://example.com:80"},
+		"scheme https and hostname":  {value: "https://example.com", expect: "https://example.com:443"},
+		"scheme, hostname, and port": {value: "https://example.com:1234", expect: "https://example.com:1234"},
+		"trailing slash":             {value: "example.com/", expect: "http://example.com:11434"},
+		"trailing slash port":        {value: "example.com:1234/", expect: "http://example.com:1234"},
+	}
+
+	for k, v := range testCases {
+		t.Run(k, func(t *testing.T) {
+			t.Setenv("OLLAMA_HOST", v.value)
+
+			client, err := ClientFromEnvironment()
+			if err != v.err {
+				t.Fatalf("expected %s, got %s", v.err, err)
+			}
+
+			if client.base.String() != v.expect {
+				t.Fatalf("expected %s, got %s", v.expect, client.base.String())
+			}
+		})
+	}
+}