Repository: ollama/ollama
File: server/images.go (Go)

Code before:

		e.opts.EmbeddingOnly = true
		llmModel, err := llm.New(e.model, []string{}, e.opts)
		if err != nil {
			return nil, fmt.Errorf("load model to generate embeddings: %v", err)
		}
		defer func() {
			if llmModel != nil {
				llmModel.Close()
			}
		}()

		addedFiles := make(map[string]bool) // keep track of files that have already been added
		for _, filePattern := range e.files {
			matchingFiles, err := filepath.Glob(filePattern)
			if err != nil {
				return nil, fmt.Errorf("could not find files with pattern %s: %w", filePattern, err)
			}

			for _, filePath := range matchingFiles {
				if addedFiles[filePath] {
					continue
				}
				addedFiles[filePath] = true
				// check if we already have embeddings for this file path
				layerIdentifier := fmt.Sprintf("%s:%s", filePath, e.model)
				digest, _ := GetSHA256Digest(strings.NewReader(layerIdentifier))
				existing, err := existingFileEmbeddings(digest)
				if err != nil {
					return nil, fmt.Errorf("failed to check existing embeddings for file %s: %v", filePath, err)
				}

				// TODO: check file type
				f, err := os.Open(filePath)
				if err != nil {
					return nil, fmt.Errorf("could not open embed file: %w", err)
				}
				scanner := bufio.NewScanner(f)
				scanner.Split(bufio.ScanLines)

				data := []string{}
				for scanner.Scan() {
					data = append(data, scanner.Text())
				}
				f.Close()

				// the digest of the file is set here so that the client knows a new operation is in progress
				fileDigest, _ := GetSHA256Digest(bytes.NewReader([]byte(filePath)))

				embeddings := []vector.Embedding{}
				for i, d := range data {

Review comment (suggestion):
Shouldn't this be hashing the contents of the file + the model weights? Instead of the file name and the model name?

To add: the model sha256 should already be calculated at this point (shouldn't need to hash it again)

Code after:

		e.opts.EmbeddingOnly = true
		llmModel, err := llm.New(e.model, []string{}, e.opts)
		if err != nil {
			return nil, fmt.Errorf("load model to generate embeddings: %v", err)
		}
		defer func() {
			if llmModel != nil {
				llmModel.Close()
			}
		}()

		// this will be used to check if we already have embeddings for a file
		modelInfo, err := os.Stat(e.model)
		if err != nil {
			return nil, fmt.Errorf("failed to get model file info: %v", err)
		}

		addedFiles := make(map[string]bool) // keep track of files that have already been added
		for _, filePattern := range e.files {
			matchingFiles, err := filepath.Glob(filePattern)
			if err != nil {
				return nil, fmt.Errorf("could not find files with pattern %s: %w", filePattern, err)
			}

			for _, filePath := range matchingFiles {
				if addedFiles[filePath] {
					continue
				}
				addedFiles[filePath] = true
				// check if we already have embeddings for this file path
				layerIdentifier := fmt.Sprintf("%s:%s:%s:%d", filePath, e.model, modelInfo.ModTime().Format("2006-01-02 15:04:05"), modelInfo.Size())
				digest, _ := GetSHA256Digest(strings.NewReader(layerIdentifier))
				existing, err := existingFileEmbeddings(digest)
				if err != nil {
					return nil, fmt.Errorf("failed to check existing embeddings for file %s: %v", filePath, err)
				}

				// TODO: check file type
				f, err := os.Open(filePath)
				if err != nil {
					return nil, fmt.Errorf("could not open embed file: %w", err)
				}
				scanner := bufio.NewScanner(f)
				scanner.Split(bufio.ScanLines)

				data := []string{}
				for scanner.Scan() {
					data = append(data, scanner.Text())
				}
				f.Close()