Repository: ollama/ollama
File: model/models/mllama/model.go (Go)

Code before:

	encoderCache := kvcache.NewEncoderCache()
	encoderCache.SetConfig(ml.CacheConfig{})
	m.Cache = kvcache.NewWrapperCache(encoderCache, kvcache.NewCausalCache(m.TextModel.Shift))

	return &m, nil
}

func (m *Model) EncodeMultimodal(ctx ml.Context, multimodalData []byte) (any, error) {
	if len(m.VisionModel.Transformer.Layers) == 0 || len(m.GlobalTransformer.Layers) == 0 {
		return nil, model.ErrNoVisionModel
	}

	image, _, err := image.Decode(bytes.NewReader(multimodalData))
	if err != nil {
		return nil, err
	}

	f32s, ratio, err := m.ImageProcessor.ProcessImage(image)
	if err != nil {
		return nil, err
	}

	if ratio.numTiles() < m.maxNumTiles {
		f32s = slices.Grow(f32s, m.imageSize*m.imageSize*m.numChannels*m.maxNumTiles)
		f32s = f32s[:m.imageSize*m.imageSize*m.numChannels*m.maxNumTiles]
	}

	pixelValues, err := ctx.Input().FromFloatSlice(f32s, m.imageSize, m.imageSize, m.numChannels, m.maxNumTiles)
	if err != nil {
		return nil, err
	}

	aspectRatio, err := ctx.Input().FromIntSlice([]int32{int32(ratio.rank)}, 1)
	if err != nil {
		return nil, err
	}

	positionIDs := ctx.Arange(0, 1601, 1, ml.DTypeI32)
	crossAttentionStates := m.VisionModel.Forward(ctx, pixelValues, positionIDs, aspectRatio)
	return m.Projector.Forward(ctx, crossAttentionStates), nil
}

func (m *Model) PostTokenize(inputs []input.Input) ([]input.Input, error) {
	for i := range inputs {
		if inputs[i].Multimodal != nil {
			inputs[i].Token = 128256 // 

		}
	}

	return inputs, nil

Review comment (suggestion):
```suggestion
		// pad the pixel values
		f32s = slices.Grow(f32s, m.imageSize*m.imageSize*m.numChannels*m.maxNumTiles)
```

Code after:

	encoderCache := kvcache.NewEncoderCache()
	encoderCache.SetConfig(ml.CacheConfig{})
	m.Cache = kvcache.NewWrapperCache(encoderCache, kvcache.NewCausalCache(m.TextModel.Shift))

	return &m, nil
}

func (m *Model) EncodeMultimodal(ctx ml.Context, multimodalData []byte) (any, error) {
	if len(m.VisionModel.Transformer.Layers) == 0 || len(m.GlobalTransformer.Layers) == 0 {
		return nil, model.ErrNoVisionModel
	}

	image, _, err := image.Decode(bytes.NewReader(multimodalData))
	if err != nil {
		return nil, err
	}

	f32s, ratio, err := m.ImageProcessor.ProcessImage(image)
	if err != nil {
		return nil, err
	}

	if ratio.numTiles() < m.maxNumTiles {
		// Pad tiles to maxNumTiles
		f32s = slices.Grow(f32s, m.imageSize*m.imageSize*m.numChannels*m.maxNumTiles)
		f32s = f32s[:m.imageSize*m.imageSize*m.numChannels*m.maxNumTiles]
	}

	pixelValues, err := ctx.Input().FromFloatSlice(f32s, m.imageSize, m.imageSize, m.numChannels, m.maxNumTiles)
	if err != nil {
		return nil, err
	}

	aspectRatio, err := ctx.Input().FromIntSlice([]int32{int32(ratio.rank)}, 1)
	if err != nil {
		return nil, err
	}

	positionIDs := ctx.Arange(0, 1601, 1, ml.DTypeI32)
	crossAttentionStates := m.VisionModel.Forward(ctx, pixelValues, positionIDs, aspectRatio)
	return m.Projector.Forward(ctx, crossAttentionStates), nil
}

func (m *Model) PostTokenize(inputs []input.Input) ([]input.Input, error) {
	for i := range inputs {
		if inputs[i].Multimodal != nil {
			inputs[i].Token = 128256 // 

		}
	}