From d835368eb8599b4f4c2f8a766bad5b57498a988d Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Mon, 22 Jul 2024 16:16:22 -0400 Subject: [PATCH] convert: capture `head_dim` for mistral (#5818) --- convert/mistral.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/convert/mistral.go b/convert/mistral.go index da6874cf..8fe066d6 100644 --- a/convert/mistral.go +++ b/convert/mistral.go @@ -71,6 +71,11 @@ func (m *MistralModel) WriteGGUF(ws io.WriteSeeker) error { "tokenizer.ggml.unknown_token_id": uint32(0), } + if m.Params.HeadDimension > 0 { + kv["llama.attention.key_length"] = uint32(m.Params.HeadDimension) + kv["llama.attention.value_length"] = uint32(m.Params.HeadDimension) + } + return llm.NewGGUFV3(m.Params.ByteOrder).Encode(ws, kv, m.Tensors) }