diff --git a/server/model.go b/server/model.go index a79f549a..d33ffaec 100644 --- a/server/model.go +++ b/server/model.go @@ -4,6 +4,7 @@ import ( "archive/zip" "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -259,13 +260,27 @@ func detectChatTemplate(layers []*layerGGML) ([]*layerGGML, error) { if t, err := template.Named(s); err != nil { slog.Debug("template detection", "error", err) } else { - tmpl, err := NewLayer(t.Reader(), "application/vnd.ollama.image.template") + layer, err := NewLayer(t.Reader(), "application/vnd.ollama.image.template") if err != nil { return nil, err } - tmpl.status = fmt.Sprintf("using autodetected template %s", t.Name) - layers = append(layers, &layerGGML{tmpl, nil}) + layer.status = fmt.Sprintf("using autodetected template %s", t.Name) + layers = append(layers, &layerGGML{layer, nil}) + + if t.Parameters != nil { + var b bytes.Buffer + if err := json.NewEncoder(&b).Encode(t.Parameters); err != nil { + return nil, err + } + + layer, err := NewLayer(&b, "application/vnd.ollama.image.params") + if err != nil { + return nil, err + } + + layers = append(layers, &layerGGML{layer, nil}) + } } } } diff --git a/server/routes_create_test.go b/server/routes_create_test.go index 04174b92..84672087 100644 --- a/server/routes_create_test.go +++ b/server/routes_create_test.go @@ -545,9 +545,10 @@ func TestCreateDetectTemplate(t *testing.T) { } checkFileExists(t, filepath.Join(p, "blobs", "*"), []string{ + filepath.Join(p, "blobs", "sha256-0d79f567714c62c048378f2107fb332dabee0135d080c302d884317da9433cc5"), filepath.Join(p, "blobs", "sha256-553c4a3f747b3d22a4946875f1cc8ed011c2930d83f864a0c7265f9ec0a20413"), filepath.Join(p, "blobs", "sha256-c608dc615584cd20d9d830363dabf8a4783ae5d34245c3d8c115edb3bc7b28e4"), - filepath.Join(p, "blobs", "sha256-f836ee110db21567f826332e4cedd746c06d10664fd5a9ea3659e3683a944510"), + filepath.Join(p, "blobs", "sha256-ea34c57ba5b78b740aafe2aeb74dc6507fc3ad14170b64c26a04fb9e36c88d75"), }) }) diff --git a/template/alfred.json b/template/alfred.json new file mode 100644 index 00000000..edac21af --- /dev/null +++ b/template/alfred.json @@ -0,0 +1,8 @@ +{ + "stop": [ + "", + "", + "", + "" + ] +} diff --git a/template/alpaca.json b/template/alpaca.json new file mode 100644 index 00000000..eafe2b8a --- /dev/null +++ b/template/alpaca.json @@ -0,0 +1,6 @@ +{ + "stop": [ + "### Instruction:", + "### Response" + ] +} diff --git a/template/chatml.json b/template/chatml.json new file mode 100644 index 00000000..7afeb3de --- /dev/null +++ b/template/chatml.json @@ -0,0 +1,6 @@ +{ + "stop": [ + "<|im_start|>", + "<|im_end|>" + ] +} diff --git a/template/chatqa.json b/template/chatqa.json new file mode 100644 index 00000000..64dd0f33 --- /dev/null +++ b/template/chatqa.json @@ -0,0 +1,8 @@ +{ + "stop": [ + "System:", + "User:", + "Assistant:", + "<|begin_of_text|>" + ] +} diff --git a/template/codellama-70b-instruct.json b/template/codellama-70b-instruct.json new file mode 100644 index 00000000..a56a63f1 --- /dev/null +++ b/template/codellama-70b-instruct.json @@ -0,0 +1,7 @@ +{ + "stop": [ + "Source:", + "Destination:", + "" + ] +} diff --git a/template/falcon-instruct.json b/template/falcon-instruct.json new file mode 100644 index 00000000..a0da0e81 --- /dev/null +++ b/template/falcon-instruct.json @@ -0,0 +1,6 @@ +{ + "stop": [ + "User:", + "Assistant:" + ] +} diff --git a/template/gemma-instruct.json b/template/gemma-instruct.json new file mode 100644 index 00000000..f4ad415c --- /dev/null +++ b/template/gemma-instruct.json @@ -0,0 +1,6 @@ +{ + "stop": [ + "", + "" + ] +} diff --git a/template/granite-instruct.json b/template/granite-instruct.json new file mode 100644 index 00000000..0933e4b5 --- /dev/null +++ b/template/granite-instruct.json @@ -0,0 +1,7 @@ +{ + "stop": [ + "System:", + "Question:", + "Answer:" + ] +} diff --git a/template/llama2-chat.json b/template/llama2-chat.json new file mode 100644 index 00000000..17590ab4 --- /dev/null +++ b/template/llama2-chat.json @@ -0,0 +1,8 @@ +{ + "stop": [ + "[INST]", + "[/INST]", + "<>", + "<>" + ] +} diff --git a/template/llama3-instruct.json b/template/llama3-instruct.json new file mode 100644 index 00000000..c4e9d448 --- /dev/null +++ b/template/llama3-instruct.json @@ -0,0 +1,7 @@ +{ + "stop": [ + "<|start_header_id|>", + "<|end_header_id|>", + "<|eot_id|>" + ] +} diff --git a/template/magicoder.json b/template/magicoder.json new file mode 100644 index 00000000..6f67cab0 --- /dev/null +++ b/template/magicoder.json @@ -0,0 +1,6 @@ +{ + "stop": [ + "@@ Instruction", + "@@ Response" + ] +} diff --git a/template/mistral-instruct.json b/template/mistral-instruct.json new file mode 100644 index 00000000..7afeb3de --- /dev/null +++ b/template/mistral-instruct.json @@ -0,0 +1,6 @@ +{ + "stop": [ + "<|im_start|>", + "<|im_end|>" + ] +} diff --git a/template/openchat.json b/template/openchat.json new file mode 100644 index 00000000..0edc341f --- /dev/null +++ b/template/openchat.json @@ -0,0 +1,5 @@ +{ + "stop": [ + "<|end_of_turn|>" + ] +} diff --git a/template/phi-3.json b/template/phi-3.json new file mode 100644 index 00000000..27bf7664 --- /dev/null +++ b/template/phi-3.json @@ -0,0 +1,8 @@ +{ + "stop": [ + "<|end|>", + "<|system|>", + "<|user|>", + "<|assistant|>" + ] +} diff --git a/template/solar-instruct.json b/template/solar-instruct.json new file mode 100644 index 00000000..7b7a9050 --- /dev/null +++ b/template/solar-instruct.json @@ -0,0 +1,7 @@ +{ + "stop": [ + "### System:", + "### User:", + "### Assistant" + ] +} diff --git a/template/starcoder2-instruct.json b/template/starcoder2-instruct.json new file mode 100644 index 00000000..31348908 --- /dev/null +++ b/template/starcoder2-instruct.json @@ -0,0 +1,7 @@ +{ + "stop": [ + "### Instruction", + "### Response", + "<|endoftext|>" + ] +} diff --git a/template/template.go b/template/template.go index 9b351666..9bb6a399 100644 --- a/template/template.go +++ b/template/template.go @@ -23,6 +23,7 @@ import ( var indexBytes []byte //go:embed *.gotmpl +//go:embed *.json var templatesFS embed.FS var templatesOnce = sync.OnceValues(func() ([]*named, error) { @@ -39,6 +40,15 @@ var templatesOnce = sync.OnceValues(func() ([]*named, error) { // normalize line endings t.Bytes = bytes.ReplaceAll(bts, []byte("\r\n"), []byte("\n")) + + params, err := templatesFS.ReadFile(t.Name + ".json") + if err != nil { + continue + } + + if err := json.Unmarshal(params, &t.Parameters); err != nil { + return nil, err + } } return templates, nil @@ -48,6 +58,10 @@ type named struct { Name string `json:"name"` Template string `json:"template"` Bytes []byte + + Parameters *struct { + Stop []string `json:"stop"` + } } func (t named) Reader() io.Reader { diff --git a/template/vicuna.json b/template/vicuna.json new file mode 100644 index 00000000..ed7bfb0f --- /dev/null +++ b/template/vicuna.json @@ -0,0 +1,6 @@ +{ + "stop": [ + "USER:", + "ASSISTANT:" + ] +} diff --git a/template/zephyr.json b/template/zephyr.json new file mode 100644 index 00000000..f9c0115c --- /dev/null +++ b/template/zephyr.json @@ -0,0 +1,8 @@ +{ + "stop": [ + "<|system|>", + "", + "<|user|>", + "<|assistant|>" + ] +}