From 19b7a4d7150cfa69d23e7066abb6cea8d4f79cfa Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Mon, 20 Nov 2023 13:44:12 -0800 Subject: [PATCH] recent llama.cpp update added kernels for fp32, q5_0, and q5_1 --- llm/llm.go | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/llm/llm.go b/llm/llm.go index 22706da5..4901d9fe 100644 --- a/llm/llm.go +++ b/llm/llm.go @@ -41,20 +41,13 @@ func New(workDir, model string, adapters []string, opts api.Options) (LLM, error if runtime.GOOS == "darwin" { switch ggml.FileType() { - case "Q8_0": + case "F32", "Q5_0", "Q5_1", "Q8_0": if ggml.Name() != "gguf" && opts.NumGPU != 0 { // GGML Q8_0 do not support Metal API and will // cause the runner to segmentation fault so disable GPU log.Printf("WARNING: GPU disabled for F32, Q5_0, Q5_1, and Q8_0") opts.NumGPU = 0 } - case "F32", "Q5_0", "Q5_1": - if opts.NumGPU != 0 { - // F32, Q5_0, Q5_1, and Q8_0 do not support Metal API and will - // cause the runner to segmentation fault so disable GPU - log.Printf("WARNING: GPU disabled for F32, Q5_0, Q5_1, and Q8_0") - opts.NumGPU = 0 - } } var requiredMemory int64