From b25dd1795dcf7879a47508e68b650a6a53231443 Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Sat, 26 Aug 2023 08:33:03 -0700 Subject: [PATCH] allow F16 to use metal warning F16 uses significantly more memory than quantized model so the standard requires don't apply. --- llm/llm.go | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/llm/llm.go b/llm/llm.go index 0dc862b4..789caaf9 100644 --- a/llm/llm.go +++ b/llm/llm.go @@ -36,11 +36,11 @@ func New(model string, adapters []string, opts api.Options) (LLM, error) { } switch ggml.FileType().String() { - case "F32", "F16", "Q5_0", "Q5_1", "Q8_0": + case "F32", "Q5_0", "Q5_1", "Q8_0": if opts.NumGPU != 0 { // F32, F16, Q5_0, Q5_1, and Q8_0 do not support Metal API and will // cause the runner to segmentation fault so disable GPU - log.Printf("WARNING: GPU disabled for F32, F16, Q5_0, Q5_1, and Q8_0") + log.Printf("WARNING: GPU disabled for F32, Q5_0, Q5_1, and Q8_0") opts.NumGPU = 0 } } @@ -48,19 +48,27 @@ func New(model string, adapters []string, opts api.Options) (LLM, error) { totalResidentMemory := memory.TotalMemory() switch ggml.ModelType() { case ModelType3B, ModelType7B: - if totalResidentMemory < 8*1024*1024 { + if ggml.FileType().String() == "F16" && totalResidentMemory < 16*1024*1024 { + return nil, fmt.Errorf("F16 model requires at least 16GB of memory") + } else if totalResidentMemory < 8*1024*1024 { return nil, fmt.Errorf("model requires at least 8GB of memory") } case ModelType13B: - if totalResidentMemory < 16*1024*1024 { + if ggml.FileType().String() == "F16" && totalResidentMemory < 32*1024*1024 { + return nil, fmt.Errorf("F16 model requires at least 32GB of memory") + } else if totalResidentMemory < 16*1024*1024 { return nil, fmt.Errorf("model requires at least 16GB of memory") } case ModelType30B, ModelType34B: - if totalResidentMemory < 32*1024*1024 { + if ggml.FileType().String() == "F16" && totalResidentMemory < 64*1024*1024 { + return nil, fmt.Errorf("F16 model requires at least 64GB of memory") + } else if totalResidentMemory < 32*1024*1024 { return nil, fmt.Errorf("model requires at least 32GB of memory") } case ModelType65B: - if totalResidentMemory < 64*1024*1024 { + if ggml.FileType().String() == "F16" && totalResidentMemory < 128*1024*1024 { + return nil, fmt.Errorf("F16 model requires at least 128GB of memory") + } else if totalResidentMemory < 64*1024*1024 { return nil, fmt.Errorf("model requires at least 64GB of memory") } }