gered/ollama
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

only check system memory on macos

Browse source
This commit is contained in:
Michael Yang 2023-10-13 14:41:51 -07:00
parent 4a8931f634
commit 36fe2deebf
1 changed files with 21 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

49
llm/llm.go
View file

@ -56,38 +56,31 @@ func New(workDir, model string, adapters []string, opts api.Options) (LLM, error
opts.NumGPU = 0 opts.NumGPU = 0
} }
} }
}
var requiredMemory int64 var requiredMemory int64
var f16Multiplier int64 = 2 var f16Multiplier int64 = 2
switch ggml.ModelType() { switch ggml.ModelType() {
case "3B", "7B": case "3B", "7B":
requiredMemory = 8 * format.GigaByte requiredMemory = 8 * format.GigaByte
case "13B": case "13B":
requiredMemory = 16 * format.GigaByte requiredMemory = 16 * format.GigaByte
case "30B", "34B", "40B": case "30B", "34B", "40B":
requiredMemory = 32 * format.GigaByte requiredMemory = 32 * format.GigaByte
case "65B", "70B": case "65B", "70B":
requiredMemory = 64 * format.GigaByte requiredMemory = 64 * format.GigaByte
case "180B": case "180B":
requiredMemory = 128 * format.GigaByte requiredMemory = 128 * format.GigaByte
f16Multiplier = 4 f16Multiplier = 4
} }
systemMemory := int64(memory.TotalMemory()) systemMemory := int64(memory.TotalMemory())
videoMemory, err := CheckVRAM() if ggml.FileType() == "F16" && requiredMemory*f16Multiplier > systemMemory {
if err != nil{ return nil, fmt.Errorf("F16 model requires at least %s of total memory", format.HumanBytes(requiredMemory))
videoMemory = 0 } else if requiredMemory > systemMemory {
} return nil, fmt.Errorf("model requires at least %s of total memory", format.HumanBytes(requiredMemory))
}
totalMemory := systemMemory + videoMemory
if ggml.FileType() == "F16" && requiredMemory*f16Multiplier > totalMemory {
return nil, fmt.Errorf("F16 model requires at least %s of total memory", format.HumanBytes(requiredMemory))
} else if requiredMemory > totalMemory {
return nil, fmt.Errorf("model requires at least %s of total memory", format.HumanBytes(requiredMemory))
} }
switch ggml.Name() { switch ggml.Name() {