gered/ollama
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3465 from ollama/mxyng/fix-metal

Browse source 
fix metal gpu
This commit is contained in:
Michael Yang 2024-04-02 16:29:58 -07:00 committed by GitHub
parent a57818d93e 80163ebcb5
commit a039e383cd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
llm/server.go
View file

@ -85,7 +85,7 @@ func NewLlamaServer(model string, adapters, projectors []string, opts *api.Optio
graph := int64(ggml.KV().GQA()) * kv / 6 graph := int64(ggml.KV().GQA()) * kv / 6
usedMemory += graph usedMemory += graph
if usedMemory > availableMemory || slices.Contains(cpuOnlyFamilies, ggml.KV().Architecture()) { if (usedMemory > availableMemory || slices.Contains(cpuOnlyFamilies, ggml.KV().Architecture())) && info.Library != "metal" {
info.Library = "cpu" info.Library = "cpu"
} }
@ -159,7 +159,7 @@ func NewLlamaServer(model string, adapters, projectors []string, opts *api.Optio
params = append(params, "--log-disable") params = append(params, "--log-disable")
} }
if opts.NumGPU > 0 { if opts.NumGPU >= 0 {
params = append(params, "--n-gpu-layers", fmt.Sprintf("%d", opts.NumGPU)) params = append(params, "--n-gpu-layers", fmt.Sprintf("%d", opts.NumGPU))
} }