From 4458efb73afe76ead7e0ec3e088564fb3aea1900 Mon Sep 17 00:00:00 2001 From: Jeffrey Morgan Date: Mon, 22 Jan 2024 17:40:06 -0800 Subject: [PATCH] Load all layers on `arm64` macOS if model is small enough (#2149) --- llm/llm.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llm/llm.go b/llm/llm.go index 8b043f38..8e2f0714 100644 --- a/llm/llm.go +++ b/llm/llm.go @@ -70,7 +70,8 @@ func New(workDir, model string, adapters, projectors []string, opts api.Options) break } - opts.NumGPU = 1 + // TODO: implement layer splitting on macOS + opts.NumGPU = 999 default: if info.Library == "cpu" { slog.Info("GPU not available, falling back to CPU")