From 3144e2a4394a544375ac054375d838525f8096c7 Mon Sep 17 00:00:00 2001
From: Bruce MacDonald <brucewmacdonald@gmail.com>
Date: Tue, 12 Dec 2023 12:33:02 -0500
Subject: [PATCH] exponential back-off (#1484)

---
 llm/llama.go | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/llm/llama.go b/llm/llama.go
index 26a7ee77..72e67389 100644
--- a/llm/llama.go
+++ b/llm/llama.go
@@ -545,8 +545,7 @@ type prediction struct {
 }
 
 const maxBufferSize = 512 * format.KiloByte
-const maxRetries = 3
-const retryDelay = 1 * time.Second
+const maxRetries = 6
 
 type PredictOpts struct {
 	Prompt           string
@@ -610,9 +609,11 @@ func (llm *llama) Predict(ctx context.Context, predict PredictOpts, fn func(Pred
 		request["grammar"] = jsonGrammar
 	}
 
+	retryDelay := 100 * time.Microsecond
 	for retries := 0; retries < maxRetries; retries++ {
 		if retries > 0 {
 			time.Sleep(retryDelay) // wait before retrying
+			retryDelay *= 2        // exponential backoff
 		}
 
 		// Handling JSON marshaling with special characters unescaped.