gered/ollama
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

exponential back-off (#1484)

Browse source
This commit is contained in:
Bruce MacDonald 2023-12-12 12:33:02 -05:00 committed by GitHub
parent c0960e29b5
commit 3144e2a439
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 3 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
llm/llama.go
View file

@ -545,8 +545,7 @@ type prediction struct {
} }
const maxBufferSize = 512 * format.KiloByte const maxBufferSize = 512 * format.KiloByte
const maxRetries = 3 const maxRetries = 6
const retryDelay = 1 * time.Second
type PredictOpts struct { type PredictOpts struct {
Prompt string Prompt string
@ -610,9 +609,11 @@ func (llm *llama) Predict(ctx context.Context, predict PredictOpts, fn func(Pred
request["grammar"] = jsonGrammar request["grammar"] = jsonGrammar
} }
retryDelay := 100 * time.Microsecond
for retries := 0; retries < maxRetries; retries++ { for retries := 0; retries < maxRetries; retries++ {
if retries > 0 { if retries > 0 {
time.Sleep(retryDelay) // wait before retrying time.Sleep(retryDelay) // wait before retrying
retryDelay *= 2 // exponential backoff
} }
// Handling JSON marshaling with special characters unescaped. // Handling JSON marshaling with special characters unescaped.