From e9a9580bdd7082123f641ce58568459237ead8bd Mon Sep 17 00:00:00 2001 From: Bruce MacDonald Date: Mon, 14 Aug 2023 10:34:17 -0300 Subject: [PATCH 1/4] do not regenerate embeddings - re-use previously evaluated embeddings when possible - change embeddings digest identifier to be based on model name and embedded file path --- server/images.go | 52 +++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/server/images.go b/server/images.go index 2177fe6d..63ab2a68 100644 --- a/server/images.go +++ b/server/images.go @@ -514,6 +514,14 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) { continue } addedFiles[filePath] = true + // check if we already have embeddings for this file path + layerIdentifier := fmt.Sprintf("%s:%s", filePath, e.model) + digest, _ := GetSHA256Digest(strings.NewReader(layerIdentifier)) + existing, err := existingFileEmbeddings(digest) + if err != nil { + return nil, fmt.Errorf("failed to check existing embeddings for file %s: %v", filePath, err) + } + // TODO: check file type f, err := os.Open(filePath) if err != nil { @@ -542,6 +550,11 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) { Total: len(data) - 1, Completed: i, }) + if len(existing[d]) > 0 { + // already have an embedding for this line + embeddings = append(embeddings, vector.Embedding{Data: d, Vector: existing[d]}) + continue + } embed, err := llmModel.Embedding(d) if err != nil { log.Printf("failed to generate embedding for '%s' line %d: %v", filePath, i+1, err) @@ -556,17 +569,11 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) { } r := bytes.NewReader(b) - digest, size := GetSHA256Digest(r) - // Reset the position of the reader after calculating the digest - if _, err := r.Seek(0, io.SeekStart); err != nil { - return nil, fmt.Errorf("could not reset embed reader: %w", err) - } - layer := &LayerReader{ Layer: Layer{ MediaType: "application/vnd.ollama.image.embed", Digest: digest, - Size: size, + Size: r.Len(), }, Reader: r, } @@ -578,6 +585,32 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) { return layers, nil } +// existingFileEmbeddings checks if we already have embeddings for a file and loads them into a look-up map +func existingFileEmbeddings(digest string) (map[string][]float64, error) { + path, err := GetBlobsPath(digest) + if err != nil { + return nil, fmt.Errorf("embeddings blobs path: %w", err) + } + existingFileEmbeddings := make(map[string][]float64) + if _, err := os.Stat(path); err == nil { + // already have some embeddings for this file, load embeddings previously generated + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to open existing embedding file: %s", err) + } + defer file.Close() + + existing := []vector.Embedding{} + if err = json.NewDecoder(file).Decode(&existing); err != nil { + return nil, err + } + for _, e := range existing { + existingFileEmbeddings[e.Data] = e.Vector + } + } + return existingFileEmbeddings, nil +} + func removeLayerFromLayers(layers []*LayerReader, mediaType string) []*LayerReader { j := 0 for _, l := range layers { @@ -598,7 +631,8 @@ func SaveLayers(layers []*LayerReader, fn func(resp api.ProgressResponse), force } _, err = os.Stat(fp) - if os.IsNotExist(err) || force { + // note: embed layers are always written since their digest doesnt indicate anything about the contents + if os.IsNotExist(err) || force || layer.MediaType == "application/vnd.ollama.image.embed" { fn(api.ProgressResponse{Status: fmt.Sprintf("writing layer %s", layer.Digest)}) out, err := os.Create(fp) @@ -1180,7 +1214,7 @@ func makeRequest(ctx context.Context, method, url string, headers map[string]str var ok bool if retries, ok = retryCtx.(int); ok { if retries > MaxRetries { - return nil, fmt.Errorf("Maximum retries hit; are you sure you have access to this resource?") + return nil, fmt.Errorf("maximum retries hit; are you sure you have access to this resource?") } } From 99b6b600857b6441302ebf94ab9171ba48de1399 Mon Sep 17 00:00:00 2001 From: Bruce MacDonald Date: Mon, 14 Aug 2023 11:57:12 -0300 Subject: [PATCH 2/4] use model bin digest for embed digest --- server/images.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/server/images.go b/server/images.go index 63ab2a68..cc07bd0f 100644 --- a/server/images.go +++ b/server/images.go @@ -502,6 +502,12 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) { } }() + // this will be used to check if we already have embeddings for a file + modelDigest, err := modelDigest(e.model) + if err != nil { + return nil, fmt.Errorf("model digest: %w", err) + } + addedFiles := make(map[string]bool) // keep track of files that have already been added for _, filePattern := range e.files { matchingFiles, err := filepath.Glob(filePattern) @@ -515,7 +521,7 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) { } addedFiles[filePath] = true // check if we already have embeddings for this file path - layerIdentifier := fmt.Sprintf("%s:%s", filePath, e.model) + layerIdentifier := fmt.Sprintf("%s:%s", filePath, modelDigest) digest, _ := GetSHA256Digest(strings.NewReader(layerIdentifier)) existing, err := existingFileEmbeddings(digest) if err != nil { @@ -585,6 +591,17 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) { return layers, nil } +func modelDigest(modelPath string) (string, error) { + modelFile, err := os.Open(modelPath) + if err != nil { + return "", fmt.Errorf("could not open model blob: %w", err) + } + defer modelFile.Close() + + digest, _ := GetSHA256Digest(modelFile) + return digest, nil +} + // existingFileEmbeddings checks if we already have embeddings for a file and loads them into a look-up map func existingFileEmbeddings(digest string) (map[string][]float64, error) { path, err := GetBlobsPath(digest) From 2c8b680b036dd80fb1261f321e9d3e760321375f Mon Sep 17 00:00:00 2001 From: Bruce MacDonald Date: Mon, 14 Aug 2023 12:11:04 -0300 Subject: [PATCH 3/4] use file info for embeddings cache --- server/images.go | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/server/images.go b/server/images.go index cc07bd0f..494f927e 100644 --- a/server/images.go +++ b/server/images.go @@ -503,9 +503,9 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) { }() // this will be used to check if we already have embeddings for a file - modelDigest, err := modelDigest(e.model) + modelInfo, err := os.Stat(e.model) if err != nil { - return nil, fmt.Errorf("model digest: %w", err) + log.Fatalf("Error getting file info: %s", err) } addedFiles := make(map[string]bool) // keep track of files that have already been added @@ -521,7 +521,7 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) { } addedFiles[filePath] = true // check if we already have embeddings for this file path - layerIdentifier := fmt.Sprintf("%s:%s", filePath, modelDigest) + layerIdentifier := fmt.Sprintf("%s:%s:%s:%d", filePath, e.model, modelInfo.ModTime().Format("2006-01-02 15:04:05"), modelInfo.Size()) digest, _ := GetSHA256Digest(strings.NewReader(layerIdentifier)) existing, err := existingFileEmbeddings(digest) if err != nil { @@ -591,17 +591,6 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) { return layers, nil } -func modelDigest(modelPath string) (string, error) { - modelFile, err := os.Open(modelPath) - if err != nil { - return "", fmt.Errorf("could not open model blob: %w", err) - } - defer modelFile.Close() - - digest, _ := GetSHA256Digest(modelFile) - return digest, nil -} - // existingFileEmbeddings checks if we already have embeddings for a file and loads them into a look-up map func existingFileEmbeddings(digest string) (map[string][]float64, error) { path, err := GetBlobsPath(digest) From 18f2cb04722900ecac2ed2d8a483f22386c62551 Mon Sep 17 00:00:00 2001 From: Bruce MacDonald Date: Tue, 15 Aug 2023 10:39:59 -0300 Subject: [PATCH 4/4] dont log fatal --- server/images.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/images.go b/server/images.go index 494f927e..960f2d58 100644 --- a/server/images.go +++ b/server/images.go @@ -505,7 +505,7 @@ func embeddingLayers(e EmbeddingParams) ([]*LayerReader, error) { // this will be used to check if we already have embeddings for a file modelInfo, err := os.Stat(e.model) if err != nil { - log.Fatalf("Error getting file info: %s", err) + return nil, fmt.Errorf("failed to get model file info: %v", err) } addedFiles := make(map[string]bool) // keep track of files that have already been added