gered/ollama
1
0
Fork 0
Code Issues Pull requests Actions Packages Projects Releases Wiki Activity

fix resumable downloads

Browse source 
glob returns files in lexical order which is not appropriate when
rebuilding the parts list
This commit is contained in:
Michael Yang 2023-10-02 15:26:27 -07:00
parent 090d08422b
commit 711e891f0f
1 changed files with 23 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
server/download.go
View file

@ -12,6 +12,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"strconv" "strconv"
"strings"
"sync" "sync"
"sync/atomic" "sync/atomic"
"time" "time"
@ -39,9 +40,18 @@ type blobDownload struct {
} }
type blobDownloadPart struct { type blobDownloadPart struct {
N int
Offset int64 Offset int64
Size int64 Size int64
Completed int64 Completed int64
*blobDownload `json:"-"`
}
func (p *blobDownloadPart) Name() string {
return strings.Join([]string{
p.blobDownload.Name, "partial", strconv.Itoa(p.N),
}, "-")
} }
func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *RegistryOptions) error { func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *RegistryOptions) error {
@ -78,14 +88,10 @@ func (b *blobDownload) Prepare(ctx context.Context, requestURL *url.URL, opts *R
size = b.Total - offset size = b.Total - offset
} }
partName := b.Name + "-partial-" + strconv.Itoa(len(b.Parts)) if err := b.newPart(offset, size); err != nil {
part := blobDownloadPart{Offset: offset, Size: size}
if err := b.writePart(partName, &part); err != nil {
return err return err
} }
b.Parts = append(b.Parts, &part)
offset += size offset += size
} }
} }
@ -161,7 +167,6 @@ func (b *blobDownload) Run(ctx context.Context, requestURL *url.URL, opts *Regis
func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, i int, opts *RegistryOptions) error { func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, i int, opts *RegistryOptions) error {
part := b.Parts[i] part := b.Parts[i]
partName := b.File.Name() + "-" + strconv.Itoa(i)
offset := part.Offset + part.Completed offset := part.Offset + part.Completed
w := io.NewOffsetWriter(b.File, offset) w := io.NewOffsetWriter(b.File, offset)
@ -181,7 +186,7 @@ func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, i
} }
part.Completed += n part.Completed += n
if err := b.writePart(partName, part); err != nil { if err := b.writePart(part.Name(), part); err != nil {
return err return err
} }
@ -189,6 +194,16 @@ func (b *blobDownload) downloadChunk(ctx context.Context, requestURL *url.URL, i
return err return err
} }
func (b *blobDownload) newPart(offset, size int64) error {
part := blobDownloadPart{blobDownload: b, Offset: offset, Size: size, N: len(b.Parts)}
if err := b.writePart(part.Name(), &part); err != nil {
return err
}
b.Parts = append(b.Parts, &part)
return nil
}
func (b *blobDownload) readPart(partName string) (*blobDownloadPart, error) { func (b *blobDownload) readPart(partName string) (*blobDownloadPart, error) {
var part blobDownloadPart var part blobDownloadPart
partFile, err := os.Open(partName) partFile, err := os.Open(partName)
@ -201,6 +216,7 @@ func (b *blobDownload) readPart(partName string) (*blobDownloadPart, error) {
return nil, err return nil, err
} }
part.blobDownload = b
return &part, nil return &part, nil
} }