diff --git a/server/download.go b/server/download.go
index 932e29a0..72aa0d77 100644
--- a/server/download.go
+++ b/server/download.go
@@ -113,6 +113,7 @@ var chunkSize = 1024 * 1024 // 1 MiB in bytes
 
 // doDownload downloads a blob from the registry and stores it in the blobs directory
 func doDownload(ctx context.Context, mp ModelPath, regOpts *RegistryOptions, f *FileDownload, fn func(api.ProgressResponse)) error {
+	defer inProgress.Delete(f.Digest)
 	var size int64
 
 	fi, err := os.Stat(f.FilePath + "-partial")
@@ -208,8 +209,6 @@ outerLoop:
 		inProgress.Store(f.Digest, f)
 	}
 
-	inProgress.Delete(f.Digest)
-
 	log.Printf("success getting %s\n", f.Digest)
 	return nil
 }