From beb49eef65acefc64a6ae0562ce58467e6974fde Mon Sep 17 00:00:00 2001 From: Michael Yang Date: Fri, 7 Jun 2024 14:55:56 -0700 Subject: [PATCH] create bert models from cli --- cmd/cmd.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cmd/cmd.go b/cmd/cmd.go index fd7246c8..a8a02605 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -223,6 +223,14 @@ func tempZipFiles(path string) (string, error) { } files = append(files, js...) + // bert models require a nested config.json + // TODO(mxyng): merge this with the glob above + js, err = glob(filepath.Join(path, "**/*.json"), "text/plain") + if err != nil { + return "", err + } + files = append(files, js...) + if tks, _ := glob(filepath.Join(path, "tokenizer.model"), "application/octet-stream"); len(tks) > 0 { // add tokenizer.model if it exists, tokenizer.json is automatically picked up by the previous glob // tokenizer.model might be a unresolved git lfs reference; error if it is @@ -252,6 +260,11 @@ func tempZipFiles(path string) (string, error) { return "", err } + zfi.Name, err = filepath.Rel(path, file) + if err != nil { + return "", err + } + zf, err := zipfile.CreateHeader(zfi) if err != nil { return "", err