diff --git a/modules/lfs/errors.go b/modules/lfs/errors.go new file mode 100644 index 0000000000..0795f0bb79 --- /dev/null +++ b/modules/lfs/errors.go @@ -0,0 +1,60 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package lfs + +import ( + "fmt" +) + +type ErrLFS struct { + OID string + Err error +} + +type ErrLFSDownload struct { + ErrLFS +} + +type ErrLFSUpload struct { + ErrLFS +} + +type ErrLFSVerify struct { + ErrLFS +} + +func (e *ErrLFS) Error() string { + return fmt.Sprintf("LFS error for object %s: %v", e.OID, e.Err) +} + +func (e *ErrLFS) Unwrap() error { + return e.Err +} + +func (e *ErrLFSDownload) Error() string { + return fmt.Sprintf("LFS error while downloading [%s]: %s", e.OID, e.Err) +} + +func (e *ErrLFSUpload) Error() string { + return fmt.Sprintf("LFS error while uploading [%s]: %s", e.OID, e.Err) +} + +func (e *ErrLFSVerify) Error() string { + return fmt.Sprintf("LFS error while verifying [%s]: %s", e.OID, e.Err) +} + +func (e *ErrLFSDownload) Is(target error) bool { + _, ok := target.(*ErrLFSDownload) + return ok +} + +func (e *ErrLFSUpload) Is(target error) bool { + _, ok := target.(*ErrLFSUpload) + return ok +} + +func (e *ErrLFSVerify) Is(target error) bool { + _, ok := target.(*ErrLFSVerify) + return ok +} diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go index 7ee2449b0e..732698e13c 100644 --- a/modules/lfs/http_client.go +++ b/modules/lfs/http_client.go @@ -71,9 +71,12 @@ func (c *HTTPClient) batch(ctx context.Context, operation string, objects []Poin url := fmt.Sprintf("%s/objects/batch", c.endpoint) - request := &BatchRequest{operation, c.transferNames(), nil, objects} payload := new(bytes.Buffer) - err := json.NewEncoder(payload).Encode(request) + err := json.NewEncoder(payload).Encode(&BatchRequest{ + Operation: operation, + Transfers: c.transferNames(), + Objects: objects, + }) if err != nil { log.Error("Error encoding json: %v", err) return nil, err @@ -134,15 +137,19 @@ func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc return fmt.Errorf("TransferAdapter not found: %s", result.Transfer) } + var resultErrors []error for _, object := range result.Objects { if object.Error != nil { - objectError := errors.New(object.Error.Message) - log.Trace("Error on object %v: %v", object.Pointer, objectError) + log.Trace("Error on object %v: %v", object.Pointer, object.Error.Message) if uc != nil { + objectError := &ErrLFSUpload{ErrLFS{OID: object.Oid, Err: object.Error}} + resultErrors = append(resultErrors, objectError) if _, err := uc(object.Pointer, objectError); err != nil { return err } } else { + objectError := &ErrLFSDownload{ErrLFS{OID: object.Oid, Err: object.Error}} + resultErrors = append(resultErrors, objectError) if err := dc(object.Pointer, nil, objectError); err != nil { return err } @@ -164,18 +171,33 @@ func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc content, err := uc(object.Pointer, nil) if err != nil { - return err + if errors.Is(err, &ErrLFSUpload{}) { + resultErrors = append(resultErrors, err) + continue + } else { + return err + } } err = transferAdapter.Upload(ctx, link, object.Pointer, content) if err != nil { - return err + if errors.Is(err, &ErrLFSUpload{}) { + resultErrors = append(resultErrors, err) + continue + } else { + return err + } } link, ok = object.Actions["verify"] if ok { if err := transferAdapter.Verify(ctx, link, object.Pointer); err != nil { - return err + if errors.Is(err, &ErrLFSVerify{}) { + resultErrors = append(resultErrors, err) + continue + } else { + return err + } } } } else { @@ -191,16 +213,26 @@ func (c *HTTPClient) performOperation(ctx context.Context, objects []Pointer, dc content, err := transferAdapter.Download(ctx, link) if err != nil { - return err + if errors.Is(err, &ErrLFSDownload{}) { + resultErrors = append(resultErrors, err) + continue + } else { + return err + } } if err := dc(object.Pointer, content, nil); err != nil { - return err + if errors.Is(err, &ErrLFSDownload{}) { + resultErrors = append(resultErrors, err) + continue + } else { + return err + } } } } - return nil + return errors.Join(resultErrors...) } // createRequest creates a new request, and sets the headers. diff --git a/modules/lfs/shared.go b/modules/lfs/shared.go index 675d2328b7..9b8476eb6f 100644 --- a/modules/lfs/shared.go +++ b/modules/lfs/shared.go @@ -4,6 +4,7 @@ package lfs import ( + "fmt" "time" ) @@ -64,6 +65,10 @@ type ObjectError struct { Message string `json:"message"` } +func (oe *ObjectError) Error() string { + return fmt.Sprintf("code: %d, message: %s", oe.Code, oe.Message) +} + // PointerBlob associates a Git blob with a Pointer. type PointerBlob struct { Hash string diff --git a/modules/repository/repo.go b/modules/repository/repo.go index 7be7c3dba6..38cd02b021 100644 --- a/modules/repository/repo.go +++ b/modules/repository/repo.go @@ -5,6 +5,7 @@ package repository import ( "context" + "errors" "fmt" "io" "strings" @@ -178,12 +179,17 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Re errChan := make(chan error, 1) go lfs.SearchPointerBlobs(ctx, gitRepo, pointerChan, errChan) + var lfsDownloadErrors []error + downloadObjects := func(pointers []lfs.Pointer) error { err := lfsClient.Download(ctx, pointers, func(p lfs.Pointer, content io.ReadCloser, objectError error) error { if objectError != nil { - log.Error("Repo[%-v]: Ignoring LFS object %-v: %v", repo, p, objectError) - // TODO: Optionally return error to ensure data integrity of LFS objects - return nil + if errors.Is(objectError, &lfs.ErrLFSDownload{}) { + log.Trace("Repo[%-v]: Ignoring LFS error: %v", repo, objectError) + lfsDownloadErrors = append(lfsDownloadErrors, objectError) + return nil + } + return objectError } defer content.Close() @@ -210,10 +216,13 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Re default: } } + if errors.Is(err, &lfs.ErrLFSDownload{}) { + return nil + } return err } - var batch []lfs.Pointer + var downloadBatch []lfs.Pointer for pointerBlob := range pointerChan { meta, err := git_model.GetLFSMetaObjectByOid(ctx, repo.ID, pointerBlob.Oid) if err != nil && err != git_model.ErrLFSObjectNotExist { @@ -235,28 +244,32 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Re if exist { log.Trace("Repo[%-v]: LFS object %-v already present; creating meta object", repo, pointerBlob.Pointer) - _, err := git_model.NewLFSMetaObject(ctx, repo.ID, pointerBlob.Pointer) + m, err := git_model.NewLFSMetaObject(ctx, repo.ID, pointerBlob.Pointer) if err != nil { log.Error("Repo[%-v]: Error creating LFS meta object %-v: %v", repo, pointerBlob.Pointer, err) return err } + if m.Existing { + log.Trace("Repo[%-v]: LFS meta object %-v already present; skip adding it to download batch", repo, m) + continue + } } else { if setting.LFS.MaxFileSize > 0 && pointerBlob.Size > setting.LFS.MaxFileSize { log.Info("Repo[%-v]: LFS object %-v download denied because of LFS_MAX_FILE_SIZE=%d < size %d", repo, pointerBlob.Pointer, setting.LFS.MaxFileSize, pointerBlob.Size) continue } - batch = append(batch, pointerBlob.Pointer) - if len(batch) >= lfsClient.BatchSize() { - if err := downloadObjects(batch); err != nil { + downloadBatch = append(downloadBatch, pointerBlob.Pointer) + if len(downloadBatch) >= lfsClient.BatchSize() { + if err := downloadObjects(downloadBatch); err != nil { return err } - batch = nil + downloadBatch = nil } } } - if len(batch) > 0 { - if err := downloadObjects(batch); err != nil { + if len(downloadBatch) > 0 { + if err := downloadObjects(downloadBatch); err != nil { return err } } @@ -267,7 +280,7 @@ func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *repo_model.Re return err } - return nil + return errors.Join(lfsDownloadErrors...) } // shortRelease to reduce load memory, this struct can replace repo_model.Release diff --git a/services/repository/migrate.go b/services/repository/migrate.go index f2f2b51858..caabbdc0d5 100644 --- a/services/repository/migrate.go +++ b/services/repository/migrate.go @@ -169,7 +169,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, lfsClient := lfs.NewClient(endpoint, httpTransport) if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, lfsClient); err != nil { log.Error("Failed to store missing LFS objects for repository: %v", err) - // TODO: Optionally return error to ensure data integrity of LFS objects + // TODO: check for lfs.ErrLFSDownload errors and display them } } }