From b1ad8ccb7323b8e517fc39f5e37a7a74574c16f2 Mon Sep 17 00:00:00 2001 From: Giteabot Date: Wed, 12 Jun 2024 22:25:46 +0800 Subject: [PATCH] Reduce memory usage for chunked artifact uploads to MinIO (#31325) (#31338) Backport #31325 by @bohde When using the MinIO storage driver for Actions Artifacts, we found that the chunked artifact required significantly more memory usage to both upload and merge than the local storage driver. This seems to be related to hardcoding a value of `-1` for the size to the MinIO client [which has a warning about memory usage in the respective docs](https://pkg.go.dev/github.com/minio/minio-go/v7#Client.PutObject). Specifying the size in both the upload and merge case reduces memory usage of the MinIO client. Co-authored-by: Rowan Bohde Co-authored-by: Kyle D --- routers/api/actions/artifacts_chunks.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/api/actions/artifacts_chunks.go b/routers/api/actions/artifacts_chunks.go index 3a81724b3a..b0c96585cb 100644 --- a/routers/api/actions/artifacts_chunks.go +++ b/routers/api/actions/artifacts_chunks.go @@ -39,7 +39,7 @@ func saveUploadChunkBase(st storage.ObjectStorage, ctx *ArtifactContext, r = io.TeeReader(r, hasher) } // save chunk to storage - writtenSize, err := st.Save(storagePath, r, -1) + writtenSize, err := st.Save(storagePath, r, contentSize) if err != nil { return -1, fmt.Errorf("save chunk to storage error: %v", err) } @@ -208,7 +208,7 @@ func mergeChunksForArtifact(ctx *ArtifactContext, chunks []*chunkFileItem, st st // save merged file storagePath := fmt.Sprintf("%d/%d/%d.%s", artifact.RunID%255, artifact.ID%255, time.Now().UnixNano(), extension) - written, err := st.Save(storagePath, mergedReader, -1) + written, err := st.Save(storagePath, mergedReader, artifact.FileCompressedSize) if err != nil { return fmt.Errorf("save merged file error: %v", err) }