From bb96349d0958d6842a09543ca0e01a16ca2106a3 Mon Sep 17 00:00:00 2001 From: Antoine GIRARD Date: Sat, 7 Nov 2020 22:52:16 +0100 Subject: [PATCH 01/55] rebase on master --- models/migrations/migrations.go | 2 ++ models/migrations/v160.go | 16 ++++++++++++++++ models/repo.go | 22 +++++++++++++++++++--- modules/auth/repo_form.go | 2 ++ modules/git/repo.go | 7 ++++++- modules/repository/create.go | 1 + modules/structs/repo.go | 4 ++++ options/locale/locale_en-US.ini | 3 +++ routers/api/v1/repo/repo.go | 4 ++++ routers/private/hook.go | 21 +++++++++++++++++++++ routers/repo/repo.go | 1 + routers/repo/setting.go | 15 +++++++++++++++ templates/repo/settings/options.tmpl | 6 +++++- 13 files changed, 99 insertions(+), 5 deletions(-) create mode 100644 models/migrations/v160.go diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index 5a2646474c..bf340ce4de 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -254,6 +254,8 @@ var migrations = []Migration{ NewMigration("code comment replies should have the commitID of the review they are replying to", updateCodeCommentReplies), // v159 -> v160 NewMigration("update reactions constraint", updateReactionConstraint), + // v160 -> v161 + NewMigration("add size limit on repository", addSizeLimitOnRepo), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v160.go b/models/migrations/v160.go new file mode 100644 index 0000000000..82cc528bb9 --- /dev/null +++ b/models/migrations/v160.go @@ -0,0 +1,16 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import "xorm.io/xorm" + +func addSizeLimitOnRepo(x *xorm.Engine) error { + type Repository struct { + ID int64 `xorm:"pk autoincr"` + SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"` + } + + return x.Sync2(new(Repository)) +} diff --git a/models/repo.go b/models/repo.go index 2b53ac666c..55cbcec70e 100644 --- a/models/repo.go +++ b/models/repo.go @@ -230,6 +230,7 @@ type Repository struct { TemplateID int64 `xorm:"INDEX"` TemplateRepo *Repository `xorm:"-"` Size int64 `xorm:"NOT NULL DEFAULT 0"` + SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"` CodeIndexerStatus *RepoIndexerStatus `xorm:"-"` StatsIndexerStatus *RepoIndexerStatus `xorm:"-"` IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true"` @@ -860,20 +861,29 @@ func (repo *Repository) IsOwnedBy(userID int64) bool { return repo.OwnerID == userID } -func (repo *Repository) updateSize(e Engine) error { +func (repo *Repository) computeSize() (int64, error) { size, err := util.GetDirectorySize(repo.RepoPath()) if err != nil { - return fmt.Errorf("updateSize: %v", err) + return 0, fmt.Errorf("computeSize: %v", err) } objs, err := repo.GetLFSMetaObjects(-1, 0) if err != nil { - return fmt.Errorf("updateSize: GetLFSMetaObjects: %v", err) + return 0, fmt.Errorf("computeSize: GetLFSMetaObjects: %v", err) } for _, obj := range objs { size += obj.Size } + return size, nil +} + +func (repo *Repository) updateSize(e Engine) error { + size, err := repo.computeSize() + if err != nil { + return fmt.Errorf("updateSize: %v", err) + } + repo.Size = size _, err = e.ID(repo.ID).Cols("size").Update(repo) return err @@ -884,6 +894,11 @@ func (repo *Repository) UpdateSize(ctx DBContext) error { return repo.updateSize(ctx.e) } +// RepoSizeIsOversized return if is over size limitation +func (repo *Repository) RepoSizeIsOversized(additionalSize int64) bool { + return repo.SizeLimit > 0 && repo.Size+additionalSize > repo.SizeLimit +} + // CanUserFork returns true if specified user can fork repository. func (repo *Repository) CanUserFork(user *User) (bool, error) { if user == nil { @@ -1101,6 +1116,7 @@ type CreateRepoOptions struct { AutoInit bool Status RepositoryStatus TrustModel TrustModelType + SizeLimit int64 } // GetRepoInitFile returns repository init files diff --git a/modules/auth/repo_form.go b/modules/auth/repo_form.go index f27812bb1b..1a9e0e1b81 100644 --- a/modules/auth/repo_form.go +++ b/modules/auth/repo_form.go @@ -48,6 +48,7 @@ type CreateRepoForm struct { Avatar bool Labels bool TrustModel string + SizeLimit int64 } // Validate validates the fields @@ -126,6 +127,7 @@ type RepoSettingForm struct { Private bool Template bool EnablePrune bool + RepoSizeLimit int64 // Advanced settings EnableWiki bool diff --git a/modules/git/repo.go b/modules/git/repo.go index ae370d3da9..3e7868ce90 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -352,8 +352,13 @@ const ( // CountObjects returns the results of git count-objects on the repoPath func CountObjects(repoPath string) (*CountObject, error) { + return CountObjectsWithEnv(repoPath, nil) +} + +// CountObjectsWithEnv returns the results of git count-objects on the repoPath with custom env setup +func CountObjectsWithEnv(repoPath string, env []string) (*CountObject, error) { cmd := NewCommand("count-objects", "-v") - stdout, err := cmd.RunInDir(repoPath) + stdout, err := cmd.RunInDirWithEnv(repoPath, env) if err != nil { return nil, err } diff --git a/modules/repository/create.go b/modules/repository/create.go index 1408637815..8fc73439be 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -45,6 +45,7 @@ func CreateRepository(doer, u *models.User, opts models.CreateRepoOptions) (*mod Status: opts.Status, IsEmpty: !opts.AutoInit, TrustModel: opts.TrustModel, + SizeLimit: opts.SizeLimit, } if err := models.WithTx(func(ctx models.DBContext) error { diff --git a/modules/structs/repo.go b/modules/structs/repo.go index c12f8e1c18..615c1d65e9 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -122,6 +122,8 @@ type CreateRepoOption struct { // TrustModel of the repository // enum: default,collaborator,committer,collaboratorcommitter TrustModel string `json:"trust_model"` + // SizeLimit of the repository + SizeLimit int64 `json:"size_limit"` } // EditRepoOption options when editing a repository's properties @@ -168,6 +170,8 @@ type EditRepoOption struct { AllowSquash *bool `json:"allow_squash_merge,omitempty"` // set to `true` to archive this repository. Archived *bool `json:"archived,omitempty"` + // SizeLimit of the repository. + SizeLimit *int64 `json:"size_limit,omitempty"` } // CreateBranchRepoOption options when creating a branch in a repository diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index f4cdcac427..79d5619619 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -653,6 +653,7 @@ owner = Owner repo_name = Repository Name repo_name_helper = Good repository names use short, memorable and unique keywords. repo_size = Repository Size +repo_size_limit = Repository Size Limit template = Template template_select = Select a template. template_helper = Make repository a template @@ -733,6 +734,8 @@ archive.pull.nocomment = This repo is archived. You cannot comment on pull reque form.reach_limit_of_creation = You have already reached your limit of %d repositories. form.name_reserved = The repository name '%s' is reserved. form.name_pattern_not_allowed = The pattern '%s' is not allowed in a repository name. +form.repo_size_limit_negative = Repository size limitation cannot be negative. +form.repo_size_limit_only_by_admins = Only administrators can change the repository size limitation. need_auth = Clone Authorization migrate_options = Migration Options diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index 116e413125..6f110bbb55 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -246,6 +246,7 @@ func CreateUserRepo(ctx *context.APIContext, owner *models.User, opt api.CreateR DefaultBranch: opt.DefaultBranch, TrustModel: models.ToTrustModel(opt.TrustModel), IsTemplate: opt.Template, + SizeLimit: opt.SizeLimit, }) if err != nil { if models.IsErrRepoAlreadyExist(err) { @@ -568,6 +569,9 @@ func updateBasicProperties(ctx *context.APIContext, opts api.EditRepoOption) err repo.DefaultBranch = *opts.DefaultBranch } + if opts.SizeLimit != nil { + repo.SizeLimit = *opts.SizeLimit + } if err := models.UpdateRepository(repo, visibilityChanged); err != nil { ctx.Error(http.StatusInternalServerError, "UpdateRepository", err) return err diff --git a/routers/private/hook.go b/routers/private/hook.go index dac3940756..313ab7a5bb 100644 --- a/routers/private/hook.go +++ b/routers/private/hook.go @@ -155,12 +155,33 @@ func HookPreReceive(ctx *macaron.Context, opts private.HookOptions) { private.GitQuarantinePath+"="+opts.GitQuarantinePath) } + pushSize, err := git.CountObjectsWithEnv(repo.RepoPath(), env) + if err != nil { + log.Error("Unable to get repository size with env %v: %s Error: %v", repo.RepoPath(), env, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": err.Error(), + }) + return + } + log.Trace("Push size %d", pushSize.Size) + // Iterate across the provided old commit IDs for i := range opts.OldCommitIDs { oldCommitID := opts.OldCommitIDs[i] newCommitID := opts.NewCommitIDs[i] refFullName := opts.RefFullNames[i] + //Check size + if newCommitID != git.EmptySHA && repo.RepoSizeIsOversized(pushSize.Size) { //Check next size if we are not deleting a reference + log.Warn("Forbidden: new repo size is over limitation: %d", repo.SizeLimit) + ctx.JSON(http.StatusForbidden, map[string]interface{}{ + "err": fmt.Sprintf("new repo size is over limitation: %d", repo.SizeLimit), + }) + } + //TODO investigate why on force push some git objects are not cleaned on server side. + //TODO corner-case force push and branch creation -> git.EmptySHA == oldCommitID + //TODO calculate pushed LFS objects size + branchName := strings.TrimPrefix(refFullName, git.BranchPrefix) if branchName == repo.DefaultBranch && newCommitID == git.EmptySHA { log.Warn("Forbidden: Branch: %s is the default branch in %-v and cannot be deleted", branchName, repo) diff --git a/routers/repo/repo.go b/routers/repo/repo.go index 2614389aaa..7df42e3030 100644 --- a/routers/repo/repo.go +++ b/routers/repo/repo.go @@ -250,6 +250,7 @@ func CreatePost(ctx *context.Context, form auth.CreateRepoForm) { AutoInit: form.AutoInit, IsTemplate: form.Template, TrustModel: models.ToTrustModel(form.TrustModel), + SizeLimit: form.SizeLimit, }) if err == nil { log.Trace("Repository created [%d]: %s/%s", repo.ID, ctxUser.Name, repo.Name) diff --git a/routers/repo/setting.go b/routers/repo/setting.go index 368879234b..90f357355c 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -50,6 +50,7 @@ func Settings(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("repo.settings") ctx.Data["PageIsSettingsOptions"] = true ctx.Data["ForcePrivate"] = setting.Repository.ForcePrivate + ctx.Data["Err_RepoSize"] = ctx.Repo.Repository.RepoSizeIsOversized(ctx.Repo.Repository.SizeLimit / 10) // less than 10% left signing, _ := models.SigningKey(ctx.Repo.Repository.RepoPath()) ctx.Data["SigningKeyAvailable"] = len(signing) > 0 @@ -64,6 +65,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { ctx.Data["PageIsSettingsOptions"] = true repo := ctx.Repo.Repository + ctx.Data["Err_RepoSize"] = repo.RepoSizeIsOversized(repo.SizeLimit / 10) // less than 10% left switch ctx.Query("action") { case "update": @@ -128,6 +130,19 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) { return } + if form.RepoSizeLimit < 0 { + ctx.Data["Err_RepoSizeLimit"] = true + ctx.RenderWithErr(ctx.Tr("repo.form.repo_size_limit_negative"), tplSettingsOptions, &form) + return + } + + if !ctx.User.IsAdmin && repo.SizeLimit != form.RepoSizeLimit { + ctx.Data["Err_RepoSizeLimit"] = true + ctx.RenderWithErr(ctx.Tr("repo.form.repo_size_limit_only_by_admins"), tplSettingsOptions, &form) + return + } + repo.SizeLimit = form.RepoSizeLimit + repo.IsPrivate = form.Private if err := models.UpdateRepository(repo, visibilityChanged); err != nil { ctx.ServerError("UpdateRepository", err) diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 96414731bc..60e1c32237 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -17,7 +17,11 @@
- {{SizeFmt .Repository.Size}} + {{SizeFmt .Repository.Size}}{{if .Repository.SizeLimit}}/{{SizeFmt .Repository.SizeLimit}}{{end}} +
+
+ +
From 56b92a17e0d7597777fc946078d364509afde920 Mon Sep 17 00:00:00 2001 From: Antoine GIRARD Date: Sat, 7 Nov 2020 23:03:44 +0100 Subject: [PATCH 02/55] generate swagger --- templates/swagger/v1_json.tmpl | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/templates/swagger/v1_json.tmpl b/templates/swagger/v1_json.tmpl index e759a1558c..3e2967f02e 100644 --- a/templates/swagger/v1_json.tmpl +++ b/templates/swagger/v1_json.tmpl @@ -12271,6 +12271,12 @@ "type": "string", "x-go-name": "Readme" }, + "size_limit": { + "description": "SizeLimit of the repository", + "type": "integer", + "format": "int64", + "x-go-name": "SizeLimit" + }, "template": { "description": "Whether the repository is template", "type": "boolean", @@ -13029,6 +13035,12 @@ "type": "boolean", "x-go-name": "Private" }, + "size_limit": { + "description": "SizeLimit of the repository.", + "type": "integer", + "format": "int64", + "x-go-name": "SizeLimit" + }, "template": { "description": "either `true` to make this repository a template or `false` to make it a normal repository", "type": "boolean", From 2f655456d63d503a5632083ceadd1511c47bb051 Mon Sep 17 00:00:00 2001 From: Antoine GIRARD Date: Sat, 7 Nov 2020 23:18:52 +0100 Subject: [PATCH 03/55] add some tests --- .../api_helper_for_declarative_test.go | 14 ++++++++ integrations/git_test.go | 36 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/integrations/api_helper_for_declarative_test.go b/integrations/api_helper_for_declarative_test.go index b8e513958e..977d3b46f8 100644 --- a/integrations/api_helper_for_declarative_test.go +++ b/integrations/api_helper_for_declarative_test.go @@ -371,3 +371,17 @@ func doAPIAddRepoToOrganizationTeam(ctx APITestContext, teamID int64, orgName, r ctx.Session.MakeRequest(t, req, http.StatusNoContent) } } + +func doAPISetRepoSizeLimit(ctx APITestContext, owner, repo string, size int64) func(*testing.T) { + return func(t *testing.T) { + urlStr := fmt.Sprintf("/api/v1/repos/%s/%s?token=%s", + owner, repo, ctx.Token) + req := NewRequestWithJSON(t, http.MethodPatch, urlStr, &api.EditRepoOption{SizeLimit: &size}) + + if ctx.ExpectedCode != 0 { + ctx.Session.MakeRequest(t, req, ctx.ExpectedCode) + return + } + ctx.Session.MakeRequest(t, req, 200) + } +} diff --git a/integrations/git_test.go b/integrations/git_test.go index c3c1126829..8fc9eadf9d 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -68,6 +68,34 @@ func testGit(t *testing.T, u *url.URL) { rawTest(t, &httpContext, little, big, littleLFS, bigLFS) mediaTest(t, &httpContext, little, big, littleLFS, bigLFS) + t.Run("SizeLimit", func(t *testing.T) { + t.Run("Under", func(t *testing.T) { + PrintCurrentTest(t) + doCommitAndPush(t, littleSize, dstPath, "data-file-") + }) + t.Run("Over", func(t *testing.T) { + PrintCurrentTest(t) + doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, littleSize) + //TODO fix this doCommitAndPushWithExpectedError(t, bigSize, dstPath, "data-file-") + }) + t.Run("UnderAfterResize", func(t *testing.T) { + PrintCurrentTest(t) + doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, bigSize*10) + doCommitAndPush(t, littleSize, dstPath, "data-file-") + }) + t.Run("Deletion", func(t *testing.T) { + PrintCurrentTest(t) + //TODO delete a file + //doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") + }) + //TODO delete branch + //TODO delete tag + //TODO add big commit that will be over with the push + //TODO add lfs + //TODO remove lfs + //TODO add missing case + }) + t.Run("BranchProtectMerge", doBranchProtectPRMerge(&httpContext, dstPath)) t.Run("MergeFork", func(t *testing.T) { defer PrintCurrentTest(t)() @@ -291,6 +319,14 @@ func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string { return name } +func doCommitAndPushWithExpectedError(t *testing.T, size int, repoPath, prefix string) string { + name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix) + assert.NoError(t, err) + _, err = git.NewCommand("push", "origin", "master").RunInDir(repoPath) //Push + assert.Error(t, err) + return name +} + func generateCommitWithNewData(size int, repoPath, email, fullName, prefix string) (string, error) { //Generate random file bufSize := 4 * 1024 From 7c31cfcbe3fb92539a2f4697e0bbc3ffb0be1681 Mon Sep 17 00:00:00 2001 From: Antoine GIRARD Date: Sat, 7 Nov 2020 23:30:54 +0100 Subject: [PATCH 04/55] add some tests --- integrations/git_test.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integrations/git_test.go b/integrations/git_test.go index 8fc9eadf9d..25cdc70096 100644 --- a/integrations/git_test.go +++ b/integrations/git_test.go @@ -76,7 +76,7 @@ func testGit(t *testing.T, u *url.URL) { t.Run("Over", func(t *testing.T) { PrintCurrentTest(t) doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, littleSize) - //TODO fix this doCommitAndPushWithExpectedError(t, bigSize, dstPath, "data-file-") + doCommitAndPushWithExpectedError(t, bigSize, dstPath, "data-file-") }) t.Run("UnderAfterResize", func(t *testing.T) { PrintCurrentTest(t) @@ -85,8 +85,7 @@ func testGit(t *testing.T, u *url.URL) { }) t.Run("Deletion", func(t *testing.T) { PrintCurrentTest(t) - //TODO delete a file - //doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") + //TODO doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") }) //TODO delete branch //TODO delete tag From df60de22bc5376c3dab6f7439b1abb001876e04e Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Sat, 12 Nov 2022 23:47:06 +0000 Subject: [PATCH 05/55] Fixing Merge errors - still few to go --- models/repo/repo.go | 2 +- modules/git/repo.go | 73 ++++++---------------------- routers/api/v1/repo/repo.go | 2 +- routers/private/hook_pre_receive.go | 6 ++- routers/web/repo/setting.go | 2 +- templates/repo/settings/options.tmpl | 4 +- 6 files changed, 25 insertions(+), 64 deletions(-) diff --git a/models/repo/repo.go b/models/repo/repo.go index 99289f4f1e..bcf315c674 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -522,7 +522,7 @@ func (repo *Repository) computeSize() (int64, error) { return 0, fmt.Errorf("computeSize: %v", err) } - objs, err := repo.GetLFSMetaObjects(-1, 0) + objs, err := repo.GetLFSMetaObjects(repo.ID, -1, 0) if err != nil { return 0, fmt.Errorf("computeSize: GetLFSMetaObjects: %v", err) } diff --git a/modules/git/repo.go b/modules/git/repo.go index c5be571a05..c6e080da76 100644 --- a/modules/git/repo.go +++ b/modules/git/repo.go @@ -257,50 +257,6 @@ func Push(ctx context.Context, repoPath string, opts PushOptions) error { return err } -// CheckoutOptions options when heck out some branch -type CheckoutOptions struct { - Timeout time.Duration - Branch string - OldBranch string -} - -// Checkout checkouts a branch -func Checkout(repoPath string, opts CheckoutOptions) error { - cmd := NewCommand("checkout") - if len(opts.OldBranch) > 0 { - cmd.AddArguments("-b") - } - - if opts.Timeout <= 0 { - opts.Timeout = -1 - } - - cmd.AddArguments(opts.Branch) - - if len(opts.OldBranch) > 0 { - cmd.AddArguments(opts.OldBranch) - } - - _, err := cmd.RunInDirTimeout(opts.Timeout, repoPath) - return err -} - -// ResetHEAD resets HEAD to given revision or head of branch. -func ResetHEAD(repoPath string, hard bool, revision string) error { - cmd := NewCommand("reset") - if hard { - cmd.AddArguments("--hard") - } - _, err := cmd.AddArguments(revision).RunInDir(repoPath) - return err -} - -// MoveFile moves a file to another file or directory. -func MoveFile(repoPath, oldTreeName, newTreeName string) error { - _, err := NewCommand("mv").AddArguments(oldTreeName, newTreeName).RunInDir(repoPath) - return err -} - // CountObject represents repository count objects report type CountObject struct { Count int64 @@ -325,14 +281,14 @@ const ( ) // CountObjects returns the results of git count-objects on the repoPath -func CountObjects(repoPath string) (*CountObject, error) { - return CountObjectsWithEnv(repoPath, nil) +func CountObjects(ctx context.Context, repoPath string) (*CountObject, error) { + return CountObjectsWithEnv(ctx, repoPath, nil) } // CountObjectsWithEnv returns the results of git count-objects on the repoPath with custom env setup -func CountObjectsWithEnv(repoPath string, env []string) (*CountObject, error) { - cmd := NewCommand("count-objects", "-v") - stdout, err := cmd.RunInDirWithEnv(repoPath, env) +func CountObjectsWithEnv(ctx context.Context, repoPath string, env []string) (*CountObject, error) { + cmd := NewCommand(ctx, "count-objects", "-v") + stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath, Env: env}) if err != nil { return nil, err } @@ -346,21 +302,24 @@ func parseSize(objects string) *CountObject { for _, line := range strings.Split(objects, "\n") { switch { case strings.HasPrefix(line, statCount): - repoSize.Count = com.StrTo(line[7:]).MustInt64() + repoSize.Count, _ = strconv.ParseInt(line[7:], 10, 64) case strings.HasPrefix(line, statSize): - repoSize.Size = com.StrTo(line[6:]).MustInt64() * 1024 + number, _ := strconv.ParseInt(line[6:], 10, 64) + repoSize.Size = number * 1024 case strings.HasPrefix(line, statInpack): - repoSize.InPack = com.StrTo(line[9:]).MustInt64() + repoSize.InPack, _ = strconv.ParseInt(line[9:], 10, 64) case strings.HasPrefix(line, statPacks): - repoSize.Packs = com.StrTo(line[7:]).MustInt64() + repoSize.Packs, _ = strconv.ParseInt(line[7:], 10, 64) case strings.HasPrefix(line, statSizePack): - repoSize.SizePack = com.StrTo(line[11:]).MustInt64() * 1024 + number, _ := strconv.ParseInt(line[11:], 10, 64) + repoSize.SizePack = number * 1024 case strings.HasPrefix(line, statPrunePackage): - repoSize.PrunePack = com.StrTo(line[16:]).MustInt64() + repoSize.PrunePack, _ = strconv.ParseInt(line[16:], 10, 64) case strings.HasPrefix(line, statGarbage): - repoSize.Garbage = com.StrTo(line[9:]).MustInt64() + repoSize.Garbage, _ = strconv.ParseInt(line[9:], 10, 64) case strings.HasPrefix(line, statSizeGarbage): - repoSize.SizeGarbage = com.StrTo(line[14:]).MustInt64() * 1024 + number, _ := strconv.ParseInt(line[14:], 10, 64) + repoSize.SizeGarbage = number * 1024 } } return repoSize diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index 8f47f06864..66616d7612 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -719,7 +719,7 @@ func updateBasicProperties(ctx *context.APIContext, opts api.EditRepoOption) err if opts.SizeLimit != nil { repo.SizeLimit = *opts.SizeLimit } - if err := models.UpdateRepository(repo, visibilityChanged); err != nil { + if err := repo_service.UpdateRepository(repo, visibilityChanged); err != nil { ctx.Error(http.StatusInternalServerError, "UpdateRepository", err) return err diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index 821ea7b9fb..1def817208 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -114,9 +114,11 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { opts: opts, } - pushSize, err := git.CountObjectsWithEnv(repo.RepoPath(), env) + repo := ourCtx.Repo.Repository + + pushSize, err := git.CountObjectsWithEnv(ctx, repo.RepoPath(), ourCtx.env) if err != nil { - log.Error("Unable to get repository size with env %v: %s Error: %v", repo.RepoPath(), env, err) + log.Error("Unable to get repository size with env %v: %s Error: %v", repo.RepoPath(), ourCtx.env, err) ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ "err": err.Error(), }) diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 14822a15f1..c794da3b70 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -192,7 +192,7 @@ func SettingsPost(ctx *context.Context) { return } - if !ctx.User.IsAdmin && repo.SizeLimit != form.RepoSizeLimit { + if !ctx.Doer.IsAdmin && repo.SizeLimit != form.RepoSizeLimit { ctx.Data["Err_RepoSizeLimit"] = true ctx.RenderWithErr(ctx.Tr("repo.form.repo_size_limit_only_by_admins"), tplSettingsOptions, &form) return diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 8db0334981..eb86d95670 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -18,10 +18,10 @@
- {{SizeFmt .Repository.Size}}{{if .Repository.SizeLimit}}/{{SizeFmt .Repository.SizeLimit}}{{end}} + {{FileSize .Repository.Size}}{{if .Repository.SizeLimit}}/{{FileSize .Repository.SizeLimit}}{{end}}
- + {{FileSize .Repository.Size}} From 136b6318ed71334df1d17a516b99ed1f8047430a Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Mon, 14 Nov 2022 15:17:00 +0000 Subject: [PATCH 06/55] Removed code frmo models/repo/repo.go to stop compile errors - TODO for later see what is still needed --- models/repo/repo.go | 55 +++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/models/repo/repo.go b/models/repo/repo.go index bcf315c674..49009af567 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -516,38 +516,39 @@ func (repo *Repository) IsOwnedBy(userID int64) bool { return repo.OwnerID == userID } -func (repo *Repository) computeSize() (int64, error) { - size, err := util.GetDirectorySize(repo.RepoPath()) - if err != nil { - return 0, fmt.Errorf("computeSize: %v", err) - } +// TODO - DmitryFrolovTri Review every commented function and see if we need to reinstate it +// func (repo *Repository) computeSize() (int64, error) { +// size, err := util.GetDirectorySize(repo.RepoPath()) +// if err != nil { +// return 0, fmt.Errorf("computeSize: %v", err) +// } - objs, err := repo.GetLFSMetaObjects(repo.ID, -1, 0) - if err != nil { - return 0, fmt.Errorf("computeSize: GetLFSMetaObjects: %v", err) - } - for _, obj := range objs { - size += obj.Size - } +// objs, err := repo.GetLFSMetaObjects(repo.ID, -1, 0) +// if err != nil { +// return 0, fmt.Errorf("computeSize: GetLFSMetaObjects: %v", err) +// } +// for _, obj := range objs { +// size += obj.Size +// } - return size, nil -} +// return size, nil +// } -func (repo *Repository) updateSize(e Engine) error { - size, err := repo.computeSize() - if err != nil { - return fmt.Errorf("updateSize: %v", err) - } +// func (repo *Repository) updateSize(ctx context.Context Engine) error { +// size, err := repo.computeSize() +// if err != nil { +// return fmt.Errorf("updateSize: %v", err) +// } - repo.Size = size - _, err = e.ID(repo.ID).Cols("size").Update(repo) - return err -} +// repo.Size = size +// _, err = e.ID(repo.ID).Cols("size").Update(repo) +// return err +// } -// UpdateSize updates the repository size, calculating it using util.GetDirectorySize -func (repo *Repository) UpdateSize(ctx DBContext) error { - return repo.updateSize(ctx.e) -} +// // UpdateSize updates the repository size, calculating it using util.GetDirectorySize +// func (repo *Repository) UpdateSize(ctx DBContext) error { +// return repo.updateSize(ctx.e) +// } // RepoSizeIsOversized return if is over size limitation func (repo *Repository) RepoSizeIsOversized(additionalSize int64) bool { From 75585dec3c204eb79a23b9025fb6c63fe631c57b Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Tue, 15 Nov 2022 15:32:58 +0000 Subject: [PATCH 07/55] lint related changes --- routers/private/hook_pre_receive.go | 10 +++++----- tests/integration/git_test.go | 24 ++++++++++++------------ 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index 1def817208..eb18da8a4d 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -132,16 +132,16 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { newCommitID := opts.NewCommitIDs[i] refFullName := opts.RefFullNames[i] - //Check size - if newCommitID != git.EmptySHA && repo.RepoSizeIsOversized(pushSize.Size) { //Check next size if we are not deleting a reference + // Check size + if newCommitID != git.EmptySHA && repo.RepoSizeIsOversized(pushSize.Size) { // Check next size if we are not deleting a reference log.Warn("Forbidden: new repo size is over limitation: %d", repo.SizeLimit) ctx.JSON(http.StatusForbidden, map[string]interface{}{ "err": fmt.Sprintf("new repo size is over limitation: %d", repo.SizeLimit), }) } - //TODO investigate why on force push some git objects are not cleaned on server side. - //TODO corner-case force push and branch creation -> git.EmptySHA == oldCommitID - //TODO calculate pushed LFS objects size + // TODO investigate why on force push some git objects are not cleaned on server side. + // TODO corner-case force push and branch creation -> git.EmptySHA == oldCommitID + // TODO calculate pushed LFS objects size switch { case strings.HasPrefix(refFullName, git.BranchPrefix): diff --git a/tests/integration/git_test.go b/tests/integration/git_test.go index 33c74b2aed..0314f2913a 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_test.go @@ -79,29 +79,29 @@ func testGit(t *testing.T, u *url.URL) { t.Run("SizeLimit", func(t *testing.T) { t.Run("Under", func(t *testing.T) { - PrintCurrentTest(t) + tests.PrintCurrentTest(t) doCommitAndPush(t, littleSize, dstPath, "data-file-") }) t.Run("Over", func(t *testing.T) { - PrintCurrentTest(t) + tests.PrintCurrentTest(t) doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, littleSize) doCommitAndPushWithExpectedError(t, bigSize, dstPath, "data-file-") }) t.Run("UnderAfterResize", func(t *testing.T) { - PrintCurrentTest(t) + tests.PrintCurrentTest(t) doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, bigSize*10) doCommitAndPush(t, littleSize, dstPath, "data-file-") }) t.Run("Deletion", func(t *testing.T) { - PrintCurrentTest(t) - //TODO doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") + tests.PrintCurrentTest(t) + // TODO doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") }) - //TODO delete branch - //TODO delete tag - //TODO add big commit that will be over with the push - //TODO add lfs - //TODO remove lfs - //TODO add missing case + // TODO delete branch + // TODO delete tag + // TODO add big commit that will be over with the push + // TODO add lfs + // TODO remove lfs + // TODO add missing case }) t.Run("CreateAgitFlowPull", doCreateAgitFlowPull(dstPath, &httpContext, "master", "test/head")) t.Run("BranchProtectMerge", doBranchProtectPRMerge(&httpContext, dstPath)) @@ -324,7 +324,7 @@ func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string { func doCommitAndPushWithExpectedError(t *testing.T, size int, repoPath, prefix string) string { name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix) assert.NoError(t, err) - _, err = git.NewCommand("push", "origin", "master").RunInDir(repoPath) //Push + _, _, err = git.NewCommand(git.DefaultContext, "push", "origin", "master").RunStdString(&git.RunOpts{Dir: repoPath}) // Push assert.Error(t, err) return name } From d9ab187078e28e9b129424edffbc288359d949ec Mon Sep 17 00:00:00 2001 From: techknowlogick Date: Sun, 4 Dec 2022 00:19:29 -0500 Subject: [PATCH 08/55] update license header --- models/migrations/v160.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/models/migrations/v160.go b/models/migrations/v160.go index 82cc528bb9..5897cb2714 100644 --- a/models/migrations/v160.go +++ b/models/migrations/v160.go @@ -1,6 +1,5 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. -// Use of this source code is governed by a MIT-style -// license that can be found in the LICENSE file. +// Copyright 2022 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT package migrations From 31359b4721470eace23c371761b9831c9a0a7e7b Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Sun, 4 Dec 2022 06:53:08 +0000 Subject: [PATCH 09/55] moved AddSizeLimitOnRepo migration to v1_19 package --- models/migrations/migrations.go | 2 +- models/migrations/{v160.go => v1_19/v236.go} | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) rename models/migrations/{v160.go => v1_19/v236.go} (71%) diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index c9ebde3dce..f9b77fae42 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -443,7 +443,7 @@ var migrations = []Migration{ // v235 -> v236 NewMigration("Add index for access_token", v1_19.AddIndexForAccessToken), // to modify later - NewMigration("add size limit on repository", addSizeLimitOnRepo), + NewMigration("add size limit on repository", v1_19.AddSizeLimitOnRepo), } // GetCurrentDBVersion returns the current db version diff --git a/models/migrations/v160.go b/models/migrations/v1_19/v236.go similarity index 71% rename from models/migrations/v160.go rename to models/migrations/v1_19/v236.go index 5897cb2714..1045835d0f 100644 --- a/models/migrations/v160.go +++ b/models/migrations/v1_19/v236.go @@ -1,11 +1,13 @@ // Copyright 2022 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package migrations +package v1_19 // nolint -import "xorm.io/xorm" +import ( + "xorm.io/xorm" +) -func addSizeLimitOnRepo(x *xorm.Engine) error { +func AddSizeLimitOnRepo(x *xorm.Engine) error { type Repository struct { ID int64 `xorm:"pk autoincr"` SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"` From 64f5b94590876b5313ab30af3b76c18980d856c1 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Fri, 20 Jan 2023 11:37:38 +0000 Subject: [PATCH 10/55] Fixing lint error in models/migrations/v1_19 for v999.go --- models/migrations/v1_19/v999.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/migrations/v1_19/v999.go b/models/migrations/v1_19/v999.go index 1045835d0f..d672413854 100644 --- a/models/migrations/v1_19/v999.go +++ b/models/migrations/v1_19/v999.go @@ -1,7 +1,7 @@ // Copyright 2022 The Gitea Authors. All rights reserved. // SPDX-License-Identifier: MIT -package v1_19 // nolint +package v1_19 //nolint import ( "xorm.io/xorm" From ae06851102fce9ed675a608e1f02c982aaadd195 Mon Sep 17 00:00:00 2001 From: truecode112 Date: Mon, 6 Feb 2023 08:57:52 +0300 Subject: [PATCH 11/55] Fixed Test(SizeLimit/Over) --- tests/integration/git_test.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/integration/git_test.go b/tests/integration/git_test.go index 03bb2954ad..9f479d81ce 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_test.go @@ -29,6 +29,7 @@ import ( api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/tests" + "code.gitea.io/gitea/modules/log" "github.com/stretchr/testify/assert" ) @@ -58,6 +59,8 @@ func testGit(t *testing.T, u *url.URL) { dstPath := t.TempDir() + dstForkedPath := t.TempDir() + t.Run("CreateRepoInDifferentUser", doAPICreateRepository(forkedUserCtx, false)) t.Run("AddUserAsCollaborator", doAPIAddCollaborator(forkedUserCtx, httpContext.Username, perm.AccessModeRead)) @@ -84,12 +87,15 @@ func testGit(t *testing.T, u *url.URL) { }) t.Run("Over", func(t *testing.T) { tests.PrintCurrentTest(t) - doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, littleSize) - doCommitAndPushWithExpectedError(t, bigSize, dstPath, "data-file-") + u.Path = forkedUserCtx.GitPath() + u.User = url.UserPassword(forkedUserCtx.Username, userPassword) + t.Run("Clone", doGitClone(dstForkedPath, u)) + t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, littleSize)) + doCommitAndPushWithExpectedError(t, bigSize, dstForkedPath, "data-file-") }) t.Run("UnderAfterResize", func(t *testing.T) { tests.PrintCurrentTest(t) - doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, bigSize*10) + t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, bigSize*10)) doCommitAndPush(t, littleSize, dstPath, "data-file-") }) t.Run("Deletion", func(t *testing.T) { @@ -114,6 +120,8 @@ func testGit(t *testing.T, u *url.URL) { mediaTest(t, &forkedUserCtx, little, big, littleLFS, bigLFS) }) + u.Path = httpContext.GitPath() + u.User = url.UserPassword(username, userPassword) t.Run("PushCreate", doPushCreate(httpContext, u)) }) t.Run("SSH", func(t *testing.T) { @@ -212,7 +220,9 @@ func commitAndPushTest(t *testing.T, dstPath, prefix string) (little, big string defer tests.PrintCurrentTest(t)() t.Run("Little", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + log.Error("before doCommitAndPush") little = doCommitAndPush(t, littleSize, dstPath, prefix) + log.Error("after doCommitAndPush") }) t.Run("Big", func(t *testing.T) { if testing.Short() { From 8494f19f29602db3ce8c64b03ee2fa9d49eb49b5 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Mon, 6 Feb 2023 07:09:35 +0000 Subject: [PATCH 12/55] fix fmt errors --- tests/integration/git_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/git_test.go b/tests/integration/git_test.go index 692a1fda29..e75f3a2500 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_test.go @@ -25,11 +25,11 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/lfs" + "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/tests" - "code.gitea.io/gitea/modules/log" "github.com/stretchr/testify/assert" ) From 0afa267c181dbfd082aa282f025491c350927861 Mon Sep 17 00:00:00 2001 From: truecode112 Date: Mon, 6 Feb 2023 16:48:19 +0300 Subject: [PATCH 13/55] Fixed unclosed logger test --- tests/integration/git_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/git_test.go b/tests/integration/git_test.go index e75f3a2500..9d2cc66eb3 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_test.go @@ -82,11 +82,11 @@ func testGit(t *testing.T, u *url.URL) { t.Run("SizeLimit", func(t *testing.T) { t.Run("Under", func(t *testing.T) { - tests.PrintCurrentTest(t) + defer tests.PrintCurrentTest(t)() doCommitAndPush(t, littleSize, dstPath, "data-file-") }) t.Run("Over", func(t *testing.T) { - tests.PrintCurrentTest(t) + defer tests.PrintCurrentTest(t)() u.Path = forkedUserCtx.GitPath() u.User = url.UserPassword(forkedUserCtx.Username, userPassword) t.Run("Clone", doGitClone(dstForkedPath, u)) @@ -94,12 +94,12 @@ func testGit(t *testing.T, u *url.URL) { doCommitAndPushWithExpectedError(t, bigSize, dstForkedPath, "data-file-") }) t.Run("UnderAfterResize", func(t *testing.T) { - tests.PrintCurrentTest(t) + defer tests.PrintCurrentTest(t)() t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, bigSize*10)) doCommitAndPush(t, littleSize, dstPath, "data-file-") }) t.Run("Deletion", func(t *testing.T) { - tests.PrintCurrentTest(t) + defer tests.PrintCurrentTest(t)() // TODO doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") }) // TODO delete branch From a22b73866a7c2f7464c99c5ecd7d91e10fb930ce Mon Sep 17 00:00:00 2001 From: truecode112 Date: Mon, 15 May 2023 06:24:17 +0300 Subject: [PATCH 14/55] Added ENABLE_SIZE_LIMIT config option --- models/repo/repo.go | 1 + 1 file changed, 1 insertion(+) diff --git a/models/repo/repo.go b/models/repo/repo.go index 361086ea42..12181ef71b 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -164,6 +164,7 @@ type Repository struct { TemplateID int64 `xorm:"INDEX"` Size int64 `xorm:"NOT NULL DEFAULT 0"` SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"` + EnableSizeLimit bool `xorm:"NOT NULL DEFAULT true"` CodeIndexerStatus *RepoIndexerStatus `xorm:"-"` StatsIndexerStatus *RepoIndexerStatus `xorm:"-"` IsFsckEnabled bool `xorm:"NOT NULL DEFAULT true"` From 66c476e5debfbd960b0c78a53323d6d7bce2431f Mon Sep 17 00:00:00 2001 From: truecode112 Date: Mon, 15 May 2023 09:57:47 +0300 Subject: [PATCH 15/55] UI related changed for ENABLE_SIZE_LIMIT & REPO_SIZE_LIMIT options --- modules/base/tool.go | 9 ++++++++- modules/setting/repository.go | 20 +++++++++++++++++++ routers/web/admin/repos.go | 36 ++++++++++++++++++++++++++++++++++ routers/web/explore/repo.go | 5 +++++ routers/web/web.go | 2 ++ services/forms/repo_form.go | 6 ++++++ templates/admin/repo/list.tmpl | 21 ++++++++++++++++++++ 7 files changed, 98 insertions(+), 1 deletion(-) diff --git a/modules/base/tool.go b/modules/base/tool.go index 13b07c043e..96d1391e3a 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -138,7 +138,14 @@ func CreateTimeLimitCode(data string, minutes int, startInf interface{}) string // FileSize calculates the file size and generate user-friendly string. func FileSize(s int64) string { - return humanize.IBytes(uint64(s)) + return humanize.Bytes(uint64(s)) +} + +// Get FileSize bytes value from String. +func GetFileSize(s string) (int64, error) { + v, err := humanize.ParseBytes(s) + iv := int64(v) + return iv, err } // EllipsisString returns a truncated short string, diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 56e7e6f4ac..a495f471d6 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -7,6 +7,7 @@ import ( "os/exec" "path" "path/filepath" + "strconv" "strings" "code.gitea.io/gitea/modules/log" @@ -265,12 +266,31 @@ var ( RepoArchive = struct { Storage }{} + + EnableSizeLimit = true + RepoSizeLimit int64 = 0 ) +func SaveGlobalRepositorySetting(enable_size_limit bool, repo_size_limit int64) { + EnableSizeLimit = enable_size_limit + RepoSizeLimit = repo_size_limit + sec := CfgProvider.Section("repository") + if EnableSizeLimit { + sec.Key("ENABLE_SIZE_LIMIT").SetValue("true") + } else { + sec.Key("ENABLE_SIZE_LIMIT").SetValue("false") + } + + sec.Key("REPO_SIZE_LIMIT").SetValue(strconv.FormatInt(RepoSizeLimit, 10)) +} + func loadRepositoryFrom(rootCfg ConfigProvider) { var err error + // Determine and create root git repository path. sec := rootCfg.Section("repository") + EnableSizeLimit = sec.Key("ENABLE_SIZE_LIMIT").MustBool() + RepoSizeLimit = sec.Key("REPO_SIZE_LIMIT").MustInt64(0) Repository.DisableHTTPGit = sec.Key("DISABLE_HTTP_GIT").MustBool() Repository.UseCompatSSHURI = sec.Key("USE_COMPAT_SSH_URI").MustBool() Repository.MaxCreationLimit = sec.Key("MAX_CREATION_LIMIT").MustInt(-1) diff --git a/routers/web/admin/repos.go b/routers/web/admin/repos.go index 9a0e467b48..2287f81fe1 100644 --- a/routers/web/admin/repos.go +++ b/routers/web/admin/repos.go @@ -17,7 +17,9 @@ import ( repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/web/explore" + "code.gitea.io/gitea/services/forms" repo_service "code.gitea.io/gitea/services/repository" ) @@ -31,6 +33,9 @@ func Repos(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("admin.repositories") ctx.Data["PageIsAdminRepositories"] = true + ctx.Data["EnableSizeLimit"] = setting.EnableSizeLimit + ctx.Data["RepoSizeLimit"] = base.FileSize(setting.RepoSizeLimit) + explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{ Private: true, PageSize: setting.UI.Admin.RepoPagingNum, @@ -39,6 +44,37 @@ func Repos(ctx *context.Context) { }) } +func UpdateRepoPost(ctx *context.Context) { + form := web.GetForm(ctx).(*forms.UpdateGlobalRepoFrom) + ctx.Data["Title"] = ctx.Tr("admin.repositories") + ctx.Data["PageIsAdminRepositories"] = true + + repo_size_limit, err := base.GetFileSize(form.RepoSizeLimit) + + ctx.Data["EnableSizeLimit"] = form.EnableSizeLimit + ctx.Data["RepoSizeLimit"] = form.RepoSizeLimit + + if err != nil { + ctx.Data["Err_Repo_Size_Limit"] = true + explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{ + Private: true, + PageSize: setting.UI.Admin.RepoPagingNum, + TplName: tplRepos, + OnlyShowRelevant: false, + }) + return + } + + setting.SaveGlobalRepositorySetting(form.EnableSizeLimit, repo_size_limit) + ctx.Flash.Success(ctx.Tr("admin.config.repository_setting_success")) + + ctx.Data["RepoSizeLimit"] = base.FileSize(setting.RepoSizeLimit) + + ctx.Flash.Success(ctx.Tr("admin.config.repository_setting_success")) + ctx.Redirect(setting.AppSubURL + "/admin/repos") + +} + // DeleteRepo delete one repository func DeleteRepo(ctx *context.Context) { repo, err := repo_model.GetRepositoryByID(ctx, ctx.FormInt64("id")) diff --git a/routers/web/explore/repo.go b/routers/web/explore/repo.go index be5ad1b015..a3450a142c 100644 --- a/routers/web/explore/repo.go +++ b/routers/web/explore/repo.go @@ -141,6 +141,11 @@ func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) { pager.AddParamString(relevantReposOnlyParam, fmt.Sprint(opts.OnlyShowRelevant)) ctx.Data["Page"] = pager + if ctx.Data["Err_Repo_Size_Limit"] != nil { + ctx.RenderWithErr(ctx.Tr("admin.config.invalid_repo_size"), opts.TplName, nil) + return + } + ctx.HTML(http.StatusOK, opts.TplName) } diff --git a/routers/web/web.go b/routers/web/web.go index 8784b7c5f7..b3b9d65dbf 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -580,6 +580,8 @@ func registerRoutes(m *web.Route) { m.Get("", admin.Repos) m.Combo("/unadopted").Get(admin.UnadoptedRepos).Post(admin.AdoptOrDeleteRepository) m.Post("/delete", admin.DeleteRepo) + m.Post("", web.Bind(forms.UpdateGlobalRepoFrom{}), admin.UpdateRepoPost) + }) m.Group("/packages", func() { diff --git a/services/forms/repo_form.go b/services/forms/repo_form.go index 9e5e5697ff..5e57a7effb 100644 --- a/services/forms/repo_form.go +++ b/services/forms/repo_form.go @@ -28,6 +28,12 @@ import ( // |____|_ /_______ / |____| \_______ /_______ /|___| |____| \_______ /____|_ // ______| // \/ \/ \/ \/ \/ \/ \/ +// UpdateGlobalRepoFrom for updating global repository setting +type UpdateGlobalRepoFrom struct { + RepoSizeLimit string + EnableSizeLimit bool +} + // CreateRepoForm form for creating repository type CreateRepoForm struct { UID int64 `binding:"Required"` diff --git a/templates/admin/repo/list.tmpl b/templates/admin/repo/list.tmpl index f485784d0c..bd29db0905 100644 --- a/templates/admin/repo/list.tmpl +++ b/templates/admin/repo/list.tmpl @@ -1,5 +1,26 @@ {{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin")}}
+

+ {{.locale.Tr "admin.config.repository_config"}} +

+
+
+ {{.CsrfTokenHtml}} +
+ +
+ +
+
+
+ + +
+
+ +
+
+

{{.locale.Tr "admin.repos.repo_manage_panel"}} ({{.locale.Tr "admin.total" .Total}})
From d843a6736bdea7cfdcbdb818893151e368e224e3 Mon Sep 17 00:00:00 2001 From: truecode112 Date: Mon, 15 May 2023 11:22:07 +0300 Subject: [PATCH 16/55] Overall Done for Global Repo Size Limit Option --- models/repo/repo.go | 12 ++++++++++-- modules/setting/repository.go | 4 ++-- options/locale/locale_en-US.ini | 6 ++++++ routers/web/admin/repos.go | 3 --- routers/web/repo/setting.go | 4 +++- templates/repo/settings/options.tmpl | 14 +++++++++----- 6 files changed, 30 insertions(+), 13 deletions(-) diff --git a/models/repo/repo.go b/models/repo/repo.go index 12181ef71b..21c0c097ab 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -162,8 +162,8 @@ type Repository struct { BaseRepo *Repository `xorm:"-"` IsTemplate bool `xorm:"INDEX NOT NULL DEFAULT false"` TemplateID int64 `xorm:"INDEX"` - Size int64 `xorm:"NOT NULL DEFAULT 0"` SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"` + Size int64 `xorm:"NOT NULL DEFAULT 0"` EnableSizeLimit bool `xorm:"NOT NULL DEFAULT true"` CodeIndexerStatus *RepoIndexerStatus `xorm:"-"` StatsIndexerStatus *RepoIndexerStatus `xorm:"-"` @@ -538,9 +538,17 @@ func (repo *Repository) IsOwnedBy(userID int64) bool { // return repo.updateSize(ctx.e) // } +func (repo *Repository) GetActualSizeLimit() int64 { + sizeLimit := repo.SizeLimit + if setting.RepoSizeLimit > 0 && sizeLimit == 0 { + sizeLimit = setting.RepoSizeLimit + } + return sizeLimit +} + // RepoSizeIsOversized return if is over size limitation func (repo *Repository) RepoSizeIsOversized(additionalSize int64) bool { - return repo.SizeLimit > 0 && repo.Size+additionalSize > repo.SizeLimit + return setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 && repo.Size+additionalSize > repo.GetActualSizeLimit() } // CanCreateBranch returns true if repository meets the requirements for creating new branches. diff --git a/modules/setting/repository.go b/modules/setting/repository.go index a495f471d6..57eeaf99ce 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -289,8 +289,8 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { // Determine and create root git repository path. sec := rootCfg.Section("repository") - EnableSizeLimit = sec.Key("ENABLE_SIZE_LIMIT").MustBool() - RepoSizeLimit = sec.Key("REPO_SIZE_LIMIT").MustInt64(0) + EnableSizeLimit = sec.Key("ENABLE_SIZE_LIMIT").MustBool(true) + RepoSizeLimit = sec.Key("REPO_SIZE_LIMIT").MustInt64(1024 * 1024 * 10) Repository.DisableHTTPGit = sec.Key("DISABLE_HTTP_GIT").MustBool() Repository.UseCompatSSHURI = sec.Key("USE_COMPAT_SSH_URI").MustBool() Repository.MaxCreationLimit = sec.Key("MAX_CREATION_LIMIT").MustInt(-1) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 5a922c0eda..d6a28a9c0f 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3009,6 +3009,12 @@ config.picture_service = Picture Service config.disable_gravatar = Disable Gravatar config.enable_federated_avatar = Enable Federated Avatars +config.repository_config = Repository Configuration +config.enable_size_limit = Enable Size Limit +config.repo_size_limit = Default Repository Size Limit +config.invalid_repo_size = Invalid repository size. +config.repository_setting_success = Global repository setting has been updated. + config.git_config = Git Configuration config.git_disable_diff_highlight = Disable Diff Syntax Highlight config.git_max_diff_lines = Max Diff Lines (for a single file) diff --git a/routers/web/admin/repos.go b/routers/web/admin/repos.go index 2287f81fe1..13c4962c34 100644 --- a/routers/web/admin/repos.go +++ b/routers/web/admin/repos.go @@ -66,9 +66,6 @@ func UpdateRepoPost(ctx *context.Context) { } setting.SaveGlobalRepositorySetting(form.EnableSizeLimit, repo_size_limit) - ctx.Flash.Success(ctx.Tr("admin.config.repository_setting_success")) - - ctx.Data["RepoSizeLimit"] = base.FileSize(setting.RepoSizeLimit) ctx.Flash.Success(ctx.Tr("admin.config.repository_setting_success")) ctx.Redirect(setting.AppSubURL + "/admin/repos") diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index c30276c426..2999fc247e 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -67,7 +67,9 @@ func SettingsCtxData(ctx *context.Context) { ctx.Data["DisableNewPushMirrors"] = setting.Mirror.DisableNewPush ctx.Data["DefaultMirrorInterval"] = setting.Mirror.DefaultInterval ctx.Data["MinimumMirrorInterval"] = setting.Mirror.MinInterval - ctx.Data["Err_RepoSize"] = ctx.Repo.Repository.RepoSizeIsOversized(ctx.Repo.Repository.SizeLimit / 10) // less than 10% left + ctx.Data["Err_RepoSize"] = ctx.Repo.Repository.RepoSizeIsOversized(ctx.Repo.Repository.GetActualSizeLimit() / 10) // less than 10% left + ctx.Data["ActualSizeLimit"] = ctx.Repo.Repository.GetActualSizeLimit() + ctx.Data["EnableSizeLimit"] = setting.EnableSizeLimit signing, _ := asymkey_service.SigningKey(ctx, ctx.Repo.Repository.RepoPath()) ctx.Data["SigningKeyAvailable"] = len(signing) > 0 diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index b193233774..0a8232f58b 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -13,14 +13,18 @@
- - {{FileSize .Repository.Size}}{{if .Repository.SizeLimit}}/{{FileSize .Repository.SizeLimit}}{{end}} + + {{FileSize .Repository.Size}} + {{if and .Repository.SizeLimit .EnableSizeLimit}} + /{{FileSize .ActualSizeLimit}} + {{end}} +
- - - {{FileSize .Repository.Size}} +
From 0231760b60b261921f50a0a38dca241c114680d4 Mon Sep 17 00:00:00 2001 From: truecode112 Date: Mon, 15 May 2023 11:41:02 +0300 Subject: [PATCH 17/55] All done for Global Repo Size Option --- templates/repo/settings/options.tmpl | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 0a8232f58b..df028f3048 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -15,15 +15,17 @@
{{FileSize .Repository.Size}} - {{if and .Repository.SizeLimit .EnableSizeLimit}} + {{if .ActualSizeLimit }} + {{if .EnableSizeLimit }} /{{FileSize .ActualSizeLimit}} + {{end }} {{end}}
From 82f3e2d47574fa2027fe2bfcbc04357f0d852df3 Mon Sep 17 00:00:00 2001 From: truecode112 Date: Mon, 15 May 2023 12:27:22 +0300 Subject: [PATCH 18/55] add test for global repo size limit option --- routers/web/admin/repos_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 routers/web/admin/repos_test.go diff --git a/routers/web/admin/repos_test.go b/routers/web/admin/repos_test.go new file mode 100644 index 0000000000..dcfb1c7b38 --- /dev/null +++ b/routers/web/admin/repos_test.go @@ -0,0 +1,26 @@ +// Copyright 2019 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package admin + +import ( + "testing" + + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/test" + + "github.com/stretchr/testify/assert" +) + +func TestUpdateRepoPost(t *testing.T) { + unittest.PrepareTestEnv(t) + ctx := test.MockContext(t, "admin/repos") + test.LoadUser(t, ctx, 1) + + ctx.Req.Form.Set("enable_size_limit", "on") + ctx.Req.Form.Set("repo_size_limit", "222 kcmcm") + + UpdateRepoPost(ctx) + + assert.NotEmpty(t, ctx.Flash.ErrorMsg) +} From 2a1abe5006e9c0b598bbbd6573bdfa6e0a25f200 Mon Sep 17 00:00:00 2001 From: truecode112 Date: Mon, 15 May 2023 17:50:36 +0300 Subject: [PATCH 19/55] Add test script --- routers/web/admin/repos_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routers/web/admin/repos_test.go b/routers/web/admin/repos_test.go index dcfb1c7b38..961eb86276 100644 --- a/routers/web/admin/repos_test.go +++ b/routers/web/admin/repos_test.go @@ -23,4 +23,5 @@ func TestUpdateRepoPost(t *testing.T) { UpdateRepoPost(ctx) assert.NotEmpty(t, ctx.Flash.ErrorMsg) + } From 3d0004a0c4850c96c78d44069351ec2e8445cff2 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Mon, 15 May 2023 19:35:43 +0000 Subject: [PATCH 20/55] Modified app.example.ini with REPO_SIZE_LIMIT and ENABLE_SIZE_LIMIT options in the repository section --- custom/conf/app.example.ini | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index 3ceb53dcd0..05439b03d0 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -977,6 +977,14 @@ ROUTER = console ;; Allow fork repositories without maximum number limit ;ALLOW_FORK_WITHOUT_MAXIMUM_LIMIT = true +; +;; Enable applying a global size limit defined by REPO_SIZE_LIMIT. Each repository can have a value that overrides the global limit +;; "false" means no limit will be enforced, even if specified on a repository +;ENABLE_SIZE_LIMIT = false +; +;; Specify a global repository size limit to apply for each repository. 0 - No limit +;; If repository has it's own limit set in UI it will override the global setting +;REPO_SIZE_LIMIT = 0 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; From 414c90263a3e8fd03371bebce69177d0a476c49a Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Mon, 15 May 2023 20:13:27 +0000 Subject: [PATCH 21/55] updated content/doc/administration/config-cheat-sheet.en-us.md with ENABLE_SIZE_LIMIT and REPO_SIZE_LIMIT options --- docs/content/doc/administration/config-cheat-sheet.en-us.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index c1befed489..e24b2839d3 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -112,6 +112,9 @@ In addition there is _`StaticRootPath`_ which can be set as a built-in at build - `ALLOW_DELETION_OF_UNADOPTED_REPOSITORIES`: **false**: Allow non-admin users to delete unadopted repositories - `DISABLE_DOWNLOAD_SOURCE_ARCHIVES`: **false**: Don't allow download source archive files from UI - `ALLOW_FORK_WITHOUT_MAXIMUM_LIMIT`: **true**: Allow fork repositories without maximum number limit +- `ENABLE_SIZE_LIMIT`: **false**: Enable global per repository size +limit defined in `REPO_SIZE_LIMIT` or at the repository itself. +- `REPO_SIZE_LIMIT`: **0**: The global default size limit to apply per repository in bytes when `ENABLE_SIZE_LIMIT` is `true`. Value `0` - no limit. The repository limit set via user interface by admin takes precedence over this value. ### Repository - Editor (`repository.editor`) From 31600a0159806e64637af13f020ad32e9c7cf432 Mon Sep 17 00:00:00 2001 From: truecode112 Date: Tue, 16 May 2023 05:35:16 +0300 Subject: [PATCH 22/55] Config parameter REPO_SIZE_LIMIT = XXXX (should accept bytes human readable) part1, sensible defaults, app.ini saving --- modules/setting/repository.go | 23 ++++++++++++++++++----- options/locale/locale_en-US.ini | 1 + routers/web/admin/repos.go | 15 +++++++++++++-- routers/web/explore/repo.go | 9 ++++++++- 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 57eeaf99ce..08c7293d37 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -7,10 +7,10 @@ import ( "os/exec" "path" "path/filepath" - "strconv" "strings" "code.gitea.io/gitea/modules/log" + "github.com/dustin/go-humanize" ) // enumerates all the policy repository creating @@ -271,7 +271,7 @@ var ( RepoSizeLimit int64 = 0 ) -func SaveGlobalRepositorySetting(enable_size_limit bool, repo_size_limit int64) { +func SaveGlobalRepositorySetting(enable_size_limit bool, repo_size_limit int64) error { EnableSizeLimit = enable_size_limit RepoSizeLimit = repo_size_limit sec := CfgProvider.Section("repository") @@ -281,7 +281,12 @@ func SaveGlobalRepositorySetting(enable_size_limit bool, repo_size_limit int64) sec.Key("ENABLE_SIZE_LIMIT").SetValue("false") } - sec.Key("REPO_SIZE_LIMIT").SetValue(strconv.FormatInt(RepoSizeLimit, 10)) + sec.Key("REPO_SIZE_LIMIT").SetValue(humanize.Bytes(uint64(RepoSizeLimit))) + if err := CfgProvider.Save(); err != nil { + log.Fatal("Failed to save config file: %v", err) + return err + } + return nil } func loadRepositoryFrom(rootCfg ConfigProvider) { @@ -289,8 +294,16 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { // Determine and create root git repository path. sec := rootCfg.Section("repository") - EnableSizeLimit = sec.Key("ENABLE_SIZE_LIMIT").MustBool(true) - RepoSizeLimit = sec.Key("REPO_SIZE_LIMIT").MustInt64(1024 * 1024 * 10) + EnableSizeLimit = sec.Key("ENABLE_SIZE_LIMIT").MustBool(false) + + v, err := humanize.ParseBytes(sec.Key("REPO_SIZE_LIMIT").MustString("0")) + + if err == nil { + RepoSizeLimit = int64(v) + } else { + RepoSizeLimit = 0 + } + Repository.DisableHTTPGit = sec.Key("DISABLE_HTTP_GIT").MustBool() Repository.UseCompatSSHURI = sec.Key("USE_COMPAT_SSH_URI").MustBool() Repository.MaxCreationLimit = sec.Key("MAX_CREATION_LIMIT").MustInt(-1) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index d6a28a9c0f..e5878f8b69 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3013,6 +3013,7 @@ config.repository_config = Repository Configuration config.enable_size_limit = Enable Size Limit config.repo_size_limit = Default Repository Size Limit config.invalid_repo_size = Invalid repository size. +config.save_repo_size_setting_failed = Failed to save global repository settings config.repository_setting_success = Global repository setting has been updated. config.git_config = Git Configuration diff --git a/routers/web/admin/repos.go b/routers/web/admin/repos.go index 13c4962c34..a3d7c3d989 100644 --- a/routers/web/admin/repos.go +++ b/routers/web/admin/repos.go @@ -55,7 +55,7 @@ func UpdateRepoPost(ctx *context.Context) { ctx.Data["RepoSizeLimit"] = form.RepoSizeLimit if err != nil { - ctx.Data["Err_Repo_Size_Limit"] = true + ctx.Data["Err_Repo_Size_Limit"] = err.Error() explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{ Private: true, PageSize: setting.UI.Admin.RepoPagingNum, @@ -65,7 +65,18 @@ func UpdateRepoPost(ctx *context.Context) { return } - setting.SaveGlobalRepositorySetting(form.EnableSizeLimit, repo_size_limit) + err = setting.SaveGlobalRepositorySetting(form.EnableSizeLimit, repo_size_limit) + + if err != nil { + ctx.Data["Err_Repo_Size_Save"] = err.Error() + explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{ + Private: true, + PageSize: setting.UI.Admin.RepoPagingNum, + TplName: tplRepos, + OnlyShowRelevant: false, + }) + return + } ctx.Flash.Success(ctx.Tr("admin.config.repository_setting_success")) ctx.Redirect(setting.AppSubURL + "/admin/repos") diff --git a/routers/web/explore/repo.go b/routers/web/explore/repo.go index a3450a142c..365c9196a1 100644 --- a/routers/web/explore/repo.go +++ b/routers/web/explore/repo.go @@ -142,7 +142,14 @@ func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) { ctx.Data["Page"] = pager if ctx.Data["Err_Repo_Size_Limit"] != nil { - ctx.RenderWithErr(ctx.Tr("admin.config.invalid_repo_size"), opts.TplName, nil) + ctx.RenderWithErr(ctx.Tr("admin.config.invalid_repo_size")+" "+ctx.Data["Err_Repo_Size_Limit"].(string), + opts.TplName, nil) + return + } + + if ctx.Data["Err_Repo_Size_Save"] != nil { + ctx.RenderWithErr(ctx.Tr("admin.config.save_repo_size_setting_failed")+" "+ctx.Data["Err_Repo_Size_Save"].(string), + opts.TplName, nil) return } From 703ba07cea79530d9c6b42f8ac058a619157ebfc Mon Sep 17 00:00:00 2001 From: truecode112 Date: Tue, 16 May 2023 05:40:30 +0300 Subject: [PATCH 23/55] Config parameter REPO_SIZE_LIMIT = XXXX (should accept bytes human readable) part2 --- options/locale/locale_en-US.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index e5878f8b69..f94356348c 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3014,7 +3014,7 @@ config.enable_size_limit = Enable Size Limit config.repo_size_limit = Default Repository Size Limit config.invalid_repo_size = Invalid repository size. config.save_repo_size_setting_failed = Failed to save global repository settings -config.repository_setting_success = Global repository setting has been updated. +config.repository_setting_success = Global repository setting has been updated config.git_config = Git Configuration config.git_disable_diff_highlight = Disable Diff Syntax Highlight From c55ccfdb487216eddd636802fcc3b8c5825553d2 Mon Sep 17 00:00:00 2001 From: truecode112 Date: Tue, 16 May 2023 10:13:01 +0300 Subject: [PATCH 24/55] revert saving config part --- modules/setting/repository.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 08c7293d37..d220f25f2b 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -282,10 +282,10 @@ func SaveGlobalRepositorySetting(enable_size_limit bool, repo_size_limit int64) } sec.Key("REPO_SIZE_LIMIT").SetValue(humanize.Bytes(uint64(RepoSizeLimit))) - if err := CfgProvider.Save(); err != nil { - log.Fatal("Failed to save config file: %v", err) - return err - } + // if err := CfgProvider.Save(); err != nil { + // log.Fatal("Failed to save config file: %v", err) + // return err + // } return nil } From 9e5f2fd4e3b7f91aab527c33e7aecaaba4372cda Mon Sep 17 00:00:00 2001 From: truecode112 Date: Tue, 16 May 2023 11:15:54 +0300 Subject: [PATCH 25/55] lint error fixed. --- modules/setting/repository.go | 10 +++++----- routers/web/admin/repos.go | 5 ++--- routers/web/admin/repos_test.go | 1 - routers/web/web.go | 1 - templates/repo/settings/options.tmpl | 2 +- 5 files changed, 8 insertions(+), 11 deletions(-) diff --git a/modules/setting/repository.go b/modules/setting/repository.go index d220f25f2b..0690415eb4 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -267,13 +267,13 @@ var ( Storage }{} - EnableSizeLimit = true - RepoSizeLimit int64 = 0 + EnableSizeLimit = true + RepoSizeLimit int64 ) -func SaveGlobalRepositorySetting(enable_size_limit bool, repo_size_limit int64) error { - EnableSizeLimit = enable_size_limit - RepoSizeLimit = repo_size_limit +func SaveGlobalRepositorySetting(enableSizeLimit bool, repoSizeLimit int64) error { + EnableSizeLimit = enableSizeLimit + RepoSizeLimit = repoSizeLimit sec := CfgProvider.Section("repository") if EnableSizeLimit { sec.Key("ENABLE_SIZE_LIMIT").SetValue("true") diff --git a/routers/web/admin/repos.go b/routers/web/admin/repos.go index a3d7c3d989..f02ae8521a 100644 --- a/routers/web/admin/repos.go +++ b/routers/web/admin/repos.go @@ -49,7 +49,7 @@ func UpdateRepoPost(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("admin.repositories") ctx.Data["PageIsAdminRepositories"] = true - repo_size_limit, err := base.GetFileSize(form.RepoSizeLimit) + repoSizeLimit, err := base.GetFileSize(form.RepoSizeLimit) ctx.Data["EnableSizeLimit"] = form.EnableSizeLimit ctx.Data["RepoSizeLimit"] = form.RepoSizeLimit @@ -65,7 +65,7 @@ func UpdateRepoPost(ctx *context.Context) { return } - err = setting.SaveGlobalRepositorySetting(form.EnableSizeLimit, repo_size_limit) + err = setting.SaveGlobalRepositorySetting(form.EnableSizeLimit, repoSizeLimit) if err != nil { ctx.Data["Err_Repo_Size_Save"] = err.Error() @@ -80,7 +80,6 @@ func UpdateRepoPost(ctx *context.Context) { ctx.Flash.Success(ctx.Tr("admin.config.repository_setting_success")) ctx.Redirect(setting.AppSubURL + "/admin/repos") - } // DeleteRepo delete one repository diff --git a/routers/web/admin/repos_test.go b/routers/web/admin/repos_test.go index 961eb86276..dcfb1c7b38 100644 --- a/routers/web/admin/repos_test.go +++ b/routers/web/admin/repos_test.go @@ -23,5 +23,4 @@ func TestUpdateRepoPost(t *testing.T) { UpdateRepoPost(ctx) assert.NotEmpty(t, ctx.Flash.ErrorMsg) - } diff --git a/routers/web/web.go b/routers/web/web.go index b3b9d65dbf..e8fdafb15f 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -581,7 +581,6 @@ func registerRoutes(m *web.Route) { m.Combo("/unadopted").Get(admin.UnadoptedRepos).Post(admin.AdoptOrDeleteRepository) m.Post("/delete", admin.DeleteRepo) m.Post("", web.Bind(forms.UpdateGlobalRepoFrom{}), admin.UpdateRepoPost) - }) m.Group("/packages", func() { diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index df028f3048..2aaf0c3b4a 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -24,7 +24,7 @@
-
From 80f8977e596db4fbeac43c37af3c123c4c4b95f6 Mon Sep 17 00:00:00 2001 From: truecode112 Date: Tue, 16 May 2023 12:27:49 +0300 Subject: [PATCH 26/55] fixed issues for testing --- modules/base/tool.go | 5 ++++- modules/base/tool_test.go | 7 +++++++ routers/web/admin/repos.go | 13 ++++++++++++- 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/modules/base/tool.go b/modules/base/tool.go index 96d1391e3a..5fe486b9b9 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -138,7 +138,10 @@ func CreateTimeLimitCode(data string, minutes int, startInf interface{}) string // FileSize calculates the file size and generate user-friendly string. func FileSize(s int64) string { - return humanize.Bytes(uint64(s)) + // sizes := []string{"B", "kB", "MB", "GB", "TB", "PB", "EB"} + // return humanize.humanateBytes(uint64(s), 1000, sizes) + // return humanize.Bytes(uint64(s)) + return humanize.IBytes(uint64(s)) } // Get FileSize bytes value from String. diff --git a/modules/base/tool_test.go b/modules/base/tool_test.go index 0c3e76704e..8834d0e972 100644 --- a/modules/base/tool_test.go +++ b/modules/base/tool_test.go @@ -114,6 +114,13 @@ func TestFileSize(t *testing.T) { assert.Equal(t, "2.0 EiB", FileSize(size)) } +func TestGetFileSize(t *testing.T) { + var size int64 = 512 * 1024 * 1024 * 1024 + s, err := GetFileSize("512 GiB") + assert.Equal(t, s, size) + assert.Equal(t, err, nil) +} + func TestEllipsisString(t *testing.T) { assert.Equal(t, "...", EllipsisString("foobar", 0)) assert.Equal(t, "...", EllipsisString("foobar", 1)) diff --git a/routers/web/admin/repos.go b/routers/web/admin/repos.go index f02ae8521a..360d316604 100644 --- a/routers/web/admin/repos.go +++ b/routers/web/admin/repos.go @@ -45,7 +45,18 @@ func Repos(ctx *context.Context) { } func UpdateRepoPost(ctx *context.Context) { - form := web.GetForm(ctx).(*forms.UpdateGlobalRepoFrom) + temp := web.GetForm(ctx) + if temp == nil { + ctx.Data["Err_Repo_Size_Limit"] = "" + explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{ + Private: true, + PageSize: setting.UI.Admin.RepoPagingNum, + TplName: tplRepos, + OnlyShowRelevant: false, + }) + return + } + form := temp.(*forms.UpdateGlobalRepoFrom) ctx.Data["Title"] = ctx.Tr("admin.repositories") ctx.Data["PageIsAdminRepositories"] = true From 8ac833ae621b7973fa2e38f749d9b9b5595345f6 Mon Sep 17 00:00:00 2001 From: truecode112 Date: Wed, 17 May 2023 02:20:59 +0300 Subject: [PATCH 27/55] Fixed issues in test scripts. --- modules/setting/repository.go | 1 + templates/repo/settings/options.tmpl | 6 +++--- tests/integration/git_test.go | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 0690415eb4..3bca41076f 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -10,6 +10,7 @@ import ( "strings" "code.gitea.io/gitea/modules/log" + "github.com/dustin/go-humanize" ) diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index 2aaf0c3b4a..1f408d2e31 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -15,10 +15,10 @@
{{FileSize .Repository.Size}} - {{if .ActualSizeLimit }} - {{if .EnableSizeLimit }} + {{if .ActualSizeLimit}} + {{if .EnableSizeLimit}} /{{FileSize .ActualSizeLimit}} - {{end }} + {{end}} {{end}}
diff --git a/tests/integration/git_test.go b/tests/integration/git_test.go index f163d8e835..cdff57f21f 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_test.go @@ -82,6 +82,7 @@ func testGit(t *testing.T, u *url.URL) { mediaTest(t, &httpContext, little, big, littleLFS, bigLFS) t.Run("SizeLimit", func(t *testing.T) { + setting.SaveGlobalRepositorySetting(true, 0) t.Run("Under", func(t *testing.T) { defer tests.PrintCurrentTest(t)() doCommitAndPush(t, littleSize, dstPath, "data-file-") From 4936e49096dba0d44c131076bf2863a163f43a08 Mon Sep 17 00:00:00 2001 From: truecode112 Date: Wed, 17 May 2023 21:39:33 +0300 Subject: [PATCH 28/55] present a correct message when repo is over --- routers/private/hook_pre_receive.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index 35850dcaae..708eb2057f 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -17,6 +17,7 @@ import ( access_model "code.gitea.io/gitea/models/perm/access" "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/base" gitea_context "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" @@ -131,11 +132,15 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { refFullName := opts.RefFullNames[i] // Check size - if newCommitID != git.EmptySHA && repo.RepoSizeIsOversized(pushSize.Size) { // Check next size if we are not deleting a reference - log.Warn("Forbidden: new repo size is over limitation: %d", repo.SizeLimit) - ctx.JSON(http.StatusForbidden, map[string]interface{}{ - "err": fmt.Sprintf("new repo size is over limitation: %d", repo.SizeLimit), + if newCommitID != git.EmptySHA && pushSize.Size > 0 && repo.RepoSizeIsOversized(pushSize.Size) { // Check next size if we are not deleting a reference + log.Warn("Forbidden: new repo size is over limitation: %s", base.FileSize(repo.GetActualSizeLimit())) + ctx.JSON(http.StatusForbidden, private.Response{ + UserMsg: fmt.Sprintf("new repo size is over limitation: %s", base.FileSize(repo.GetActualSizeLimit())), }) + return + // ctx.JSON(http.StatusForbidden, map[string]interface{}{ + // "err": fmt.Sprintf("new repo size is over limitation: %d", repo.GetActualSizeLimit()), + // }) } // TODO investigate why on force push some git objects are not cleaned on server side. // TODO corner-case force push and branch creation -> git.EmptySHA == oldCommitID From e9c29f82f24ded9adcba4c14acb17ada5e886127 Mon Sep 17 00:00:00 2001 From: truecode112 Date: Thu, 18 May 2023 11:47:12 +0300 Subject: [PATCH 29/55] doDeleteCommitAndPush implemented --- modules/git/commit.go | 8 +++++++ tests/integration/git_test.go | 44 +++++++++++++++++++++++++++++------ 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/modules/git/commit.go b/modules/git/commit.go index ff654f394d..b3db96a649 100644 --- a/modules/git/commit.go +++ b/modules/git/commit.go @@ -100,6 +100,14 @@ func AddChangesWithArgs(repoPath string, globalArgs TrustedCmdArgs, all bool, fi return err } +// DeleteChangesWithArgs marks local changes to be ready for commit. +func DeleteChangesWithArgs(repoPath string, globalArgs TrustedCmdArgs, files ...string) error { + cmd := NewCommandContextNoGlobals(DefaultContext, globalArgs...).AddArguments("rm") + cmd.AddDashesAndList(files...) + _, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) + return err +} + // CommitChangesOptions the options when a commit created type CommitChangesOptions struct { Committer *Signature diff --git a/tests/integration/git_test.go b/tests/integration/git_test.go index cdff57f21f..25f702ff14 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_test.go @@ -102,14 +102,11 @@ func testGit(t *testing.T, u *url.URL) { }) t.Run("Deletion", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - // TODO doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") + filename := doCommitAndPush(t, bigSize, dstForkedPath, "data-file-") + t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, littleSize)) + err := doDeleteCommitAndPush(t, dstPath, filename) + assert.Error(t, err) }) - // TODO delete branch - // TODO delete tag - // TODO add big commit that will be over with the push - // TODO add lfs - // TODO remove lfs - // TODO add missing case }) t.Run("CreateAgitFlowPull", doCreateAgitFlowPull(dstPath, &httpContext, "master", "test/head")) t.Run("BranchProtectMerge", doBranchProtectPRMerge(&httpContext, dstPath)) @@ -333,6 +330,15 @@ func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string { return name } +func doDeleteCommitAndPush(t *testing.T, repoPath, filename string) error { + var err = generateDeleteCommit(repoPath, "user2@example.com", "User Two", filename) + if err != nil { + return err + } + _, _, err = git.NewCommand(git.DefaultContext, "push", "origin", "master").RunStdString(&git.RunOpts{Dir: repoPath}) // Push + return err +} + func doCommitAndPushWithExpectedError(t *testing.T, size int, repoPath, prefix string) string { name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix) assert.NoError(t, err) @@ -398,6 +404,30 @@ func generateCommitWithNewData(size int, repoPath, email, fullName, prefix strin return filepath.Base(tmpFile.Name()), err } +func generateDeleteCommit(repoPath, email, fullName, file string) error { + // Commit + // Now here we should explicitly allow lfs filters to run + globalArgs := git.AllowLFSFiltersArgs() + err := git.DeleteChangesWithArgs(repoPath, globalArgs, filepath.Base(file)) + if err != nil { + return err + } + err = git.CommitChangesWithArgs(repoPath, globalArgs, git.CommitChangesOptions{ + Committer: &git.Signature{ + Email: email, + Name: fullName, + When: time.Now(), + }, + Author: &git.Signature{ + Email: email, + Name: fullName, + When: time.Now(), + }, + Message: fmt.Sprintf("Testing commit @ %v", time.Now()), + }) + return err +} + func doBranchProtectPRMerge(baseCtx *APITestContext, dstPath string) func(t *testing.T) { return func(t *testing.T) { defer tests.PrintCurrentTest(t)() From 57c94e63cbb0da55635ef503febe9a06400bc7ea Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Fri, 19 May 2023 06:36:59 +0000 Subject: [PATCH 30/55] Allow operations that potentially reduce repository size through even if size limit is over --- routers/private/hook_pre_receive.go | 116 ++++++++++++++++++++++++---- 1 file changed, 103 insertions(+), 13 deletions(-) diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index 708eb2057f..3813ce66b4 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -7,6 +7,7 @@ import ( "fmt" "net/http" "os" + "strconv" "strings" "code.gitea.io/gitea/models" @@ -103,6 +104,68 @@ func (ctx *preReceiveContext) AssertCreatePullRequest() bool { return true } +// CalculateSizeOfAddedObjects calculates the total size of objects provided as output from rev-list command +func CalculateSizeOfAddedObjects(ctx *gitea_context.PrivateContext, opts *git.RunOpts, revlistObjects string) int64 { + // Calculate the size of added Objects. + var totalSize int64 + for _, object := range strings.Split(revlistObjects, "\n") { + if len(object) == 0 { + continue + } + objectID := strings.Split(object, " ")[0] + objectSizeStr, _, err := git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(objectID).RunStdString(opts) + if err != nil { + log.Trace("CalculateSizeOfAddedObjects: Error during git cat-file -s on object %s", objectID) + return totalSize + } + objectSize, _ := strconv.ParseInt(strings.TrimSpace(objectSizeStr), 10, 64) + if err != nil { + log.Trace("CalculateSizeOfAddedObjects: Error during ParseInt on string: '%s'", objectSizeStr) + return totalSize + } + totalSize += objectSize + } + return totalSize +} + +// CalculateSizeOfRemovedObjects calculates the size of removed objects provided as output from rev-list command +// and confirms that the object is not referenced anywhere +func CalculateSizeOfRemovedObjects(ctx *gitea_context.PrivateContext, opts *git.RunOpts, revlistObjects string) int64 { + var totalSize int64 + for _, object := range strings.Split(revlistObjects, "\n") { + if len(object) == 0 { + continue + } + objectID := strings.Split(object, " ")[0] + + // Confirm that the object is still reachable from anywhere in the repository. + isReachable, _, err := git.NewCommand(ctx, "rev-list", "--objects", "--all", "--").AddDynamicArguments(objectID).RunStdString(opts) + if err != nil { + log.Trace("CalculateSizeOfRemovedObjects: Error during git rev-list --objects --all on object: %s", objectID) + return totalSize + } + + if isReachable != "" { + // The object is still reachable, therefore we don't add it's size + continue + } + + objectSizeStr, _, err := git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(objectID).RunStdString(opts) + if err != nil { + log.Trace("CalculateSizeOfRemovedObjects: Error during git cat-file -s on object: %s", objectID) + return totalSize + } + + objectSize, _ := strconv.ParseInt(strings.TrimSpace(objectSizeStr), 10, 64) + if err != nil { + log.Trace("CalculateSizeOfRemovedObjects: Error during ParseInt on string '%s'", objectID) + return totalSize + } + totalSize += objectSize + } + return totalSize +} + // HookPreReceive checks whether a individual commit is acceptable func HookPreReceive(ctx *gitea_context.PrivateContext) { opts := web.GetForm(ctx).(*private.HookOptions) @@ -115,6 +178,10 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { repo := ourCtx.Repo.Repository + var removedSize int64 + var addedSize int64 + + // Calculating total size of the push using git count-objects pushSize, err := git.CountObjectsWithEnv(ctx, repo.RepoPath(), ourCtx.env) if err != nil { log.Error("Unable to get repository size with env %v: %s Error: %v", repo.RepoPath(), ourCtx.env, err) @@ -123,6 +190,9 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { }) return } + + // Cash whether the repository would breach the size limit after the operation + isRepoOversized := repo.RepoSizeIsOversized(pushSize.Size) log.Trace("Push size %d", pushSize.Size) // Iterate across the provided old commit IDs @@ -131,20 +201,31 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { newCommitID := opts.NewCommitIDs[i] refFullName := opts.RefFullNames[i] - // Check size - if newCommitID != git.EmptySHA && pushSize.Size > 0 && repo.RepoSizeIsOversized(pushSize.Size) { // Check next size if we are not deleting a reference - log.Warn("Forbidden: new repo size is over limitation: %s", base.FileSize(repo.GetActualSizeLimit())) - ctx.JSON(http.StatusForbidden, private.Response{ - UserMsg: fmt.Sprintf("new repo size is over limitation: %s", base.FileSize(repo.GetActualSizeLimit())), - }) - return - // ctx.JSON(http.StatusForbidden, map[string]interface{}{ - // "err": fmt.Sprintf("new repo size is over limitation: %d", repo.GetActualSizeLimit()), - // }) + // If operation is in potential breach of size limit prepare data for analysis + if isRepoOversized { + + // Objects that are in newCommitID but not in oldCommitID are added + addedObjects, _, err := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(newCommitID, "^"+oldCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) + if err != nil { + log.Error("Unable to list objects added between commits: %s and %s in %-v Error: %v", oldCommitID, newCommitID, repo, err) + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: fmt.Sprintf("Fail to list objects added: %v", err), + }) + return + } + addedSize += CalculateSizeOfAddedObjects(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}, addedObjects) + + // Objects that are in oldCommitID but not in newCommitID are removed + removedObjects, _, err := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(oldCommitID, "^"+newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) + if err != nil { + log.Error("Unable to list objects removed between commits: %s and %s in %-v Error: %v", oldCommitID, newCommitID, repo, err) + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: fmt.Sprintf("Fail to list objects removed: %v", err), + }) + return + } + removedSize += CalculateSizeOfRemovedObjects(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}, removedObjects) } - // TODO investigate why on force push some git objects are not cleaned on server side. - // TODO corner-case force push and branch creation -> git.EmptySHA == oldCommitID - // TODO calculate pushed LFS objects size switch { case strings.HasPrefix(refFullName, git.BranchPrefix): @@ -161,6 +242,15 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { } } + // If total of commits add more size then they remove and we are in a potential breach of size limit -- abort + if (addedSize > removedSize) && isRepoOversized { // Check next size if we are not deleting a reference + log.Warn("Forbidden: new repo size is over limitation: %s", base.FileSize(repo.GetActualSizeLimit())) + ctx.JSON(http.StatusForbidden, private.Response{ + UserMsg: fmt.Sprintf("Repository size is over limitation of %s", base.FileSize(repo.GetActualSizeLimit())), + }) + return + } + ctx.PlainText(http.StatusOK, "ok") } From 656a3dc72190510f7841ef9873e532886cb95336 Mon Sep 17 00:00:00 2001 From: Dmitry Frolov <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Fri, 19 May 2023 06:44:48 +0000 Subject: [PATCH 31/55] Update modules/setting/repository.go Co-authored-by: KN4CK3R --- modules/setting/repository.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 1733310f3f..a94291d4df 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -298,13 +298,7 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { sec := rootCfg.Section("repository") EnableSizeLimit = sec.Key("ENABLE_SIZE_LIMIT").MustBool(false) - v, err := humanize.ParseBytes(sec.Key("REPO_SIZE_LIMIT").MustString("0")) - - if err == nil { - RepoSizeLimit = int64(v) - } else { - RepoSizeLimit = 0 - } + RepoSizeLimit, _ = humanize.ParseBytes(sec.Key("REPO_SIZE_LIMIT").MustString("0")) Repository.DisableHTTPGit = sec.Key("DISABLE_HTTP_GIT").MustBool() Repository.UseCompatSSHURI = sec.Key("USE_COMPAT_SSH_URI").MustBool() From ef760d7e66160f0b0a861e3f670c716af5aa8fc2 Mon Sep 17 00:00:00 2001 From: Dmitry Frolov <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Fri, 19 May 2023 06:47:34 +0000 Subject: [PATCH 32/55] Update modules/base/tool_test.go Co-authored-by: KN4CK3R --- modules/base/tool_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/base/tool_test.go b/modules/base/tool_test.go index 8834d0e972..00b951efb2 100644 --- a/modules/base/tool_test.go +++ b/modules/base/tool_test.go @@ -118,7 +118,7 @@ func TestGetFileSize(t *testing.T) { var size int64 = 512 * 1024 * 1024 * 1024 s, err := GetFileSize("512 GiB") assert.Equal(t, s, size) - assert.Equal(t, err, nil) + assert.Nil(t, err) } func TestEllipsisString(t *testing.T) { From 51a769720f5f2fd2c632aa8a8edee8bfde61abf7 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Fri, 19 May 2023 06:58:56 +0000 Subject: [PATCH 33/55] Fix in /modules/setting/repository.go --- modules/setting/repository.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/setting/repository.go b/modules/setting/repository.go index a94291d4df..0adf6a95a0 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -298,7 +298,8 @@ func loadRepositoryFrom(rootCfg ConfigProvider) { sec := rootCfg.Section("repository") EnableSizeLimit = sec.Key("ENABLE_SIZE_LIMIT").MustBool(false) - RepoSizeLimit, _ = humanize.ParseBytes(sec.Key("REPO_SIZE_LIMIT").MustString("0")) + v, _ := humanize.ParseBytes(sec.Key("REPO_SIZE_LIMIT").MustString("0")) + RepoSizeLimit = int64(v) Repository.DisableHTTPGit = sec.Key("DISABLE_HTTP_GIT").MustBool() Repository.UseCompatSSHURI = sec.Key("USE_COMPAT_SSH_URI").MustBool() From 2cbdee0104a379ce4c07260b67be6edc339fcfd2 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Fri, 19 May 2023 14:37:11 +0000 Subject: [PATCH 34/55] Revert "doDeleteCommitAndPush implemented" This reverts commit e9c29f82f24ded9adcba4c14acb17ada5e886127. --- modules/git/commit.go | 8 ------- tests/integration/git_test.go | 44 ++++++----------------------------- 2 files changed, 7 insertions(+), 45 deletions(-) diff --git a/modules/git/commit.go b/modules/git/commit.go index b3db96a649..ff654f394d 100644 --- a/modules/git/commit.go +++ b/modules/git/commit.go @@ -100,14 +100,6 @@ func AddChangesWithArgs(repoPath string, globalArgs TrustedCmdArgs, all bool, fi return err } -// DeleteChangesWithArgs marks local changes to be ready for commit. -func DeleteChangesWithArgs(repoPath string, globalArgs TrustedCmdArgs, files ...string) error { - cmd := NewCommandContextNoGlobals(DefaultContext, globalArgs...).AddArguments("rm") - cmd.AddDashesAndList(files...) - _, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}) - return err -} - // CommitChangesOptions the options when a commit created type CommitChangesOptions struct { Committer *Signature diff --git a/tests/integration/git_test.go b/tests/integration/git_test.go index 25f702ff14..cdff57f21f 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_test.go @@ -102,11 +102,14 @@ func testGit(t *testing.T, u *url.URL) { }) t.Run("Deletion", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - filename := doCommitAndPush(t, bigSize, dstForkedPath, "data-file-") - t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, littleSize)) - err := doDeleteCommitAndPush(t, dstPath, filename) - assert.Error(t, err) + // TODO doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") }) + // TODO delete branch + // TODO delete tag + // TODO add big commit that will be over with the push + // TODO add lfs + // TODO remove lfs + // TODO add missing case }) t.Run("CreateAgitFlowPull", doCreateAgitFlowPull(dstPath, &httpContext, "master", "test/head")) t.Run("BranchProtectMerge", doBranchProtectPRMerge(&httpContext, dstPath)) @@ -330,15 +333,6 @@ func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string { return name } -func doDeleteCommitAndPush(t *testing.T, repoPath, filename string) error { - var err = generateDeleteCommit(repoPath, "user2@example.com", "User Two", filename) - if err != nil { - return err - } - _, _, err = git.NewCommand(git.DefaultContext, "push", "origin", "master").RunStdString(&git.RunOpts{Dir: repoPath}) // Push - return err -} - func doCommitAndPushWithExpectedError(t *testing.T, size int, repoPath, prefix string) string { name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix) assert.NoError(t, err) @@ -404,30 +398,6 @@ func generateCommitWithNewData(size int, repoPath, email, fullName, prefix strin return filepath.Base(tmpFile.Name()), err } -func generateDeleteCommit(repoPath, email, fullName, file string) error { - // Commit - // Now here we should explicitly allow lfs filters to run - globalArgs := git.AllowLFSFiltersArgs() - err := git.DeleteChangesWithArgs(repoPath, globalArgs, filepath.Base(file)) - if err != nil { - return err - } - err = git.CommitChangesWithArgs(repoPath, globalArgs, git.CommitChangesOptions{ - Committer: &git.Signature{ - Email: email, - Name: fullName, - When: time.Now(), - }, - Author: &git.Signature{ - Email: email, - Name: fullName, - When: time.Now(), - }, - Message: fmt.Sprintf("Testing commit @ %v", time.Now()), - }) - return err -} - func doBranchProtectPRMerge(baseCtx *APITestContext, dstPath string) func(t *testing.T) { return func(t *testing.T) { defer tests.PrintCurrentTest(t)() From 88e74d67e48bd074d51b9d2c17fd397d3cd489f8 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Sun, 21 May 2023 07:10:34 +0000 Subject: [PATCH 35/55] Correctly report messages on erros with rev-list size getting on pre recieve hook --- routers/private/hook_pre_receive.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index 3813ce66b4..ddde140d17 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -207,7 +207,7 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { // Objects that are in newCommitID but not in oldCommitID are added addedObjects, _, err := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(newCommitID, "^"+oldCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) if err != nil { - log.Error("Unable to list objects added between commits: %s and %s in %-v Error: %v", oldCommitID, newCommitID, repo, err) + log.Error("Unable to list objects in %s and not in %s in %-v Error: %v", newCommitID, oldCommitID, repo, err) ctx.JSON(http.StatusInternalServerError, private.Response{ Err: fmt.Sprintf("Fail to list objects added: %v", err), }) @@ -218,7 +218,7 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { // Objects that are in oldCommitID but not in newCommitID are removed removedObjects, _, err := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(oldCommitID, "^"+newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) if err != nil { - log.Error("Unable to list objects removed between commits: %s and %s in %-v Error: %v", oldCommitID, newCommitID, repo, err) + log.Error("Unable to list objects in %s and not in %s in %-v Error: %v", oldCommitID, newCommitID, repo, err) ctx.JSON(http.StatusInternalServerError, private.Response{ Err: fmt.Sprintf("Fail to list objects removed: %v", err), }) From db5a8da26577ec4aece2bd3d0d0d1e1383d7766f Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Tue, 23 May 2023 04:23:59 +0000 Subject: [PATCH 36/55] Refactor hook_pre_recieve to increase speed of size checking --- routers/private/hook_pre_receive.go | 153 ++++++++++++++++------------ 1 file changed, 86 insertions(+), 67 deletions(-) diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index ddde140d17..e0be78c4dd 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -9,6 +9,7 @@ import ( "os" "strconv" "strings" + "time" "code.gitea.io/gitea/models" asymkey_model "code.gitea.io/gitea/models/asymkey" @@ -104,70 +105,72 @@ func (ctx *preReceiveContext) AssertCreatePullRequest() bool { return true } -// CalculateSizeOfAddedObjects calculates the total size of objects provided as output from rev-list command -func CalculateSizeOfAddedObjects(ctx *gitea_context.PrivateContext, opts *git.RunOpts, revlistObjects string) int64 { - // Calculate the size of added Objects. - var totalSize int64 - for _, object := range strings.Split(revlistObjects, "\n") { - if len(object) == 0 { - continue - } - objectID := strings.Split(object, " ")[0] - objectSizeStr, _, err := git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(objectID).RunStdString(opts) - if err != nil { - log.Trace("CalculateSizeOfAddedObjects: Error during git cat-file -s on object %s", objectID) - return totalSize - } - objectSize, _ := strconv.ParseInt(strings.TrimSpace(objectSizeStr), 10, 64) - if err != nil { - log.Trace("CalculateSizeOfAddedObjects: Error during ParseInt on string: '%s'", objectSizeStr) - return totalSize - } - totalSize += objectSize +// CalculateSizeOfObject calculates the size of one git object via git cat-file -s command +func CalculateSizeOfObject(ctx *gitea_context.PrivateContext, opts *git.RunOpts, objectID string) (objectSize int64) { + objectSizeStr, _, err := git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(objectID).RunStdString(opts) + if err != nil { + log.Trace("CalculateSizeOfRemovedObjects: Error during git cat-file -s on object: %s", objectID) + return } - return totalSize + + objectSize, _ = strconv.ParseInt(strings.TrimSpace(objectSizeStr), 10, 64) + if err != nil { + log.Trace("CalculateSizeOfRemovedObjects: Error during ParseInt on string '%s'", objectID) + return + } + return } -// CalculateSizeOfRemovedObjects calculates the size of removed objects provided as output from rev-list command -// and confirms that the object is not referenced anywhere -func CalculateSizeOfRemovedObjects(ctx *gitea_context.PrivateContext, opts *git.RunOpts, revlistObjects string) int64 { - var totalSize int64 - for _, object := range strings.Split(revlistObjects, "\n") { +// CalculateSizeOfObjects calculates the size of objects added and removed from the repository by new commit +func CalculateSizeOfObjects(ctx *gitea_context.PrivateContext, opts *git.RunOpts, newCommitObjects map[string]bool, oldCommitObjects map[string]bool, otherCommitObjects map[string]bool) (addedSize int64, removedSize int64) { + + // Calculate size of objects that were added + for objectID := range newCommitObjects { + if _, exists := oldCommitObjects[objectID]; !exists { + // objectID is not referenced in the list of objects of old commit so it is a new object + // Calculate its size and add it to the addedSize + addedSize += CalculateSizeOfObject(ctx, opts, objectID) + } + // We might check here if new object is not already in the rest of repo to be precise + // However our goal is to prevent growth of repository so on determination of addedSize + // We can skip this preciseness, addedSize will be more then real addedSize + // TODO - do not count size of object that is referenced in other part of repo but not referenced neither in old nor new commit + // git will not add the object twice + } + + // Calculate size of objects that were removed + for objectID := range oldCommitObjects { + if _, exists := newCommitObjects[objectID]; !exists { + // objectID is not referenced in the list of new commit objects so it was possibly removed + if _, exists := otherCommitObjects[objectID]; !exists { + // objectID is not referenced in rest of the objects of the repository so it was removed + // Calculate its size and add it to the addedSize + removedSize += CalculateSizeOfObject(ctx, opts, objectID) + } + } + + } + return +} + +// ConvertObjectsToMap takes a newline-separated string of git objects and +// converts it into a map for efficient lookup. +func ConvertObjectsToMap(objects string) map[string]bool { + objectsMap := make(map[string]bool) + for _, object := range strings.Split(objects, "\n") { if len(object) == 0 { continue } objectID := strings.Split(object, " ")[0] - - // Confirm that the object is still reachable from anywhere in the repository. - isReachable, _, err := git.NewCommand(ctx, "rev-list", "--objects", "--all", "--").AddDynamicArguments(objectID).RunStdString(opts) - if err != nil { - log.Trace("CalculateSizeOfRemovedObjects: Error during git rev-list --objects --all on object: %s", objectID) - return totalSize - } - - if isReachable != "" { - // The object is still reachable, therefore we don't add it's size - continue - } - - objectSizeStr, _, err := git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(objectID).RunStdString(opts) - if err != nil { - log.Trace("CalculateSizeOfRemovedObjects: Error during git cat-file -s on object: %s", objectID) - return totalSize - } - - objectSize, _ := strconv.ParseInt(strings.TrimSpace(objectSizeStr), 10, 64) - if err != nil { - log.Trace("CalculateSizeOfRemovedObjects: Error during ParseInt on string '%s'", objectID) - return totalSize - } - totalSize += objectSize + objectsMap[objectID] = true } - return totalSize + return objectsMap } // HookPreReceive checks whether a individual commit is acceptable func HookPreReceive(ctx *gitea_context.PrivateContext) { + startTime := time.Now() + opts := web.GetForm(ctx).(*private.HookOptions) ourCtx := &preReceiveContext{ @@ -178,8 +181,8 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { repo := ourCtx.Repo.Repository - var removedSize int64 var addedSize int64 + var removedSize int64 // Calculating total size of the push using git count-objects pushSize, err := git.CountObjectsWithEnv(ctx, repo.RepoPath(), ourCtx.env) @@ -191,7 +194,7 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { return } - // Cash whether the repository would breach the size limit after the operation + // Cache whether the repository would breach the size limit after the operation isRepoOversized := repo.RepoSizeIsOversized(pushSize.Size) log.Trace("Push size %d", pushSize.Size) @@ -203,28 +206,41 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { // If operation is in potential breach of size limit prepare data for analysis if isRepoOversized { - - // Objects that are in newCommitID but not in oldCommitID are added - addedObjects, _, err := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(newCommitID, "^"+oldCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) + // Create cache of objects in old commit + gitObjects, _, err := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(oldCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) if err != nil { - log.Error("Unable to list objects in %s and not in %s in %-v Error: %v", newCommitID, oldCommitID, repo, err) + log.Error("Unable to list objects in old commit: %s in %-v Error: %v", oldCommitID, repo, err) ctx.JSON(http.StatusInternalServerError, private.Response{ - Err: fmt.Sprintf("Fail to list objects added: %v", err), + Err: fmt.Sprintf("Fail to list objects in old commit: %v", err), }) return } - addedSize += CalculateSizeOfAddedObjects(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}, addedObjects) + oldCommitObjects := ConvertObjectsToMap(gitObjects) - // Objects that are in oldCommitID but not in newCommitID are removed - removedObjects, _, err := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(oldCommitID, "^"+newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) + // Create cache of objects in new commit + gitObjects, _, err = git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) if err != nil { - log.Error("Unable to list objects in %s and not in %s in %-v Error: %v", oldCommitID, newCommitID, repo, err) + log.Error("Unable to list objects in new commit %s in %-v Error: %v", newCommitID, repo, err) ctx.JSON(http.StatusInternalServerError, private.Response{ - Err: fmt.Sprintf("Fail to list objects removed: %v", err), + Err: fmt.Sprintf("Fail to list objects in new commit: %v", err), }) return } - removedSize += CalculateSizeOfRemovedObjects(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}, removedObjects) + newCommitObjects := ConvertObjectsToMap(gitObjects) + + // Create cache of objects that are in the repository but not part of old or new commit + gitObjects, _, err = git.NewCommand(ctx, "rev-list", "--objects", "--all").AddDynamicArguments("^"+oldCommitID, "^"+newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) + if err != nil { + log.Error("Unable to list objects in the repo that are missing from both old %s and new %s commits in %-v Error: %v", oldCommitID, newCommitID, repo, err) + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: fmt.Sprintf("Fail to list objects missing from both old and new commits: %v", err), + }) + return + } + otherCommitObjects := ConvertObjectsToMap(gitObjects) + + // Calculate size that was added and removed by the new commit + addedSize, removedSize = CalculateSizeOfObjects(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}, newCommitObjects, oldCommitObjects, otherCommitObjects) } switch { @@ -242,9 +258,12 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { } } + duration := time.Since(startTime) + log.Warn("Addition in size is: %d, removal in size is: %d, limit size: %s, push size: %d. Took %s seconds.", addedSize, removedSize, base.FileSize(repo.GetActualSizeLimit()), pushSize.Size, duration) + // If total of commits add more size then they remove and we are in a potential breach of size limit -- abort - if (addedSize > removedSize) && isRepoOversized { // Check next size if we are not deleting a reference - log.Warn("Forbidden: new repo size is over limitation: %s", base.FileSize(repo.GetActualSizeLimit())) + if (addedSize > removedSize) && isRepoOversized { + log.Warn("Forbidden: new repo size %s is over limitation of %s. Push size: %s. Took %s seconds.", base.FileSize(addedSize-removedSize), base.FileSize(repo.GetActualSizeLimit()), base.FileSize(pushSize.Size), duration) ctx.JSON(http.StatusForbidden, private.Response{ UserMsg: fmt.Sprintf("Repository size is over limitation of %s", base.FileSize(repo.GetActualSizeLimit())), }) From 08c65ffb1725d9fad8421705245270f8ceb87a4e Mon Sep 17 00:00:00 2001 From: truecode112 Date: Tue, 23 May 2023 08:18:41 +0300 Subject: [PATCH 37/55] Update for TestScript to Reduce RepoSize --- modules/repository/create.go | 20 ++++++++ routers/private/hook_pre_receive.go | 6 ++- services/repository/push.go | 10 ++-- tests/integration/git_test.go | 79 ++++++++++++++++++++++++++++- 4 files changed, 109 insertions(+), 6 deletions(-) diff --git a/modules/repository/create.go b/modules/repository/create.go index 63be5cac25..8240600be1 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -332,6 +332,26 @@ func UpdateRepoSize(ctx context.Context, repo *repo_model.Repository) error { return fmt.Errorf("updateSize: GetLFSMetaObjects: %w", err) } + if setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 && size+lfsSize > repo.GetActualSizeLimit() { + // return fmt.Errorf("updateSize: Git Reflog Failed Size(%d) > Limit(%d)", size+lfsSize, repo.GetActualSizeLimit()) + + _, _, err = git.NewCommand(git.DefaultContext, "reflog", "expire", "--expire-unreachable=all", "--all").RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) // Push + if err != nil { + return fmt.Errorf("updateSize: Git Reflog Failed: %w", err) + } + + _, _, err = git.NewCommand(git.DefaultContext, "gc", "--prune=now").RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) // Push + if err != nil { + return fmt.Errorf("updateSize: Git GC Failed: %w", err) + } + + size, err = getDirectorySize(repo.RepoPath()) + if err != nil { + return fmt.Errorf("updateSize: %w", err) + } + // return fmt.Errorf("updateSize: Git Reflog Failed Size(%d) > Limit(%d)", size+lfsSize, repo.GetActualSizeLimit()) + } + return repo_model.UpdateRepoSize(ctx, repo.ID, size+lfsSize) } diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index 3813ce66b4..a4e1ad94f6 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -204,6 +204,10 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { // If operation is in potential breach of size limit prepare data for analysis if isRepoOversized { + // ctx.JSON(http.StatusForbidden, private.Response{ + // UserMsg: fmt.Sprintf("oldCommitID(%s) newCommitID(%s)", oldCommitID, newCommitID), + // }) + // Objects that are in newCommitID but not in oldCommitID are added addedObjects, _, err := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(newCommitID, "^"+oldCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) if err != nil { @@ -246,7 +250,7 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { if (addedSize > removedSize) && isRepoOversized { // Check next size if we are not deleting a reference log.Warn("Forbidden: new repo size is over limitation: %s", base.FileSize(repo.GetActualSizeLimit())) ctx.JSON(http.StatusForbidden, private.Response{ - UserMsg: fmt.Sprintf("Repository size is over limitation of %s", base.FileSize(repo.GetActualSizeLimit())), + UserMsg: fmt.Sprintf("Repository size is over limitation of %s addedSize(%d) removedSize(%d) repo(%s)", base.FileSize(repo.GetActualSizeLimit()), addedSize, removedSize, repo.RepoPath()), }) return } diff --git a/services/repository/push.go b/services/repository/push.go index c7ea8f336e..a6ba284bc7 100644 --- a/services/repository/push.go +++ b/services/repository/push.go @@ -93,9 +93,9 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { } defer gitRepo.Close() - if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { - log.Error("Failed to update size for repository: %v", err) - } + // if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + // log.Error("Failed to update size for repository: %v", err) + // } addTags := make([]string, 0, len(optsList)) delTags := make([]string, 0, len(optsList)) @@ -293,6 +293,10 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { return fmt.Errorf("UpdateRepositoryUpdatedTime: %w", err) } + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } + return nil } diff --git a/tests/integration/git_test.go b/tests/integration/git_test.go index cdff57f21f..d2bb383a71 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_test.go @@ -35,7 +35,7 @@ import ( ) const ( - littleSize = 1024 // 1ko + littleSize = 1024 * 10 // 1ko bigSize = 128 * 1024 * 1024 // 128Mo ) @@ -60,7 +60,8 @@ func testGit(t *testing.T, u *url.URL) { dstPath := t.TempDir() - dstForkedPath := t.TempDir() + // dstForkedPath := t.TempDir() + dstForkedPath4Reduce := t.TempDir() t.Run("CreateRepoInDifferentUser", doAPICreateRepository(forkedUserCtx, false)) t.Run("AddUserAsCollaborator", doAPIAddCollaborator(forkedUserCtx, httpContext.Username, perm.AccessModeRead)) @@ -104,6 +105,38 @@ func testGit(t *testing.T, u *url.URL) { defer tests.PrintCurrentTest(t)() // TODO doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") }) + + t.Run("ReduceRepoSize", func(t *testing.T) { + defer tests.PrintCurrentTest(t)() + u.Path = forkedUserCtx.GitPath() + u.User = url.UserPassword(forkedUserCtx.Username, userPassword) + + t.Run("Clone", doGitClone(dstForkedPath4Reduce, u)) + fmt.Fprintf(os.Stdout, "dstForkedPath4Reduce = %s\n", dstForkedPath4Reduce) + doCommitAndPush(t, littleSize, dstForkedPath4Reduce, "data-file-") + bigOne := doCommitAndPush(t, bigSize, dstForkedPath4Reduce, "my-file-") + repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, forkedUserCtx.Username, forkedUserCtx.Reponame) + assert.NoError(t, err) + orgGitRepoSize := repo.Size //doGetCountObjects(t, dstForkedPath4Reduce) + fmt.Fprintf(os.Stdout, "Original GitSize = %d\n", orgGitRepoSize) + + t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, bigSize/2)) + fmt.Fprintf(os.Stdout, "Original GitSize = %d\n", orgGitRepoSize) + + //Delete Object from Oversized Repository, This should be Accepdted + doDeleteObjectAndClean(t, dstForkedPath4Reduce, bigOne) + + repo, err = repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, forkedUserCtx.Username, forkedUserCtx.Reponame) + assert.NoError(t, err) + reducedGitRepoSize := repo.Size //doGetCountObjects(t, dstForkedPath4Reduce) + + fmt.Fprintf(os.Stdout, "Reduced GitSize = %d\n", reducedGitRepoSize) + + assert.Less(t, reducedGitRepoSize, orgGitRepoSize, "Repo size is not reduced.") + + setting.SaveGlobalRepositorySetting(false, 0) + }) + // TODO delete branch // TODO delete tag // TODO add big commit that will be over with the push @@ -325,6 +358,20 @@ func lockFileTest(t *testing.T, filename, repoPath string) { assert.NoError(t, err) } +func doDeleteObjectAndClean(t *testing.T, repoPath, filename string) { + var err error + err = deleteCommit(repoPath, "user4@example.com", "User Four", filename) + assert.NoError(t, err) + _, _, err = git.NewCommand(git.DefaultContext, "push", "origin", "master").RunStdString(&git.RunOpts{Dir: repoPath}) // Push + assert.NoError(t, err) + + // _, _, err = git.NewCommand(git.DefaultContext, "reflog", "expire", "--expire-unreachable=all", "--all").RunStdString(&git.RunOpts{Dir: repoPath}) // Push + // assert.NoError(t, err) + // _, _, err = git.NewCommand(git.DefaultContext, "gc", "--prune=now").RunStdString(&git.RunOpts{Dir: repoPath}) // Push + // assert.NoError(t, err) + +} + func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string { name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix) assert.NoError(t, err) @@ -341,6 +388,34 @@ func doCommitAndPushWithExpectedError(t *testing.T, size int, repoPath, prefix s return name } +func deleteCommit(repoPath, email, fullName, filename string) error { + + var err error + globalArgs := git.AllowLFSFiltersArgs() + + cmd := git.NewCommand(git.DefaultContext, "rm").AddDashesAndList(filename) + + _, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoPath}) // Push + if err != nil { + return err + } + + return git.CommitChangesWithArgs(repoPath, globalArgs, git.CommitChangesOptions{ + Committer: &git.Signature{ + Email: email, + Name: fullName, + When: time.Now(), + }, + Author: &git.Signature{ + Email: email, + Name: fullName, + When: time.Now(), + }, + Message: fmt.Sprintf("Testing commit @ %v", time.Now()), + }) + +} + func generateCommitWithNewData(size int, repoPath, email, fullName, prefix string) (string, error) { // Generate random file bufSize := 4 * 1024 From ed4865ceedf617fba7ab03e6d8cd852d3cd5f094 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Tue, 23 May 2023 04:23:59 +0000 Subject: [PATCH 38/55] Refactor hook_pre_recieve to increase speed of size checking and minor gofumpt/lint fixes --- routers/private/hook_pre_receive.go | 151 ++++++++++++++++------------ 1 file changed, 84 insertions(+), 67 deletions(-) diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index ddde140d17..aaa8d275fc 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -9,6 +9,7 @@ import ( "os" "strconv" "strings" + "time" "code.gitea.io/gitea/models" asymkey_model "code.gitea.io/gitea/models/asymkey" @@ -104,70 +105,70 @@ func (ctx *preReceiveContext) AssertCreatePullRequest() bool { return true } -// CalculateSizeOfAddedObjects calculates the total size of objects provided as output from rev-list command -func CalculateSizeOfAddedObjects(ctx *gitea_context.PrivateContext, opts *git.RunOpts, revlistObjects string) int64 { - // Calculate the size of added Objects. - var totalSize int64 - for _, object := range strings.Split(revlistObjects, "\n") { - if len(object) == 0 { - continue - } - objectID := strings.Split(object, " ")[0] - objectSizeStr, _, err := git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(objectID).RunStdString(opts) - if err != nil { - log.Trace("CalculateSizeOfAddedObjects: Error during git cat-file -s on object %s", objectID) - return totalSize - } - objectSize, _ := strconv.ParseInt(strings.TrimSpace(objectSizeStr), 10, 64) - if err != nil { - log.Trace("CalculateSizeOfAddedObjects: Error during ParseInt on string: '%s'", objectSizeStr) - return totalSize - } - totalSize += objectSize +// CalculateSizeOfObject calculates the size of one git object via git cat-file -s command +func CalculateSizeOfObject(ctx *gitea_context.PrivateContext, opts *git.RunOpts, objectID string) (objectSize int64) { + objectSizeStr, _, err := git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(objectID).RunStdString(opts) + if err != nil { + log.Trace("CalculateSizeOfRemovedObjects: Error during git cat-file -s on object: %s", objectID) + return } - return totalSize + + objectSize, _ = strconv.ParseInt(strings.TrimSpace(objectSizeStr), 10, 64) + if err != nil { + log.Trace("CalculateSizeOfRemovedObjects: Error during ParseInt on string '%s'", objectID) + return + } + return } -// CalculateSizeOfRemovedObjects calculates the size of removed objects provided as output from rev-list command -// and confirms that the object is not referenced anywhere -func CalculateSizeOfRemovedObjects(ctx *gitea_context.PrivateContext, opts *git.RunOpts, revlistObjects string) int64 { - var totalSize int64 - for _, object := range strings.Split(revlistObjects, "\n") { +// CalculateSizeOfObjects calculates the size of objects added and removed from the repository by new commit +func CalculateSizeOfObjects(ctx *gitea_context.PrivateContext, opts *git.RunOpts, newCommitObjects, oldCommitObjects, otherCommitObjects map[string]bool) (addedSize, removedSize int64) { + // Calculate size of objects that were added + for objectID := range newCommitObjects { + if _, exists := oldCommitObjects[objectID]; !exists { + // objectID is not referenced in the list of objects of old commit so it is a new object + // Calculate its size and add it to the addedSize + addedSize += CalculateSizeOfObject(ctx, opts, objectID) + } + // We might check here if new object is not already in the rest of repo to be precise + // However our goal is to prevent growth of repository so on determination of addedSize + // We can skip this preciseness, addedSize will be more then real addedSize + // TODO - do not count size of object that is referenced in other part of repo but not referenced neither in old nor new commit + // git will not add the object twice + } + + // Calculate size of objects that were removed + for objectID := range oldCommitObjects { + if _, exists := newCommitObjects[objectID]; !exists { + // objectID is not referenced in the list of new commit objects so it was possibly removed + if _, exists := otherCommitObjects[objectID]; !exists { + // objectID is not referenced in rest of the objects of the repository so it was removed + // Calculate its size and add it to the addedSize + removedSize += CalculateSizeOfObject(ctx, opts, objectID) + } + } + } + return +} + +// ConvertObjectsToMap takes a newline-separated string of git objects and +// converts it into a map for efficient lookup. +func ConvertObjectsToMap(objects string) map[string]bool { + objectsMap := make(map[string]bool) + for _, object := range strings.Split(objects, "\n") { if len(object) == 0 { continue } objectID := strings.Split(object, " ")[0] - - // Confirm that the object is still reachable from anywhere in the repository. - isReachable, _, err := git.NewCommand(ctx, "rev-list", "--objects", "--all", "--").AddDynamicArguments(objectID).RunStdString(opts) - if err != nil { - log.Trace("CalculateSizeOfRemovedObjects: Error during git rev-list --objects --all on object: %s", objectID) - return totalSize - } - - if isReachable != "" { - // The object is still reachable, therefore we don't add it's size - continue - } - - objectSizeStr, _, err := git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(objectID).RunStdString(opts) - if err != nil { - log.Trace("CalculateSizeOfRemovedObjects: Error during git cat-file -s on object: %s", objectID) - return totalSize - } - - objectSize, _ := strconv.ParseInt(strings.TrimSpace(objectSizeStr), 10, 64) - if err != nil { - log.Trace("CalculateSizeOfRemovedObjects: Error during ParseInt on string '%s'", objectID) - return totalSize - } - totalSize += objectSize + objectsMap[objectID] = true } - return totalSize + return objectsMap } // HookPreReceive checks whether a individual commit is acceptable func HookPreReceive(ctx *gitea_context.PrivateContext) { + startTime := time.Now() + opts := web.GetForm(ctx).(*private.HookOptions) ourCtx := &preReceiveContext{ @@ -178,8 +179,8 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { repo := ourCtx.Repo.Repository - var removedSize int64 var addedSize int64 + var removedSize int64 // Calculating total size of the push using git count-objects pushSize, err := git.CountObjectsWithEnv(ctx, repo.RepoPath(), ourCtx.env) @@ -191,7 +192,7 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { return } - // Cash whether the repository would breach the size limit after the operation + // Cache whether the repository would breach the size limit after the operation isRepoOversized := repo.RepoSizeIsOversized(pushSize.Size) log.Trace("Push size %d", pushSize.Size) @@ -203,28 +204,41 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { // If operation is in potential breach of size limit prepare data for analysis if isRepoOversized { - - // Objects that are in newCommitID but not in oldCommitID are added - addedObjects, _, err := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(newCommitID, "^"+oldCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) + // Create cache of objects in old commit + gitObjects, _, err := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(oldCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) if err != nil { - log.Error("Unable to list objects in %s and not in %s in %-v Error: %v", newCommitID, oldCommitID, repo, err) + log.Error("Unable to list objects in old commit: %s in %-v Error: %v", oldCommitID, repo, err) ctx.JSON(http.StatusInternalServerError, private.Response{ - Err: fmt.Sprintf("Fail to list objects added: %v", err), + Err: fmt.Sprintf("Fail to list objects in old commit: %v", err), }) return } - addedSize += CalculateSizeOfAddedObjects(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}, addedObjects) + oldCommitObjects := ConvertObjectsToMap(gitObjects) - // Objects that are in oldCommitID but not in newCommitID are removed - removedObjects, _, err := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(oldCommitID, "^"+newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) + // Create cache of objects in new commit + gitObjects, _, err = git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) if err != nil { - log.Error("Unable to list objects in %s and not in %s in %-v Error: %v", oldCommitID, newCommitID, repo, err) + log.Error("Unable to list objects in new commit %s in %-v Error: %v", newCommitID, repo, err) ctx.JSON(http.StatusInternalServerError, private.Response{ - Err: fmt.Sprintf("Fail to list objects removed: %v", err), + Err: fmt.Sprintf("Fail to list objects in new commit: %v", err), }) return } - removedSize += CalculateSizeOfRemovedObjects(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}, removedObjects) + newCommitObjects := ConvertObjectsToMap(gitObjects) + + // Create cache of objects that are in the repository but not part of old or new commit + gitObjects, _, err = git.NewCommand(ctx, "rev-list", "--objects", "--all").AddDynamicArguments("^"+oldCommitID, "^"+newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) + if err != nil { + log.Error("Unable to list objects in the repo that are missing from both old %s and new %s commits in %-v Error: %v", oldCommitID, newCommitID, repo, err) + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: fmt.Sprintf("Fail to list objects missing from both old and new commits: %v", err), + }) + return + } + otherCommitObjects := ConvertObjectsToMap(gitObjects) + + // Calculate size that was added and removed by the new commit + addedSize, removedSize = CalculateSizeOfObjects(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}, newCommitObjects, oldCommitObjects, otherCommitObjects) } switch { @@ -242,9 +256,12 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { } } + duration := time.Since(startTime) + log.Trace("During size checking - Addition in size is: %d, removal in size is: %d, limit size: %s, push size: %d. Took %s seconds.", addedSize, removedSize, base.FileSize(repo.GetActualSizeLimit()), pushSize.Size, duration) + // If total of commits add more size then they remove and we are in a potential breach of size limit -- abort - if (addedSize > removedSize) && isRepoOversized { // Check next size if we are not deleting a reference - log.Warn("Forbidden: new repo size is over limitation: %s", base.FileSize(repo.GetActualSizeLimit())) + if (addedSize > removedSize) && isRepoOversized { + log.Warn("Forbidden: new repo size %s is over limitation of %s. Push size: %s. Took %s seconds.", base.FileSize(addedSize-removedSize), base.FileSize(repo.GetActualSizeLimit()), base.FileSize(pushSize.Size), duration) ctx.JSON(http.StatusForbidden, private.Response{ UserMsg: fmt.Sprintf("Repository size is over limitation of %s", base.FileSize(repo.GetActualSizeLimit())), }) From 256173fc2580bea7c96b225f567d7dfa03d96652 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Tue, 23 May 2023 06:42:42 +0000 Subject: [PATCH 39/55] Fixed lint errors in hook_pre_recieve.go --- routers/private/hook_pre_receive.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index aaa8d275fc..e9fbbc75d3 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -110,15 +110,15 @@ func CalculateSizeOfObject(ctx *gitea_context.PrivateContext, opts *git.RunOpts, objectSizeStr, _, err := git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(objectID).RunStdString(opts) if err != nil { log.Trace("CalculateSizeOfRemovedObjects: Error during git cat-file -s on object: %s", objectID) - return + return objectSize } objectSize, _ = strconv.ParseInt(strings.TrimSpace(objectSizeStr), 10, 64) if err != nil { log.Trace("CalculateSizeOfRemovedObjects: Error during ParseInt on string '%s'", objectID) - return + return objectSize } - return + return objectSize } // CalculateSizeOfObjects calculates the size of objects added and removed from the repository by new commit @@ -148,7 +148,7 @@ func CalculateSizeOfObjects(ctx *gitea_context.PrivateContext, opts *git.RunOpts } } } - return + return addedSize, removedSize } // ConvertObjectsToMap takes a newline-separated string of git objects and From 7bc95545ec575cacf7dc51db6af964f28facdca3 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Tue, 23 May 2023 15:04:44 +0000 Subject: [PATCH 40/55] The size unit of measurements should be mentioned in the config and documentation file --- custom/conf/app.example.ini | 3 ++- docs/content/doc/administration/config-cheat-sheet.en-us.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini index b3d2117b4b..fb6cee734d 100644 --- a/custom/conf/app.example.ini +++ b/custom/conf/app.example.ini @@ -943,8 +943,9 @@ LEVEL = Info ;; "false" means no limit will be enforced, even if specified on a repository ;ENABLE_SIZE_LIMIT = false ; -;; Specify a global repository size limit to apply for each repository. 0 - No limit +;; Specify a global repository size limit in bytes to apply for each repository. 0 - No limit ;; If repository has it's own limit set in UI it will override the global setting +;; Standard units of measurements for size can be used like B, KB, KiB, ... , EB, EiB, ... ;REPO_SIZE_LIMIT = 0 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; diff --git a/docs/content/doc/administration/config-cheat-sheet.en-us.md b/docs/content/doc/administration/config-cheat-sheet.en-us.md index f7b678ec46..a74195d98a 100644 --- a/docs/content/doc/administration/config-cheat-sheet.en-us.md +++ b/docs/content/doc/administration/config-cheat-sheet.en-us.md @@ -116,7 +116,7 @@ In addition there is _`StaticRootPath`_ which can be set as a built-in at build - `ALLOW_FORK_WITHOUT_MAXIMUM_LIMIT`: **true**: Allow fork repositories without maximum number limit - `ENABLE_SIZE_LIMIT`: **false**: Enable global per repository size limit defined in `REPO_SIZE_LIMIT` or at the repository itself. -- `REPO_SIZE_LIMIT`: **0**: The global default size limit to apply per repository in bytes when `ENABLE_SIZE_LIMIT` is `true`. Value `0` - no limit. The repository limit set via user interface by admin takes precedence over this value. +- `REPO_SIZE_LIMIT`: **0**: The global default size limit to apply per repository in bytes when `ENABLE_SIZE_LIMIT` is `true`. Value `0` - no limit. The repository limit set via user interface by admin takes precedence over this value. Standard units of measurements can be used with this number like - `B`, `KB`, `KiB`, `MB`, `MiB`, .... , `EB`, `EiB`, ... ### Repository - Editor (`repository.editor`) From 22bb76c9c646b3082031d8f5ee6b94a68a7b7e72 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Wed, 24 May 2023 03:09:32 +0000 Subject: [PATCH 41/55] Revert "Update for TestScript to Reduce RepoSize" This reverts commit 08c65ffb1725d9fad8421705245270f8ceb87a4e. --- modules/repository/create.go | 20 -------- routers/private/hook_pre_receive.go | 2 +- services/repository/push.go | 10 ++-- tests/integration/git_test.go | 79 +---------------------------- 4 files changed, 6 insertions(+), 105 deletions(-) diff --git a/modules/repository/create.go b/modules/repository/create.go index 8240600be1..63be5cac25 100644 --- a/modules/repository/create.go +++ b/modules/repository/create.go @@ -332,26 +332,6 @@ func UpdateRepoSize(ctx context.Context, repo *repo_model.Repository) error { return fmt.Errorf("updateSize: GetLFSMetaObjects: %w", err) } - if setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 && size+lfsSize > repo.GetActualSizeLimit() { - // return fmt.Errorf("updateSize: Git Reflog Failed Size(%d) > Limit(%d)", size+lfsSize, repo.GetActualSizeLimit()) - - _, _, err = git.NewCommand(git.DefaultContext, "reflog", "expire", "--expire-unreachable=all", "--all").RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) // Push - if err != nil { - return fmt.Errorf("updateSize: Git Reflog Failed: %w", err) - } - - _, _, err = git.NewCommand(git.DefaultContext, "gc", "--prune=now").RunStdString(&git.RunOpts{Dir: repo.RepoPath()}) // Push - if err != nil { - return fmt.Errorf("updateSize: Git GC Failed: %w", err) - } - - size, err = getDirectorySize(repo.RepoPath()) - if err != nil { - return fmt.Errorf("updateSize: %w", err) - } - // return fmt.Errorf("updateSize: Git Reflog Failed Size(%d) > Limit(%d)", size+lfsSize, repo.GetActualSizeLimit()) - } - return repo_model.UpdateRepoSize(ctx, repo.ID, size+lfsSize) } diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index aa44ca7b35..e9fbbc75d3 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -263,7 +263,7 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { if (addedSize > removedSize) && isRepoOversized { log.Warn("Forbidden: new repo size %s is over limitation of %s. Push size: %s. Took %s seconds.", base.FileSize(addedSize-removedSize), base.FileSize(repo.GetActualSizeLimit()), base.FileSize(pushSize.Size), duration) ctx.JSON(http.StatusForbidden, private.Response{ - UserMsg: fmt.Sprintf("Repository size is over limitation of %s addedSize(%d) removedSize(%d) repo(%s)", base.FileSize(repo.GetActualSizeLimit()), addedSize, removedSize, repo.RepoPath()), + UserMsg: fmt.Sprintf("Repository size is over limitation of %s", base.FileSize(repo.GetActualSizeLimit())), }) return } diff --git a/services/repository/push.go b/services/repository/push.go index a6ba284bc7..c7ea8f336e 100644 --- a/services/repository/push.go +++ b/services/repository/push.go @@ -93,9 +93,9 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { } defer gitRepo.Close() - // if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { - // log.Error("Failed to update size for repository: %v", err) - // } + if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { + log.Error("Failed to update size for repository: %v", err) + } addTags := make([]string, 0, len(optsList)) delTags := make([]string, 0, len(optsList)) @@ -293,10 +293,6 @@ func pushUpdates(optsList []*repo_module.PushUpdateOptions) error { return fmt.Errorf("UpdateRepositoryUpdatedTime: %w", err) } - if err = repo_module.UpdateRepoSize(ctx, repo); err != nil { - log.Error("Failed to update size for repository: %v", err) - } - return nil } diff --git a/tests/integration/git_test.go b/tests/integration/git_test.go index d2bb383a71..cdff57f21f 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_test.go @@ -35,7 +35,7 @@ import ( ) const ( - littleSize = 1024 * 10 // 1ko + littleSize = 1024 // 1ko bigSize = 128 * 1024 * 1024 // 128Mo ) @@ -60,8 +60,7 @@ func testGit(t *testing.T, u *url.URL) { dstPath := t.TempDir() - // dstForkedPath := t.TempDir() - dstForkedPath4Reduce := t.TempDir() + dstForkedPath := t.TempDir() t.Run("CreateRepoInDifferentUser", doAPICreateRepository(forkedUserCtx, false)) t.Run("AddUserAsCollaborator", doAPIAddCollaborator(forkedUserCtx, httpContext.Username, perm.AccessModeRead)) @@ -105,38 +104,6 @@ func testGit(t *testing.T, u *url.URL) { defer tests.PrintCurrentTest(t)() // TODO doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") }) - - t.Run("ReduceRepoSize", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - u.Path = forkedUserCtx.GitPath() - u.User = url.UserPassword(forkedUserCtx.Username, userPassword) - - t.Run("Clone", doGitClone(dstForkedPath4Reduce, u)) - fmt.Fprintf(os.Stdout, "dstForkedPath4Reduce = %s\n", dstForkedPath4Reduce) - doCommitAndPush(t, littleSize, dstForkedPath4Reduce, "data-file-") - bigOne := doCommitAndPush(t, bigSize, dstForkedPath4Reduce, "my-file-") - repo, err := repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, forkedUserCtx.Username, forkedUserCtx.Reponame) - assert.NoError(t, err) - orgGitRepoSize := repo.Size //doGetCountObjects(t, dstForkedPath4Reduce) - fmt.Fprintf(os.Stdout, "Original GitSize = %d\n", orgGitRepoSize) - - t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, bigSize/2)) - fmt.Fprintf(os.Stdout, "Original GitSize = %d\n", orgGitRepoSize) - - //Delete Object from Oversized Repository, This should be Accepdted - doDeleteObjectAndClean(t, dstForkedPath4Reduce, bigOne) - - repo, err = repo_model.GetRepositoryByOwnerAndName(db.DefaultContext, forkedUserCtx.Username, forkedUserCtx.Reponame) - assert.NoError(t, err) - reducedGitRepoSize := repo.Size //doGetCountObjects(t, dstForkedPath4Reduce) - - fmt.Fprintf(os.Stdout, "Reduced GitSize = %d\n", reducedGitRepoSize) - - assert.Less(t, reducedGitRepoSize, orgGitRepoSize, "Repo size is not reduced.") - - setting.SaveGlobalRepositorySetting(false, 0) - }) - // TODO delete branch // TODO delete tag // TODO add big commit that will be over with the push @@ -358,20 +325,6 @@ func lockFileTest(t *testing.T, filename, repoPath string) { assert.NoError(t, err) } -func doDeleteObjectAndClean(t *testing.T, repoPath, filename string) { - var err error - err = deleteCommit(repoPath, "user4@example.com", "User Four", filename) - assert.NoError(t, err) - _, _, err = git.NewCommand(git.DefaultContext, "push", "origin", "master").RunStdString(&git.RunOpts{Dir: repoPath}) // Push - assert.NoError(t, err) - - // _, _, err = git.NewCommand(git.DefaultContext, "reflog", "expire", "--expire-unreachable=all", "--all").RunStdString(&git.RunOpts{Dir: repoPath}) // Push - // assert.NoError(t, err) - // _, _, err = git.NewCommand(git.DefaultContext, "gc", "--prune=now").RunStdString(&git.RunOpts{Dir: repoPath}) // Push - // assert.NoError(t, err) - -} - func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string { name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix) assert.NoError(t, err) @@ -388,34 +341,6 @@ func doCommitAndPushWithExpectedError(t *testing.T, size int, repoPath, prefix s return name } -func deleteCommit(repoPath, email, fullName, filename string) error { - - var err error - globalArgs := git.AllowLFSFiltersArgs() - - cmd := git.NewCommand(git.DefaultContext, "rm").AddDashesAndList(filename) - - _, _, err = cmd.RunStdString(&git.RunOpts{Dir: repoPath}) // Push - if err != nil { - return err - } - - return git.CommitChangesWithArgs(repoPath, globalArgs, git.CommitChangesOptions{ - Committer: &git.Signature{ - Email: email, - Name: fullName, - When: time.Now(), - }, - Author: &git.Signature{ - Email: email, - Name: fullName, - When: time.Now(), - }, - Message: fmt.Sprintf("Testing commit @ %v", time.Now()), - }) - -} - func generateCommitWithNewData(size int, repoPath, email, fullName, prefix string) (string, error) { // Generate random file bufSize := 4 * 1024 From 64f6dd69675cc64f098bd5fdf737ae9a5058747b Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Wed, 24 May 2023 16:54:23 +0000 Subject: [PATCH 42/55] Documentation modification for the repo size limit feature Signed-off-by: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> --- .../administration/repo-size-limit.en-us.md | 54 +++++++++++++++++++ docs/content/page/index.en-us.md | 2 + 2 files changed, 56 insertions(+) create mode 100644 docs/content/doc/administration/repo-size-limit.en-us.md diff --git a/docs/content/doc/administration/repo-size-limit.en-us.md b/docs/content/doc/administration/repo-size-limit.en-us.md new file mode 100644 index 0000000000..09c378d335 --- /dev/null +++ b/docs/content/doc/administration/repo-size-limit.en-us.md @@ -0,0 +1,54 @@ +--- +date: "2023-05-24T13:00:00+00:00" +title: "Per repository size limit" +slug: "repo-size-limit" +weight: 12 +toc: false +draft: false +aliases: + - /en-us/repo-size-limit +menu: + sidebar: + parent: "administration" + name: "Per repository size limit" + weight: 12 + identifier: "repo-size-limit" +--- + +# Gitea per repository size limit setup + +To use Gitea's experimental built-in per repository size limit support, Administrator must update the `app.ini` file: + +```ini +;; Enable applying a global size limit defined by REPO_SIZE_LIMIT. Each repository can have a value that overrides the global limit +;; "false" means no limit will be enforced, even if specified on a repository +ENABLE_SIZE_LIMIT = true + +;; Specify a global repository size limit in bytes to apply for each repository. 0 - No limit +;; If repository has it's own limit set in UI it will override the global setting +;; Standard units of measurements for size can be used like B, KB, KiB, ... , EB, EiB, ... +REPO_SIZE_LIMIT = 500 MB +``` +This setting is persistent. + + +The size limitation is triggered when repository `disk size` + `new commit size` > `defined repository size limit` + +If size limitation is triggered the feature would prevent commits that increase repository size on disk +of gitea server and allow those that decrease it + + +# Gitea per repository size limit setup in UI + +1. For Gitea admin it is possible during runtime to enable/disable limit size feature, change the global size limit on the fly. +**This setting is not persistent across restarts** + +`Admin panel/Site settings` -> `Repository management` + +Persistance can be achieved if the limit is maintained by editing `app.ini` file + +2. The individually set per repository limit in `Settings` of the +repository would take precedence over global limit when the size limit +feature is enabled. Only admin can modify those limits + +**Note**: Size checking for large repositories is time consuming operation so time of push under size limit might increase up to a minute depending on your server hardware diff --git a/docs/content/page/index.en-us.md b/docs/content/page/index.en-us.md index 93fbc0a8b1..dc450b64c9 100644 --- a/docs/content/page/index.en-us.md +++ b/docs/content/page/index.en-us.md @@ -112,6 +112,7 @@ You can try it out using [the online demo](https://try.gitea.io/). - Hooks - Repository management - See all repository information and manage repositories + - Set per repository global size limit, disable/enable the repository limit (runtime only), permanent should be done via Configuration file - Authentication sources - OAuth - PAM @@ -170,6 +171,7 @@ You can try it out using [the online demo](https://try.gitea.io/). - Git LFS - Watch, Star, Fork - View watchers, stars, and forks + - Size limit (excl. LFS) - Code - Branch browser - Web based file upload and creation From 20d4d4f0a47197a95ce59bddf3697a11a5b6c8ed Mon Sep 17 00:00:00 2001 From: truecode112 Date: Thu, 25 May 2023 15:55:31 +0300 Subject: [PATCH 43/55] test script to check if repo size is reduced. --- .../administration/repo-size-limit.en-us.md | 4 +- tests/integration/git_test.go | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/docs/content/doc/administration/repo-size-limit.en-us.md b/docs/content/doc/administration/repo-size-limit.en-us.md index 09c378d335..7e35531294 100644 --- a/docs/content/doc/administration/repo-size-limit.en-us.md +++ b/docs/content/doc/administration/repo-size-limit.en-us.md @@ -28,16 +28,14 @@ ENABLE_SIZE_LIMIT = true ;; If repository has it's own limit set in UI it will override the global setting ;; Standard units of measurements for size can be used like B, KB, KiB, ... , EB, EiB, ... REPO_SIZE_LIMIT = 500 MB -``` -This setting is persistent. +This setting is persistent. The size limitation is triggered when repository `disk size` + `new commit size` > `defined repository size limit` If size limitation is triggered the feature would prevent commits that increase repository size on disk of gitea server and allow those that decrease it - # Gitea per repository size limit setup in UI 1. For Gitea admin it is possible during runtime to enable/disable limit size feature, change the global size limit on the fly. diff --git a/tests/integration/git_test.go b/tests/integration/git_test.go index cdff57f21f..7369f2694b 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_test.go @@ -102,6 +102,14 @@ func testGit(t *testing.T, u *url.URL) { }) t.Run("Deletion", func(t *testing.T) { defer tests.PrintCurrentTest(t)() + doCommitAndPush(t, littleSize, dstPath, "data-file-") + + bigFileName := doCommitAndPush(t, bigSize, dstPath, "data-file-") + oldRepoSize := doCalculateRepoSize(t, dstPath) + doDeleteAndCliean(t, dstPath, bigFileName) + newRepoSize := doCalculateRepoSize(t, dstPath) + assert.Less(t, newRepoSize, oldRepoSize) + setting.SaveGlobalRepositorySetting(false, 0) // TODO doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") }) // TODO delete branch @@ -325,6 +333,42 @@ func lockFileTest(t *testing.T, filename, repoPath string) { assert.NoError(t, err) } +const notRegularFileMode = os.ModeSymlink | os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular + +func doCalculateRepoSize(t *testing.T, path string) int64 { + var size int64 + err := filepath.WalkDir(path, func(_ string, info os.DirEntry, err error) error { + if err != nil { + if os.IsNotExist(err) { // ignore the error because the file maybe deleted during traversing. + return nil + } + return err + } + if info.IsDir() { + return nil + } + f, err := info.Info() + if err != nil { + return err + } + if (f.Mode() & notRegularFileMode) == 0 { + size += f.Size() + } + return err + }) + assert.NoError(t, err) + return size +} + +func doDeleteAndCliean(t *testing.T, repoPath, filename string) { + _, _, err := git.NewCommand(git.DefaultContext, "rm").AddDashesAndList(filename).RunStdString(&git.RunOpts{Dir: repoPath}) // Delete + assert.NoError(t, err) + _, _, err = git.NewCommand(git.DefaultContext, "reflog", "expire", "--expire-unreachable=all", "--all").RunStdString(&git.RunOpts{Dir: repoPath}) // reflog + assert.NoError(t, err) + _, _, err = git.NewCommand(git.DefaultContext, "gc", "--prune=now").RunStdString(&git.RunOpts{Dir: repoPath}) // reflog + assert.NoError(t, err) +} + func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string { name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix) assert.NoError(t, err) From 0c3a46fd43a75811a392a752d40d2dc99b1cc901 Mon Sep 17 00:00:00 2001 From: truecode112 Date: Fri, 26 May 2023 16:53:03 +0300 Subject: [PATCH 44/55] checking reduced size scenario 2 cleanup unused code (save app.ini) --- modules/setting/repository.go | 4 --- tests/integration/git_test.go | 61 ++++++++++++++++++++++++++--------- 2 files changed, 46 insertions(+), 19 deletions(-) diff --git a/modules/setting/repository.go b/modules/setting/repository.go index 6223e51485..dcc996f7c7 100644 --- a/modules/setting/repository.go +++ b/modules/setting/repository.go @@ -284,10 +284,6 @@ func SaveGlobalRepositorySetting(enableSizeLimit bool, repoSizeLimit int64) erro } sec.Key("REPO_SIZE_LIMIT").SetValue(humanize.Bytes(uint64(RepoSizeLimit))) - // if err := CfgProvider.Save(); err != nil { - // log.Fatal("Failed to save config file: %v", err) - // return err - // } return nil } diff --git a/tests/integration/git_test.go b/tests/integration/git_test.go index 7369f2694b..5e18ec19d2 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_test.go @@ -13,6 +13,7 @@ import ( "path" "path/filepath" "strconv" + "strings" "testing" "time" @@ -103,21 +104,15 @@ func testGit(t *testing.T, u *url.URL) { t.Run("Deletion", func(t *testing.T) { defer tests.PrintCurrentTest(t)() doCommitAndPush(t, littleSize, dstPath, "data-file-") - bigFileName := doCommitAndPush(t, bigSize, dstPath, "data-file-") - oldRepoSize := doCalculateRepoSize(t, dstPath) - doDeleteAndCliean(t, dstPath, bigFileName) - newRepoSize := doCalculateRepoSize(t, dstPath) - assert.Less(t, newRepoSize, oldRepoSize) + oldRepoSize := doGetRemoteRepoSize(t, forkedUserCtx) + lastCommitID := doGetAddCommitID(t, dstPath, bigFileName) + doDeleteAndPush(t, dstPath, bigFileName) + doRebaseCommitAndPush(t, dstPath, lastCommitID) + newRepoSize := doGetRemoteRepoSize(t, forkedUserCtx) + assert.LessOrEqual(t, newRepoSize, oldRepoSize) setting.SaveGlobalRepositorySetting(false, 0) - // TODO doDeleteCommitAndPush(t, littleSize, dstPath, "data-file-") }) - // TODO delete branch - // TODO delete tag - // TODO add big commit that will be over with the push - // TODO add lfs - // TODO remove lfs - // TODO add missing case }) t.Run("CreateAgitFlowPull", doCreateAgitFlowPull(dstPath, &httpContext, "master", "test/head")) t.Run("BranchProtectMerge", doBranchProtectPRMerge(&httpContext, dstPath)) @@ -335,6 +330,11 @@ func lockFileTest(t *testing.T, filename, repoPath string) { const notRegularFileMode = os.ModeSymlink | os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular +func doGetRemoteRepoSize(t *testing.T, ctx APITestContext) int64 { + dirPath := filepath.Join(setting.RepoRootPath, strings.ToLower(ctx.Username), strings.ToLower(ctx.Reponame)+".git") + return doCalculateRepoSize(t, dirPath) +} + func doCalculateRepoSize(t *testing.T, path string) int64 { var size int64 err := filepath.WalkDir(path, func(_ string, info os.DirEntry, err error) error { @@ -360,12 +360,43 @@ func doCalculateRepoSize(t *testing.T, path string) int64 { return size } -func doDeleteAndCliean(t *testing.T, repoPath, filename string) { +func doDeleteAndPush(t *testing.T, repoPath, filename string) { _, _, err := git.NewCommand(git.DefaultContext, "rm").AddDashesAndList(filename).RunStdString(&git.RunOpts{Dir: repoPath}) // Delete assert.NoError(t, err) - _, _, err = git.NewCommand(git.DefaultContext, "reflog", "expire", "--expire-unreachable=all", "--all").RunStdString(&git.RunOpts{Dir: repoPath}) // reflog + signature := git.Signature{ + Email: "user2@example.com", + Name: "User Two", + When: time.Now(), + } + _, _, err = git.NewCommand(git.DefaultContext, "status").RunStdString(&git.RunOpts{Dir: repoPath}) assert.NoError(t, err) - _, _, err = git.NewCommand(git.DefaultContext, "gc", "--prune=now").RunStdString(&git.RunOpts{Dir: repoPath}) // reflog + err2 := git.CommitChanges(repoPath, git.CommitChangesOptions{ // Commit + Committer: &signature, + Author: &signature, + Message: "Delete Commit", + }) + assert.NoError(t, err2) + _, _, err = git.NewCommand(git.DefaultContext, "status").RunStdString(&git.RunOpts{Dir: repoPath}) + assert.NoError(t, err) + _, _, err = git.NewCommand(git.DefaultContext, "push", "origin", "master").RunStdString(&git.RunOpts{Dir: repoPath}) // Push + assert.NoError(t, err) +} + +func doGetAddCommitID(t *testing.T, repoPath, filename string) string { + output, _, err := git.NewCommand(git.DefaultContext, "log", "origin", "master").RunStdString(&git.RunOpts{Dir: repoPath}) // Push + assert.NoError(t, err) + list := strings.Fields(output) + assert.LessOrEqual(t, 2, len(list)) + return list[1] +} + +func doRebaseCommitAndPush(t *testing.T, repoPath, commitID string) { + command := git.NewCommand(git.DefaultContext, "rebase", "--interactive").AddDashesAndList(commitID) + env := os.Environ() + env = append(env, "GIT_SEQUENCE_EDITOR=true") + _, _, err := command.RunStdString(&git.RunOpts{Dir: repoPath, Env: env}) // Push + assert.NoError(t, err) + _, _, err = git.NewCommand(git.DefaultContext, "push", "origin", "master", "-f").RunStdString(&git.RunOpts{Dir: repoPath}) // Push assert.NoError(t, err) } From 851c9883868a1bee0947d80065b4ac20c2e219d1 Mon Sep 17 00:00:00 2001 From: truecode112 Date: Wed, 31 May 2023 13:28:57 +0300 Subject: [PATCH 45/55] Fixed getting repo size via api call. --- .../api_helper_for_declarative_test.go | 16 +++++++++ tests/integration/git_test.go | 36 +++---------------- 2 files changed, 20 insertions(+), 32 deletions(-) diff --git a/tests/integration/api_helper_for_declarative_test.go b/tests/integration/api_helper_for_declarative_test.go index a8f42cdd9d..d3fefafd30 100644 --- a/tests/integration/api_helper_for_declarative_test.go +++ b/tests/integration/api_helper_for_declarative_test.go @@ -129,6 +129,22 @@ func doAPIForkRepository(ctx APITestContext, username string, callback ...func(* } } +func doAPIGetRepositorySize(ctx APITestContext, owner, repo string) func(*testing.T) int64 { + return func(t *testing.T) int64 { + urlStr := fmt.Sprintf("/api/v1/repos/%s/%s?token=%s", ctx.Username, ctx.Reponame, ctx.Token) + + req := NewRequest(t, "GET", urlStr) + if ctx.ExpectedCode != 0 { + ctx.Session.MakeRequest(t, req, ctx.ExpectedCode) + } + resp := ctx.Session.MakeRequest(t, req, http.StatusOK) + + var repository api.Repository + DecodeJSON(t, resp, &repository) + return int64(repository.Size) + } +} + func doAPIGetRepository(ctx APITestContext, callback ...func(*testing.T, api.Repository)) func(*testing.T) { return func(t *testing.T) { urlStr := fmt.Sprintf("/api/v1/repos/%s/%s?token=%s", ctx.Username, ctx.Reponame, ctx.Token) diff --git a/tests/integration/git_test.go b/tests/integration/git_test.go index 5e18ec19d2..7f42d927b8 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_test.go @@ -105,11 +105,11 @@ func testGit(t *testing.T, u *url.URL) { defer tests.PrintCurrentTest(t)() doCommitAndPush(t, littleSize, dstPath, "data-file-") bigFileName := doCommitAndPush(t, bigSize, dstPath, "data-file-") - oldRepoSize := doGetRemoteRepoSize(t, forkedUserCtx) + oldRepoSize := doGetRemoteRepoSizeViaAPI(t, forkedUserCtx) lastCommitID := doGetAddCommitID(t, dstPath, bigFileName) doDeleteAndPush(t, dstPath, bigFileName) doRebaseCommitAndPush(t, dstPath, lastCommitID) - newRepoSize := doGetRemoteRepoSize(t, forkedUserCtx) + newRepoSize := doGetRemoteRepoSizeViaAPI(t, forkedUserCtx) assert.LessOrEqual(t, newRepoSize, oldRepoSize) setting.SaveGlobalRepositorySetting(false, 0) }) @@ -328,36 +328,8 @@ func lockFileTest(t *testing.T, filename, repoPath string) { assert.NoError(t, err) } -const notRegularFileMode = os.ModeSymlink | os.ModeNamedPipe | os.ModeSocket | os.ModeDevice | os.ModeCharDevice | os.ModeIrregular - -func doGetRemoteRepoSize(t *testing.T, ctx APITestContext) int64 { - dirPath := filepath.Join(setting.RepoRootPath, strings.ToLower(ctx.Username), strings.ToLower(ctx.Reponame)+".git") - return doCalculateRepoSize(t, dirPath) -} - -func doCalculateRepoSize(t *testing.T, path string) int64 { - var size int64 - err := filepath.WalkDir(path, func(_ string, info os.DirEntry, err error) error { - if err != nil { - if os.IsNotExist(err) { // ignore the error because the file maybe deleted during traversing. - return nil - } - return err - } - if info.IsDir() { - return nil - } - f, err := info.Info() - if err != nil { - return err - } - if (f.Mode() & notRegularFileMode) == 0 { - size += f.Size() - } - return err - }) - assert.NoError(t, err) - return size +func doGetRemoteRepoSizeViaAPI(t *testing.T, ctx APITestContext) int64 { + return doAPIGetRepositorySize(ctx, ctx.Username, ctx.Reponame)(t) } func doDeleteAndPush(t *testing.T, repoPath, filename string) { From 649da240b260c58a4a20fbf71bcaf17cdaeb62b5 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Wed, 7 Jun 2023 17:24:52 +0000 Subject: [PATCH 46/55] Removing unneeded comments --- models/repo/repo.go | 38 +++----------------------------------- modules/base/tool.go | 3 --- 2 files changed, 3 insertions(+), 38 deletions(-) diff --git a/models/repo/repo.go b/models/repo/repo.go index 95e9684dfb..e528afca04 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -496,40 +496,8 @@ func (repo *Repository) IsOwnedBy(userID int64) bool { return repo.OwnerID == userID } -// TODO - DmitryFrolovTri Review every commented function and see if we need to reinstate it -// func (repo *Repository) computeSize() (int64, error) { -// size, err := util.GetDirectorySize(repo.RepoPath()) -// if err != nil { -// return 0, fmt.Errorf("computeSize: %v", err) -// } - -// objs, err := repo.GetLFSMetaObjects(repo.ID, -1, 0) -// if err != nil { -// return 0, fmt.Errorf("computeSize: GetLFSMetaObjects: %v", err) -// } -// for _, obj := range objs { -// size += obj.Size -// } - -// return size, nil -// } - -// func (repo *Repository) updateSize(ctx context.Context Engine) error { -// size, err := repo.computeSize() -// if err != nil { -// return fmt.Errorf("updateSize: %v", err) -// } - -// repo.Size = size -// _, err = e.ID(repo.ID).Cols("size").Update(repo) -// return err -// } - -// // UpdateSize updates the repository size, calculating it using util.GetDirectorySize -// func (repo *Repository) UpdateSize(ctx DBContext) error { -// return repo.updateSize(ctx.e) -// } - +// GetActualSizeLimit returns repository size limit in bytes +// or global repository limit setting if per repository size limit is not set func (repo *Repository) GetActualSizeLimit() int64 { sizeLimit := repo.SizeLimit if setting.RepoSizeLimit > 0 && sizeLimit == 0 { @@ -538,7 +506,7 @@ func (repo *Repository) GetActualSizeLimit() int64 { return sizeLimit } -// RepoSizeIsOversized return if is over size limitation +// RepoSizeIsOversized return true if is over size limitation func (repo *Repository) RepoSizeIsOversized(additionalSize int64) bool { return setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 && repo.Size+additionalSize > repo.GetActualSizeLimit() } diff --git a/modules/base/tool.go b/modules/base/tool.go index 5fe486b9b9..a6579ab515 100644 --- a/modules/base/tool.go +++ b/modules/base/tool.go @@ -138,9 +138,6 @@ func CreateTimeLimitCode(data string, minutes int, startInf interface{}) string // FileSize calculates the file size and generate user-friendly string. func FileSize(s int64) string { - // sizes := []string{"B", "kB", "MB", "GB", "TB", "PB", "EB"} - // return humanize.humanateBytes(uint64(s), 1000, sizes) - // return humanize.Bytes(uint64(s)) return humanize.IBytes(uint64(s)) } From 9c9f75fb413a777ce7cbeb26516c59f3fab0b7fd Mon Sep 17 00:00:00 2001 From: Dmitry Frolov <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Mon, 12 Jun 2023 21:24:58 +0000 Subject: [PATCH 47/55] Limit repo size packs (#1) Add usage of `git verify-pack -v` and `git batch-check` in calculating repo and push sizes during pre-receive hook. --- models/repo/repo.go | 8 +- routers/private/hook_pre_receive.go | 364 +++++++++++++++++++++++++--- routers/web/repo/setting.go | 4 +- 3 files changed, 336 insertions(+), 40 deletions(-) diff --git a/models/repo/repo.go b/models/repo/repo.go index e528afca04..4ba84cd260 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -507,10 +507,16 @@ func (repo *Repository) GetActualSizeLimit() int64 { } // RepoSizeIsOversized return true if is over size limitation -func (repo *Repository) RepoSizeIsOversized(additionalSize int64) bool { +func (repo *Repository) IsRepoSizeOversized(additionalSize int64) bool { return setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 && repo.Size+additionalSize > repo.GetActualSizeLimit() } +// RepoSizeLimitEnabled return true if size limit checking is enabled and limit is non zero for this specific repository +// this is used to enable size checking during pre-receive hook +func (repo *Repository) IsRepoSizeLimitEnabled() bool { + return setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 +} + // CanCreateBranch returns true if repository meets the requirements for creating new branches. func (repo *Repository) CanCreateBranch() bool { return !repo.IsMirror diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index 03c96b6d63..88cb1c4867 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -7,8 +7,10 @@ import ( "fmt" "net/http" "os" + "path/filepath" "strconv" "strings" + "sync" "time" "code.gitea.io/gitea/models" @@ -105,8 +107,8 @@ func (ctx *preReceiveContext) AssertCreatePullRequest() bool { return true } -// CalculateSizeOfObject calculates the size of one git object via git cat-file -s command -func CalculateSizeOfObject(ctx *gitea_context.PrivateContext, opts *git.RunOpts, objectID string) (objectSize int64) { +// calculateSizeOfObject calculates the size of one git object via git cat-file -s command +func calculateSizeOfObject(ctx *gitea_context.PrivateContext, opts *git.RunOpts, objectID string) (objectSize int64) { objectSizeStr, _, err := git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(objectID).RunStdString(opts) if err != nil { log.Trace("CalculateSizeOfRemovedObjects: Error during git cat-file -s on object: %s", objectID) @@ -121,14 +123,15 @@ func CalculateSizeOfObject(ctx *gitea_context.PrivateContext, opts *git.RunOpts, return objectSize } -// CalculateSizeOfObjects calculates the size of objects added and removed from the repository by new commit -func CalculateSizeOfObjects(ctx *gitea_context.PrivateContext, opts *git.RunOpts, newCommitObjects, oldCommitObjects, otherCommitObjects map[string]bool) (addedSize, removedSize int64) { +// calculateSizeOfObjectsFromCache calculates the size of objects added and removed from the repository by new push +// it uses data that was cached about the repository for this run +func calculateSizeOfObjectsFromCache(newCommitObjects, oldCommitObjects, otherCommitObjects map[string]bool, commitObjectsSizes map[string]int64) (addedSize, removedSize int64) { // Calculate size of objects that were added for objectID := range newCommitObjects { if _, exists := oldCommitObjects[objectID]; !exists { // objectID is not referenced in the list of objects of old commit so it is a new object // Calculate its size and add it to the addedSize - addedSize += CalculateSizeOfObject(ctx, opts, objectID) + addedSize += commitObjectsSizes[objectID] } // We might check here if new object is not already in the rest of repo to be precise // However our goal is to prevent growth of repository so on determination of addedSize @@ -144,16 +147,16 @@ func CalculateSizeOfObjects(ctx *gitea_context.PrivateContext, opts *git.RunOpts if _, exists := otherCommitObjects[objectID]; !exists { // objectID is not referenced in rest of the objects of the repository so it was removed // Calculate its size and add it to the addedSize - removedSize += CalculateSizeOfObject(ctx, opts, objectID) + removedSize += commitObjectsSizes[objectID] } } } return addedSize, removedSize } -// ConvertObjectsToMap takes a newline-separated string of git objects and +// convertObjectsToMap takes a newline-separated string of git objects and // converts it into a map for efficient lookup. -func ConvertObjectsToMap(objects string) map[string]bool { +func convertObjectsToMap(objects string) map[string]bool { objectsMap := make(map[string]bool) for _, object := range strings.Split(objects, "\n") { if len(object) == 0 { @@ -165,6 +168,197 @@ func ConvertObjectsToMap(objects string) map[string]bool { return objectsMap } +// convertObjectsToSlice converts a list of hashes in a string from the git rev-list --objects command to a slice of string objects +func convertObjectsToSlice(objects string) (objectIDs []string) { + for _, object := range strings.Split(objects, "\n") { + if len(object) == 0 { + continue + } + objectID := strings.Split(object, " ")[0] + objectIDs = append(objectIDs, objectID) + } + return objectIDs +} + +// loadObjectSizesFromPack access all packs that this push or repo has +// and load compressed object size in bytes into objectSizes map +// using `git verify-pack -v` output +func loadObjectSizesFromPack(ctx *gitea_context.PrivateContext, opts *git.RunOpts, objectIDs []string, objectsSizes map[string]int64) error { + // Find the path from GIT_QUARANTINE_PATH environment variable (path to the pack file) + var packPath string + for _, envVar := range opts.Env { + split := strings.SplitN(envVar, "=", 2) + if split[0] == "GIT_QUARANTINE_PATH" { + packPath = split[1] + break + } + } + + // if no quarantinPath determined we silently ignore + if packPath == "" { + log.Trace("GIT_QUARANTINE_PATH not found in the environment variables. Will read the pack files from main repo instead") + packPath = filepath.Join(ctx.Repo.Repository.RepoPath(), "./objects/") + } + log.Warn("packPath: %s", packPath) + + // Find all pack files *.idx in the quarantine directory + packFiles, err := filepath.Glob(filepath.Join(packPath, "./pack/*.idx")) + // if pack file not found we silently ignore + if err != nil { + log.Trace("Error during finding pack files %s: %v", filepath.Join(packPath, "./pack/*.idx"), err) + } + + // Loop over each pack file + i := 0 + for _, packFile := range packFiles { + log.Trace("Processing packfile %s", packFile) + // Extract and store in cache objectsSizes the sizes of the object parsing output of the `git verify-pack` command + output, _, err := git.NewCommand(ctx, "verify-pack", "-v").AddDynamicArguments(packFile).RunStdString(opts) + if err != nil { + log.Trace("Error during git verify-pack on pack file: %s", packFile) + continue + } + + // Parsing the output of the git verify-pack command + lines := strings.Split(output, "\n") + for _, line := range lines { + fields := strings.Fields(line) + if len(fields) < 4 { + continue + } + + // Second field has object type + // If object type is not known filter it out and do not process + objectType := fields[1] + if objectType != "commit" && objectType != "tree" && objectType != "blob" && objectType != "tag" { + continue + } + + // First field would have an object hash + objectID := fields[0] + + // Forth field would have an object compressed size + size, err := strconv.ParseInt(fields[3], 10, 64) + if err != nil { + log.Trace("Failed to parse size for object %s: %v", objectID, err) + continue + } + i++ + objectsSizes[objectID] = size + } + } + + log.Trace("Loaded %d items from packfiles", i) + return nil +} + +// loadObjectsSizesViaCatFile uses hashes from objectIDs and runs `git cat-file -s` in 10 workers to return each object sizes +// Objects for which size is already loaded are skipped +// can't use `git cat-file --batch-check` here as it only provides data from git DB before the commit applied and has no knowledge on new commit objects +func loadObjectsSizesViaCatFile(ctx *gitea_context.PrivateContext, opts *git.RunOpts, objectIDs []string, objectsSizes map[string]int64) error { + // This is the number of workers that will simultaneously process CalculateSizeOfObject. + const numWorkers = 10 + + var wg sync.WaitGroup + var mu sync.Mutex + + reducedObjectIDs := make([]string, 0, len(objectIDs)) + + // Loop over all objectIDs and find which ones are missing size information + for _, objectID := range objectIDs { + _, exists := objectsSizes[objectID] + + // If object doesn't yet have size in objectsSizes add it for further processing + if !exists { + reducedObjectIDs = append(reducedObjectIDs, objectID) + } + } + + // Start workers and determine size using `git cat-file -s` store in objectsSizes cache + for w := 1; w <= numWorkers; w++ { + wg.Add(1) + go func(reducedObjectIDs []string) { + defer wg.Done() + for _, objectID := range reducedObjectIDs { + objectSize := calculateSizeOfObject(ctx, opts, objectID) + mu.Lock() // Protecting shared resource + objectsSizes[objectID] = objectSize + mu.Unlock() // Releasing shared resource for other goroutines + } + }(reducedObjectIDs) + } + + // Wait for all workers to finish processing. + wg.Wait() + + return nil +} + +// loadObjectsSizesViaBatch uses hashes from objectIDs and uses pre-opened `git cat-file --batch-check` command to slice and return each object sizes +// This function can't be used for new commit objects. +// It speeds up loading object sizes from existing git database of the repository avoiding +// multiple `git cat-files -s` +func loadObjectsSizesViaBatch(ctx *gitea_context.PrivateContext, repoPath string, objectIDs []string, objectsSizes map[string]int64) error { + var i int32 + + reducedObjectIDs := make([]string, 0, len(objectIDs)) + + // Loop over all objectIDs and find which ones are missing size information + for _, objectID := range objectIDs { + _, exists := objectsSizes[objectID] + + // If object doesn't yet have size in objectsSizes add it for further processing + if !exists { + reducedObjectIDs = append(reducedObjectIDs, objectID) + } + } + + wr, rd, cancel := git.CatFileBatchCheck(ctx, repoPath) + defer cancel() + + for _, commitID := range reducedObjectIDs { + _, err := wr.Write([]byte(commitID + "\n")) + if err != nil { + return err + } + i++ + line, err := rd.ReadString('\n') + if err != nil { + return err + } + if len(line) == 1 { + line, err = rd.ReadString('\n') + if err != nil { + return err + } + } + fields := strings.Fields(line) + objectID := fields[0] + if len(fields) < 3 || len(fields) > 3 { + log.Trace("String '%s' does not contain size ignored %s: %v", line, objectID, err) + continue + } + sizeStr := fields[2] + size, err := parseSize(sizeStr) + if err != nil { + log.Trace("String '%s' Failed to parse size for object %s: %v", line, objectID, err) + continue + } + objectsSizes[objectID] = size + } + + return nil +} + +// parseSize parses the object size from a string +func parseSize(sizeStr string) (int64, error) { + size, err := strconv.ParseInt(sizeStr, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse object size: %w", err) + } + return size, nil +} + // HookPreReceive checks whether a individual commit is acceptable func HookPreReceive(ctx *gitea_context.PrivateContext) { startTime := time.Now() @@ -181,39 +375,118 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { var addedSize int64 var removedSize int64 + var isRepoOversized bool + var pushSize *git.CountObject + var repoSize *git.CountObject + var err error + var duration time.Duration - // Calculating total size of the push using git count-objects - pushSize, err := git.CountObjectsWithEnv(ctx, repo.RepoPath(), ourCtx.env) - if err != nil { - log.Error("Unable to get repository size with env %v: %s Error: %v", repo.RepoPath(), ourCtx.env, err) - ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "err": err.Error(), - }) - return + if repo.IsRepoSizeLimitEnabled() { + + // Calculating total size of the repo using `git count-objects` + repoSize, err = git.CountObjects(ctx, repo.RepoPath()) + if err != nil { + log.Error("Unable to get repository size with env %v: %s Error: %v", repo.RepoPath(), ourCtx.env, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": err.Error(), + }) + return + } + + // Calculating total size of the push using `git count-objects` + pushSize, err = git.CountObjectsWithEnv(ctx, repo.RepoPath(), ourCtx.env) + if err != nil { + log.Error("Unable to get push size with env %v: %s Error: %v", repo.RepoPath(), ourCtx.env, err) + ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ + "err": err.Error(), + }) + return + } + + // Cache whether the repository would breach the size limit after the operation + isRepoOversized = repo.IsRepoSizeOversized(pushSize.Size + pushSize.SizePack) + log.Warn("Push counts %+v", pushSize) + log.Warn("Repo counts %+v", repoSize) } - // Cache whether the repository would breach the size limit after the operation - isRepoOversized := repo.RepoSizeIsOversized(pushSize.Size) - log.Trace("Push size %d", pushSize.Size) - // Iterate across the provided old commit IDs for i := range opts.OldCommitIDs { oldCommitID := opts.OldCommitIDs[i] newCommitID := opts.NewCommitIDs[i] refFullName := opts.RefFullNames[i] + log.Trace("Processing old commit: %s, new commit: %s, ref: %s", oldCommitID, newCommitID, refFullName) + // If operation is in potential breach of size limit prepare data for analysis if isRepoOversized { + var gitObjects string + var error error + // Create cache of objects in old commit - gitObjects, _, err := git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(oldCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) - if err != nil { - log.Error("Unable to list objects in old commit: %s in %-v Error: %v", oldCommitID, repo, err) + // if oldCommitID all 0 then it's a fresh repository on gitea server and all git operations on such oldCommitID would fail + if oldCommitID != "0000000000000000000000000000000000000000" { + gitObjects, _, err = git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(oldCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) + if err != nil { + log.Error("Unable to list objects in old commit: %s in %-v Error: %v", oldCommitID, repo, err) + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: fmt.Sprintf("Fail to list objects in old commit: %v", err), + }) + return + } + } + + commitObjectsSizes := make(map[string]int64) + oldCommitObjects := convertObjectsToMap(gitObjects) + objectIDs := convertObjectsToSlice(gitObjects) + + // Create cache of objects that are in the repository but not part of old or new commit + // if oldCommitID all 0 then it's a fresh repository on gitea server and all git operations on such oldCommitID would fail + if oldCommitID == "0000000000000000000000000000000000000000" { + gitObjects, _, err = git.NewCommand(ctx, "rev-list", "--objects", "--all").AddDynamicArguments("^" + newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) + if err != nil { + log.Error("Unable to list objects in the repo that are missing from both old %s and new %s commits in %-v Error: %v", oldCommitID, newCommitID, repo, err) + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: fmt.Sprintf("Fail to list objects missing from both old and new commits: %v", err), + }) + return + } + } else { + gitObjects, _, err = git.NewCommand(ctx, "rev-list", "--objects", "--all").AddDynamicArguments("^"+oldCommitID, "^"+newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) + if err != nil { + log.Error("Unable to list objects in the repo that are missing from both old %s and new %s commits in %-v Error: %v", oldCommitID, newCommitID, repo, err) + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: fmt.Sprintf("Fail to list objects missing from both old and new commits: %v", err), + }) + return + } + + } + + otherCommitObjects := convertObjectsToMap(gitObjects) + objectIDs = append(objectIDs, convertObjectsToSlice(gitObjects)...) + // Unfortunately `git cat-file --check-batch` shows full object size + // so we would load compressed sizes from pack file via `git verify-pack -v` if there are pack files in repo + // The result would still miss items that are loose as individual objects (not part of pack files) + if repoSize.InPack > 0 { + error = loadObjectSizesFromPack(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: nil}, objectIDs, commitObjectsSizes) + if error != nil { + log.Error("Unable to get sizes of objects from the pack in %-v Error: %v", repo, error) + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: fmt.Sprintf("Fail to get sizes of objects in repo: %v", err), + }) + return + } + } + + // Load loose objects that are missing + error = loadObjectsSizesViaBatch(ctx, repo.RepoPath(), objectIDs, commitObjectsSizes) + if error != nil { + log.Error("Unable to get sizes of objects that are missing in both old %s and new commits %s in %-v Error: %v", oldCommitID, newCommitID, repo, error) ctx.JSON(http.StatusInternalServerError, private.Response{ - Err: fmt.Sprintf("Fail to list objects in old commit: %v", err), + Err: fmt.Sprintf("Fail to get sizes of objects missing in both old and new commit and those in old commit: %v", err), }) return } - oldCommitObjects := ConvertObjectsToMap(gitObjects) // Create cache of objects in new commit gitObjects, _, err = git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) @@ -224,21 +497,36 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { }) return } - newCommitObjects := ConvertObjectsToMap(gitObjects) - // Create cache of objects that are in the repository but not part of old or new commit - gitObjects, _, err = git.NewCommand(ctx, "rev-list", "--objects", "--all").AddDynamicArguments("^"+oldCommitID, "^"+newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}) - if err != nil { - log.Error("Unable to list objects in the repo that are missing from both old %s and new %s commits in %-v Error: %v", oldCommitID, newCommitID, repo, err) + newCommitObjects := convertObjectsToMap(gitObjects) + objectIDs = convertObjectsToSlice(gitObjects) + // Unfortunately `git cat-file --check-batch` doesn't work on objects not yet accepted into git database + // so the sizes will be calculated through pack file `git verify-pack -v` if there are pack files + // The result would still miss items that were sent loose as individual objects (not part of pack files) + if pushSize.InPack > 0 { + error = loadObjectSizesFromPack(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}, objectIDs, commitObjectsSizes) + if error != nil { + log.Error("Unable to get sizes of objects from the pack in new commit %s in %-v Error: %v", newCommitID, repo, error) + ctx.JSON(http.StatusInternalServerError, private.Response{ + Err: fmt.Sprintf("Fail to get sizes of objects in new commit: %v", err), + }) + return + } + } + + // After loading everything we could from pack file, objects could have been sent as loose bunch as well + // We need to load them individually with `git cat-file -s` on any object that is missing from accumulated size cache commitObjectsSizes + error = loadObjectsSizesViaCatFile(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}, objectIDs, commitObjectsSizes) + if error != nil { + log.Error("Unable to get sizes of objects in new commit %s in %-v Error: %v", newCommitID, repo, error) ctx.JSON(http.StatusInternalServerError, private.Response{ - Err: fmt.Sprintf("Fail to list objects missing from both old and new commits: %v", err), + Err: fmt.Sprintf("Fail to get sizes of objects in new commit: %v", err), }) return } - otherCommitObjects := ConvertObjectsToMap(gitObjects) // Calculate size that was added and removed by the new commit - addedSize, removedSize = CalculateSizeOfObjects(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}, newCommitObjects, oldCommitObjects, otherCommitObjects) + addedSize, removedSize = calculateSizeOfObjectsFromCache(newCommitObjects, oldCommitObjects, otherCommitObjects, commitObjectsSizes) } switch { @@ -256,14 +544,16 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { } } - duration := time.Since(startTime) - log.Trace("During size checking - Addition in size is: %d, removal in size is: %d, limit size: %s, push size: %d. Took %s seconds.", addedSize, removedSize, base.FileSize(repo.GetActualSizeLimit()), pushSize.Size, duration) + if repo.IsRepoSizeLimitEnabled() { + duration = time.Since(startTime) + log.Warn("During size checking - Addition in size is: %d, removal in size is: %d, limit size: %d, push size: %d, repo size: %d. Took %s seconds.", addedSize, removedSize, repo.GetActualSizeLimit(), pushSize.Size+pushSize.SizePack, repo.Size, duration) + } // If total of commits add more size then they remove and we are in a potential breach of size limit -- abort if (addedSize > removedSize) && isRepoOversized { - log.Warn("Forbidden: new repo size %s is over limitation of %s. Push size: %s. Took %s seconds.", base.FileSize(addedSize-removedSize), base.FileSize(repo.GetActualSizeLimit()), base.FileSize(pushSize.Size), duration) + log.Warn("Forbidden: new repo size %s would be over limitation of %s. Push size: %s. Took %s seconds. addedSize: %s. removedSize: %s", base.FileSize(repo.Size+addedSize-removedSize), base.FileSize(repo.GetActualSizeLimit()), base.FileSize(pushSize.Size+pushSize.SizePack), duration, base.FileSize(addedSize), base.FileSize(removedSize)) ctx.JSON(http.StatusForbidden, private.Response{ - UserMsg: fmt.Sprintf("Repository size is over limitation of %s", base.FileSize(repo.GetActualSizeLimit())), + UserMsg: fmt.Sprintf("New repository size is over limitation of %s", base.FileSize(repo.GetActualSizeLimit())), }) return } diff --git a/routers/web/repo/setting.go b/routers/web/repo/setting.go index 1c82b57296..354879de68 100644 --- a/routers/web/repo/setting.go +++ b/routers/web/repo/setting.go @@ -68,7 +68,7 @@ func SettingsCtxData(ctx *context.Context) { ctx.Data["DisableNewPushMirrors"] = setting.Mirror.DisableNewPush ctx.Data["DefaultMirrorInterval"] = setting.Mirror.DefaultInterval ctx.Data["MinimumMirrorInterval"] = setting.Mirror.MinInterval - ctx.Data["Err_RepoSize"] = ctx.Repo.Repository.RepoSizeIsOversized(ctx.Repo.Repository.GetActualSizeLimit() / 10) // less than 10% left + ctx.Data["Err_RepoSize"] = ctx.Repo.Repository.IsRepoSizeOversized(ctx.Repo.Repository.GetActualSizeLimit() / 10) // less than 10% left ctx.Data["ActualSizeLimit"] = ctx.Repo.Repository.GetActualSizeLimit() ctx.Data["EnableSizeLimit"] = setting.EnableSizeLimit @@ -123,7 +123,7 @@ func SettingsPost(ctx *context.Context) { ctx.Data["CodeIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled repo := ctx.Repo.Repository - ctx.Data["Err_RepoSize"] = repo.RepoSizeIsOversized(repo.SizeLimit / 10) // less than 10% left + ctx.Data["Err_RepoSize"] = repo.IsRepoSizeOversized(repo.SizeLimit / 10) // less than 10% left switch ctx.FormString("action") { case "update": From 79251d5f319ee2dd83544d1054fd93cdfc6cd2f2 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Fri, 30 Jun 2023 06:45:26 +0000 Subject: [PATCH 48/55] fixed a lint error --- routers/web/admin/repos_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/web/admin/repos_test.go b/routers/web/admin/repos_test.go index dcfb1c7b38..e9d3325b3d 100644 --- a/routers/web/admin/repos_test.go +++ b/routers/web/admin/repos_test.go @@ -14,7 +14,7 @@ import ( func TestUpdateRepoPost(t *testing.T) { unittest.PrepareTestEnv(t) - ctx := test.MockContext(t, "admin/repos") + ctx, _ := test.MockContext(t, "admin/repos") test.LoadUser(t, ctx, 1) ctx.Req.Form.Set("enable_size_limit", "on") From b521d36be54f16f9e55397fcb038ec9391cc7f48 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Tue, 11 Jul 2023 08:21:17 +0000 Subject: [PATCH 49/55] Now in hook_pre_recieve.go, when cat file -s is called - the work is no longer shared among workers - race condition fixed --- routers/private/hook_pre_receive.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index 88cb1c4867..06c46f228f 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -262,15 +262,21 @@ func loadObjectsSizesViaCatFile(ctx *gitea_context.PrivateContext, opts *git.Run var wg sync.WaitGroup var mu sync.Mutex - reducedObjectIDs := make([]string, 0, len(objectIDs)) + // Prepare numWorker slices to store the work + reducedObjectIDs := make([][]string, 0, numWorkers) + for i := 0; i < numWorkers; i++ { + reducedObjectIDs[i] = make([]string, 0, len(objectIDs)/numWorkers+1) + } // Loop over all objectIDs and find which ones are missing size information + i := 0 for _, objectID := range objectIDs { _, exists := objectsSizes[objectID] // If object doesn't yet have size in objectsSizes add it for further processing if !exists { - reducedObjectIDs = append(reducedObjectIDs, objectID) + reducedObjectIDs[i%numWorkers] = append(reducedObjectIDs[i%numWorkers], objectID) + i++ } } @@ -280,12 +286,14 @@ func loadObjectsSizesViaCatFile(ctx *gitea_context.PrivateContext, opts *git.Run go func(reducedObjectIDs []string) { defer wg.Done() for _, objectID := range reducedObjectIDs { + ctx := ctx + opts := opts objectSize := calculateSizeOfObject(ctx, opts, objectID) mu.Lock() // Protecting shared resource objectsSizes[objectID] = objectSize mu.Unlock() // Releasing shared resource for other goroutines } - }(reducedObjectIDs) + }(reducedObjectIDs[(w-1)%numWorkers]) } // Wait for all workers to finish processing. From 8097699451478f0ed70fb9c7379416d595f9f314 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Tue, 5 Sep 2023 09:42:42 +0000 Subject: [PATCH 50/55] Updated routers/web/admin/repos_test.go to use code.gitea.io/gitea/modules/contexttest and fix lint error --- routers/web/admin/repos_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/routers/web/admin/repos_test.go b/routers/web/admin/repos_test.go index e9d3325b3d..7975d6a881 100644 --- a/routers/web/admin/repos_test.go +++ b/routers/web/admin/repos_test.go @@ -7,15 +7,15 @@ import ( "testing" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/test" + "code.gitea.io/gitea/modules/contexttest" "github.com/stretchr/testify/assert" ) func TestUpdateRepoPost(t *testing.T) { unittest.PrepareTestEnv(t) - ctx, _ := test.MockContext(t, "admin/repos") - test.LoadUser(t, ctx, 1) + ctx, _ := contexttest.MockContext(t, "admin/repos") + contexttest.LoadUser(t, ctx, 1) ctx.Req.Form.Set("enable_size_limit", "on") ctx.Req.Form.Set("repo_size_limit", "222 kcmcm") From aec62fa4743bdb744444947e63bda933f8cbb909 Mon Sep 17 00:00:00 2001 From: Dmitry Frolov <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Tue, 7 Nov 2023 13:45:15 +0000 Subject: [PATCH 51/55] Update templates/repo/settings/options.tmpl Co-authored-by: silverwind --- templates/repo/settings/options.tmpl | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/templates/repo/settings/options.tmpl b/templates/repo/settings/options.tmpl index af88080c70..94d6807777 100644 --- a/templates/repo/settings/options.tmpl +++ b/templates/repo/settings/options.tmpl @@ -15,11 +15,9 @@
{{FileSize .Repository.Size}} - {{if .ActualSizeLimit}} - {{if .EnableSizeLimit}} + {{if and .ActualSizeLimit .EnableSizeLimit}} /{{FileSize .ActualSizeLimit}} {{end}} - {{end}}
From 361214008777c25d71d56df1bc392adac4c7436a Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Sat, 2 Dec 2023 10:51:51 +0000 Subject: [PATCH 52/55] fix a panic in hook_pre_recieve.go --- routers/private/hook_pre_receive.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index e9ec5c61c3..d2c861ce5d 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -263,7 +263,7 @@ func loadObjectsSizesViaCatFile(ctx *gitea_context.PrivateContext, opts *git.Run var mu sync.Mutex // Prepare numWorker slices to store the work - reducedObjectIDs := make([][]string, 0, numWorkers) + reducedObjectIDs := make([][]string, numWorkers) for i := 0; i < numWorkers; i++ { reducedObjectIDs[i] = make([]string, 0, len(objectIDs)/numWorkers+1) } From 006f216f3d1500b21ffea2d3f86fc864a3155a70 Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Sun, 3 Dec 2023 18:12:51 +0000 Subject: [PATCH 53/55] fix race condition due to opts.Env. Exclude size breach determination using LFS + GIT size, only use GIT size --- models/repo/repo.go | 2 +- routers/private/hook_pre_receive.go | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/models/repo/repo.go b/models/repo/repo.go index 3b19825d7b..10b74bec8e 100644 --- a/models/repo/repo.go +++ b/models/repo/repo.go @@ -569,7 +569,7 @@ func (repo *Repository) GetActualSizeLimit() int64 { // RepoSizeIsOversized return true if is over size limitation func (repo *Repository) IsRepoSizeOversized(additionalSize int64) bool { - return setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 && repo.Size+additionalSize > repo.GetActualSizeLimit() + return setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 && repo.GitSize+additionalSize > repo.GetActualSizeLimit() } // RepoSizeLimitEnabled return true if size limit checking is enabled and limit is non zero for this specific repository diff --git a/routers/private/hook_pre_receive.go b/routers/private/hook_pre_receive.go index d2c861ce5d..54c618b970 100644 --- a/routers/private/hook_pre_receive.go +++ b/routers/private/hook_pre_receive.go @@ -146,7 +146,7 @@ func calculateSizeOfObjectsFromCache(newCommitObjects, oldCommitObjects, otherCo // objectID is not referenced in the list of new commit objects so it was possibly removed if _, exists := otherCommitObjects[objectID]; !exists { // objectID is not referenced in rest of the objects of the repository so it was removed - // Calculate its size and add it to the addedSize + // Calculate its size and add it to the removedSize removedSize += commitObjectsSizes[objectID] } } @@ -283,17 +283,20 @@ func loadObjectsSizesViaCatFile(ctx *gitea_context.PrivateContext, opts *git.Run // Start workers and determine size using `git cat-file -s` store in objectsSizes cache for w := 1; w <= numWorkers; w++ { wg.Add(1) - go func(reducedObjectIDs []string) { + go func(reducedObjectIDs *[]string) { defer wg.Done() - for _, objectID := range reducedObjectIDs { + for _, objectID := range *reducedObjectIDs { ctx := ctx - opts := opts - objectSize := calculateSizeOfObject(ctx, opts, objectID) + // Create a copy of opts to allow change of the Env property + tsopts := *opts + // Ensure that each worker has its own copy of the Env environment to prevent races + tsopts.Env = append([]string(nil), opts.Env...) + objectSize := calculateSizeOfObject(ctx, &tsopts, objectID) mu.Lock() // Protecting shared resource objectsSizes[objectID] = objectSize mu.Unlock() // Releasing shared resource for other goroutines } - }(reducedObjectIDs[(w-1)%numWorkers]) + }(&reducedObjectIDs[(w-1)%numWorkers]) } // Wait for all workers to finish processing. @@ -554,12 +557,12 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) { if repo.IsRepoSizeLimitEnabled() { duration = time.Since(startTime) - log.Warn("During size checking - Addition in size is: %d, removal in size is: %d, limit size: %d, push size: %d, repo size: %d. Took %s seconds.", addedSize, removedSize, repo.GetActualSizeLimit(), pushSize.Size+pushSize.SizePack, repo.Size, duration) + log.Warn("During size checking - Addition in size is: %d, removal in size is: %d, limit size: %d, push size: %d, repo size: %d. Took %s seconds.", addedSize, removedSize, repo.GetActualSizeLimit(), pushSize.Size+pushSize.SizePack, repo.GitSize, duration) } // If total of commits add more size then they remove and we are in a potential breach of size limit -- abort if (addedSize > removedSize) && isRepoOversized { - log.Warn("Forbidden: new repo size %s would be over limitation of %s. Push size: %s. Took %s seconds. addedSize: %s. removedSize: %s", base.FileSize(repo.Size+addedSize-removedSize), base.FileSize(repo.GetActualSizeLimit()), base.FileSize(pushSize.Size+pushSize.SizePack), duration, base.FileSize(addedSize), base.FileSize(removedSize)) + log.Warn("Forbidden: new repo size %s would be over limitation of %s. Push size: %s. Took %s seconds. addedSize: %s. removedSize: %s", base.FileSize(repo.GitSize+addedSize-removedSize), base.FileSize(repo.GetActualSizeLimit()), base.FileSize(pushSize.Size+pushSize.SizePack), duration, base.FileSize(addedSize), base.FileSize(removedSize)) ctx.JSON(http.StatusForbidden, private.Response{ UserMsg: fmt.Sprintf("New repository size is over limitation of %s", base.FileSize(repo.GetActualSizeLimit())), }) From ecdc2a7962a4d074cac628f13add24d5b7e659bc Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Wed, 28 Feb 2024 12:12:33 +0000 Subject: [PATCH 54/55] Fixes related to prior merge --- options/locale/locale_en-US.ini | 4 ++-- routers/web/admin/repos.go | 1 - routers/web/admin/repos_test.go | 2 +- routers/web/explore/repo.go | 4 ++-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index 9387202552..5fb6b7c011 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -3187,8 +3187,8 @@ config.open_with_editor_app_help = The "Open with" editors for the clone menu. I config.repository_config = Repository Configuration config.enable_size_limit = Enable Size Limit config.repo_size_limit = Default Repository Size Limit -config.invalid_repo_size = Invalid repository size. -config.save_repo_size_setting_failed = Failed to save global repository settings +config.invalid_repo_size = Invalid repository size %s +config.save_repo_size_setting_failed = Failed to save global repository settings %s config.repository_setting_success = Global repository setting has been updated config.git_config = Git Configuration diff --git a/routers/web/admin/repos.go b/routers/web/admin/repos.go index 7d99c7a12e..1d14f4429e 100644 --- a/routers/web/admin/repos.go +++ b/routers/web/admin/repos.go @@ -76,7 +76,6 @@ func UpdateRepoPost(ctx *context.Context) { } err = setting.SaveGlobalRepositorySetting(form.EnableSizeLimit, repoSizeLimit) - if err != nil { ctx.Data["Err_Repo_Size_Save"] = err.Error() explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{ diff --git a/routers/web/admin/repos_test.go b/routers/web/admin/repos_test.go index 7975d6a881..17594fad2e 100644 --- a/routers/web/admin/repos_test.go +++ b/routers/web/admin/repos_test.go @@ -7,7 +7,7 @@ import ( "testing" "code.gitea.io/gitea/models/unittest" - "code.gitea.io/gitea/modules/contexttest" + "code.gitea.io/gitea/services/contexttest" "github.com/stretchr/testify/assert" ) diff --git a/routers/web/explore/repo.go b/routers/web/explore/repo.go index b390490e2f..bb3fca3aea 100644 --- a/routers/web/explore/repo.go +++ b/routers/web/explore/repo.go @@ -155,13 +155,13 @@ func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) { ctx.Data["Page"] = pager if ctx.Data["Err_Repo_Size_Limit"] != nil { - ctx.RenderWithErr(ctx.Tr("admin.config.invalid_repo_size")+" "+ctx.Data["Err_Repo_Size_Limit"].(string), + ctx.RenderWithErr(ctx.Tr("admin.config.invalid_repo_size", ctx.Data["Err_Repo_Size_Limit"].(string)), opts.TplName, nil) return } if ctx.Data["Err_Repo_Size_Save"] != nil { - ctx.RenderWithErr(ctx.Tr("admin.config.save_repo_size_setting_failed")+" "+ctx.Data["Err_Repo_Size_Save"].(string), + ctx.RenderWithErr(ctx.Tr("admin.config.save_repo_size_setting_failed", ctx.Data["Err_Repo_Size_Save"].(string)), opts.TplName, nil) return } From 807da90076ec0c6277783e72f1f9b07b9f98ecbd Mon Sep 17 00:00:00 2001 From: DmitryFrolovTri <23313323+DmitryFrolovTri@users.noreply.github.com> Date: Wed, 28 Feb 2024 19:07:36 +0000 Subject: [PATCH 55/55] Postmerge update auth for TestGit/HTTP/SizeLimit - Over, UnderAfterResize, Deletion tests --- tests/integration/api_helper_for_declarative_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/tests/integration/api_helper_for_declarative_test.go b/tests/integration/api_helper_for_declarative_test.go index 9272f5938b..1e83d23255 100644 --- a/tests/integration/api_helper_for_declarative_test.go +++ b/tests/integration/api_helper_for_declarative_test.go @@ -135,9 +135,10 @@ func doAPIForkRepository(ctx APITestContext, username string, callback ...func(* func doAPIGetRepositorySize(ctx APITestContext, owner, repo string) func(*testing.T) int64 { return func(t *testing.T) int64 { - urlStr := fmt.Sprintf("/api/v1/repos/%s/%s?token=%s", ctx.Username, ctx.Reponame, ctx.Token) + urlStr := fmt.Sprintf("/api/v1/repos/%s/%s", ctx.Username, ctx.Reponame) - req := NewRequest(t, "GET", urlStr) + req := NewRequest(t, "GET", urlStr). + AddTokenAuth(ctx.Token) if ctx.ExpectedCode != 0 { ctx.Session.MakeRequest(t, req, ctx.ExpectedCode) } @@ -467,9 +468,10 @@ func doAPIAddRepoToOrganizationTeam(ctx APITestContext, teamID int64, orgName, r func doAPISetRepoSizeLimit(ctx APITestContext, owner, repo string, size int64) func(*testing.T) { return func(t *testing.T) { - urlStr := fmt.Sprintf("/api/v1/repos/%s/%s?token=%s", - owner, repo, ctx.Token) - req := NewRequestWithJSON(t, http.MethodPatch, urlStr, &api.EditRepoOption{SizeLimit: &size}) + urlStr := fmt.Sprintf("/api/v1/repos/%s/%s", + owner, repo) + req := NewRequestWithJSON(t, http.MethodPatch, urlStr, &api.EditRepoOption{SizeLimit: &size}). + AddTokenAuth(ctx.Token) if ctx.ExpectedCode != 0 { ctx.Session.MakeRequest(t, req, ctx.ExpectedCode)