This commit is contained in:
Dmitry Frolov 2024-09-20 08:48:30 +08:00 committed by GitHub
commit 0d5be3ecc7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 956 additions and 2 deletions

View File

@ -1005,6 +1005,15 @@ LEVEL = Info
;; Allow fork repositories without maximum number limit
;ALLOW_FORK_WITHOUT_MAXIMUM_LIMIT = true
;
;; Enable applying a global size limit defined by REPO_SIZE_LIMIT. Each repository can have a value that overrides the global limit
;; "false" means no limit will be enforced, even if specified on a repository
;ENABLE_SIZE_LIMIT = false
;
;; Specify a global repository size limit in bytes to apply for each repository. 0 - No limit
;; If repository has it's own limit set in UI it will override the global setting
;; Standard units of measurements for size can be used like B, KB, KiB, ... , EB, EiB, ...
;REPO_SIZE_LIMIT = 0
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

View File

@ -0,0 +1,52 @@
---
date: "2023-05-24T13:00:00+00:00"
title: "Per repository size limit"
slug: "repo-size-limit"
weight: 12
toc: false
draft: false
aliases:
- /en-us/repo-size-limit
menu:
sidebar:
parent: "administration"
name: "Per repository size limit"
weight: 12
identifier: "repo-size-limit"
---
# Gitea per repository size limit setup
To use Gitea's experimental built-in per repository size limit support, Administrator must update the `app.ini` file:
```ini
;; Enable applying a global size limit defined by REPO_SIZE_LIMIT. Each repository can have a value that overrides the global limit
;; "false" means no limit will be enforced, even if specified on a repository
ENABLE_SIZE_LIMIT = true
;; Specify a global repository size limit in bytes to apply for each repository. 0 - No limit
;; If repository has it's own limit set in UI it will override the global setting
;; Standard units of measurements for size can be used like B, KB, KiB, ... , EB, EiB, ...
REPO_SIZE_LIMIT = 500 MB
This setting is persistent.
The size limitation is triggered when repository `disk size` + `new commit size` > `defined repository size limit`
If size limitation is triggered the feature would prevent commits that increase repository size on disk
of gitea server and allow those that decrease it
# Gitea per repository size limit setup in UI
1. For Gitea admin it is possible during runtime to enable/disable limit size feature, change the global size limit on the fly.
**This setting is not persistent across restarts**
`Admin panel/Site settings` -> `Repository management`
Persistance can be achieved if the limit is maintained by editing `app.ini` file
2. The individually set per repository limit in `Settings` of the
repository would take precedence over global limit when the size limit
feature is enabled. Only admin can modify those limits
**Note**: Size checking for large repositories is time consuming operation so time of push under size limit might increase up to a minute depending on your server hardware

View File

@ -601,6 +601,8 @@ var migrations = []Migration{
NewMigration("Add metadata column for comment table", v1_23.AddCommentMetaDataColumn),
// v304 -> v305
NewMigration("Add index for release sha1", v1_23.AddIndexForReleaseSha1),
// to modify later
NewMigration("Add size limit on repository", v1_22.AddSizeLimitOnRepo),
}
// GetCurrentDBVersion returns the current db version

View File

@ -0,0 +1,17 @@
// Copyright 2022 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package v1_22 //nolint
import (
"xorm.io/xorm"
)
func AddSizeLimitOnRepo(x *xorm.Engine) error {
type Repository struct {
ID int64 `xorm:"pk autoincr"`
SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"`
}
return x.Sync2(new(Repository))
}

View File

@ -175,7 +175,9 @@ type Repository struct {
BaseRepo *Repository `xorm:"-"`
IsTemplate bool `xorm:"INDEX NOT NULL DEFAULT false"`
TemplateID int64 `xorm:"INDEX"`
SizeLimit int64 `xorm:"NOT NULL DEFAULT 0"`
Size int64 `xorm:"NOT NULL DEFAULT 0"`
EnableSizeLimit bool `xorm:"NOT NULL DEFAULT true"`
GitSize int64 `xorm:"NOT NULL DEFAULT 0"`
LFSSize int64 `xorm:"NOT NULL DEFAULT 0"`
CodeIndexerStatus *RepoIndexerStatus `xorm:"-"`
@ -584,6 +586,27 @@ func (repo *Repository) IsOwnedBy(userID int64) bool {
return repo.OwnerID == userID
}
// GetActualSizeLimit returns repository size limit in bytes
// or global repository limit setting if per repository size limit is not set
func (repo *Repository) GetActualSizeLimit() int64 {
sizeLimit := repo.SizeLimit
if setting.RepoSizeLimit > 0 && sizeLimit == 0 {
sizeLimit = setting.RepoSizeLimit
}
return sizeLimit
}
// RepoSizeIsOversized return true if is over size limitation
func (repo *Repository) IsRepoSizeOversized(additionalSize int64) bool {
return setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0 && repo.GitSize+additionalSize > repo.GetActualSizeLimit()
}
// RepoSizeLimitEnabled return true if size limit checking is enabled and limit is non zero for this specific repository
// this is used to enable size checking during pre-receive hook
func (repo *Repository) IsRepoSizeLimitEnabled() bool {
return setting.EnableSizeLimit && repo.GetActualSizeLimit() > 0
}
// CanCreateBranch returns true if repository meets the requirements for creating new branches.
func (repo *Repository) CanCreateBranch() bool {
return !repo.IsMirror

View File

@ -119,6 +119,13 @@ func FileSize(s int64) string {
return humanize.IBytes(uint64(s))
}
// Get FileSize bytes value from String.
func GetFileSize(s string) (int64, error) {
v, err := humanize.ParseBytes(s)
iv := int64(v)
return iv, err
}
// EllipsisString returns a truncated short string,
// it appends '...' in the end of the length of string is too large.
func EllipsisString(str string, length int) string {

View File

@ -114,6 +114,13 @@ func TestFileSize(t *testing.T) {
assert.Equal(t, "2.0 EiB", FileSize(size))
}
func TestGetFileSize(t *testing.T) {
var size int64 = 512 * 1024 * 1024 * 1024
s, err := GetFileSize("512 GiB")
assert.Equal(t, s, size)
assert.Nil(t, err)
}
func TestEllipsisString(t *testing.T) {
assert.Equal(t, "...", EllipsisString("foobar", 0))
assert.Equal(t, "...", EllipsisString("foobar", 1))

View File

@ -236,6 +236,74 @@ func Push(ctx context.Context, repoPath string, opts PushOptions) error {
return nil
}
// CountObject represents repository count objects report
type CountObject struct {
Count int64
Size int64
InPack int64
Packs int64
SizePack int64
PrunePack int64
Garbage int64
SizeGarbage int64
}
const (
statCount = "count: "
statSize = "size: "
statInpack = "in-pack: "
statPacks = "packs: "
statSizePack = "size-pack: "
statPrunePackage = "prune-package: "
statGarbage = "garbage: "
statSizeGarbage = "size-garbage: "
)
// CountObjects returns the results of git count-objects on the repoPath
func CountObjects(ctx context.Context, repoPath string) (*CountObject, error) {
return CountObjectsWithEnv(ctx, repoPath, nil)
}
// CountObjectsWithEnv returns the results of git count-objects on the repoPath with custom env setup
func CountObjectsWithEnv(ctx context.Context, repoPath string, env []string) (*CountObject, error) {
cmd := NewCommand(ctx, "count-objects", "-v")
stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath, Env: env})
if err != nil {
return nil, err
}
return parseSize(stdout), nil
}
// parseSize parses the output from count-objects and return a CountObject
func parseSize(objects string) *CountObject {
repoSize := new(CountObject)
for _, line := range strings.Split(objects, "\n") {
switch {
case strings.HasPrefix(line, statCount):
repoSize.Count, _ = strconv.ParseInt(line[7:], 10, 64)
case strings.HasPrefix(line, statSize):
number, _ := strconv.ParseInt(line[6:], 10, 64)
repoSize.Size = number * 1024
case strings.HasPrefix(line, statInpack):
repoSize.InPack, _ = strconv.ParseInt(line[9:], 10, 64)
case strings.HasPrefix(line, statPacks):
repoSize.Packs, _ = strconv.ParseInt(line[7:], 10, 64)
case strings.HasPrefix(line, statSizePack):
number, _ := strconv.ParseInt(line[11:], 10, 64)
repoSize.SizePack = number * 1024
case strings.HasPrefix(line, statPrunePackage):
repoSize.PrunePack, _ = strconv.ParseInt(line[16:], 10, 64)
case strings.HasPrefix(line, statGarbage):
repoSize.Garbage, _ = strconv.ParseInt(line[9:], 10, 64)
case strings.HasPrefix(line, statSizeGarbage):
number, _ := strconv.ParseInt(line[14:], 10, 64)
repoSize.SizeGarbage = number * 1024
}
}
return repoSize
}
// GetLatestCommitTime returns time for latest commit in repository (across all branches)
func GetLatestCommitTime(ctx context.Context, repoPath string) (time.Time, error) {
cmd := NewCommand(ctx, "for-each-ref", "--sort=-committerdate", BranchPrefix, "--count", "1", "--format=%(committerdate)")

View File

@ -10,6 +10,8 @@ import (
"strings"
"code.gitea.io/gitea/modules/log"
"github.com/dustin/go-humanize"
)
// enumerates all the policy repository creating
@ -268,12 +270,35 @@ var (
}
RepoRootPath string
ScriptType = "bash"
EnableSizeLimit = true
RepoSizeLimit int64
)
func SaveGlobalRepositorySetting(enableSizeLimit bool, repoSizeLimit int64) error {
EnableSizeLimit = enableSizeLimit
RepoSizeLimit = repoSizeLimit
sec := CfgProvider.Section("repository")
if EnableSizeLimit {
sec.Key("ENABLE_SIZE_LIMIT").SetValue("true")
} else {
sec.Key("ENABLE_SIZE_LIMIT").SetValue("false")
}
sec.Key("REPO_SIZE_LIMIT").SetValue(humanize.Bytes(uint64(RepoSizeLimit)))
return nil
}
func loadRepositoryFrom(rootCfg ConfigProvider) {
var err error
// Determine and create root git repository path.
sec := rootCfg.Section("repository")
EnableSizeLimit = sec.Key("ENABLE_SIZE_LIMIT").MustBool(false)
v, _ := humanize.ParseBytes(sec.Key("REPO_SIZE_LIMIT").MustString("0"))
RepoSizeLimit = int64(v)
Repository.DisableHTTPGit = sec.Key("DISABLE_HTTP_GIT").MustBool()
Repository.UseCompatSSHURI = sec.Key("USE_COMPAT_SSH_URI").MustBool()
Repository.GoGetCloneURLProtocol = sec.Key("GO_GET_CLONE_URL_PROTOCOL").MustString("https")

View File

@ -148,6 +148,8 @@ type CreateRepoOption struct {
// ObjectFormatName of the underlying git repository
// enum: sha1,sha256
ObjectFormatName string `json:"object_format_name" binding:"MaxSize(6)"`
// SizeLimit of the repository
SizeLimit int64 `json:"size_limit"`
}
// EditRepoOption options when editing a repository's properties
@ -216,6 +218,8 @@ type EditRepoOption struct {
DefaultAllowMaintainerEdit *bool `json:"default_allow_maintainer_edit,omitempty"`
// set to `true` to archive this repository.
Archived *bool `json:"archived,omitempty"`
// SizeLimit of the repository.
SizeLimit *int64 `json:"size_limit,omitempty"`
// set to a string like `8h30m0s` to set the mirror interval time
MirrorInterval *string `json:"mirror_interval,omitempty"`
// enable prune - remove obsolete remote-tracking references when mirroring

View File

@ -1004,6 +1004,7 @@ owner_helper = Some organizations may not show up in the dropdown due to a maxim
repo_name = Repository Name
repo_name_helper = Good repository names use short, memorable and unique keywords.
repo_size = Repository Size
repo_size_limit = Repository Size Limit
template = Template
template_select = Select a template.
template_helper = Make repository a template
@ -1133,6 +1134,8 @@ form.reach_limit_of_creation_1 = The owner has already reached the limit of %d r
form.reach_limit_of_creation_n = The owner has already reached the limit of %d repositories.
form.name_reserved = The repository name "%s" is reserved.
form.name_pattern_not_allowed = The pattern "%s" is not allowed in a repository name.
form.repo_size_limit_negative = Repository size limitation cannot be negative.
form.repo_size_limit_only_by_admins = Only administrators can change the repository size limitation.
need_auth = Authorization
migrate_options = Migration Options
@ -3295,6 +3298,13 @@ config.disable_gravatar = Disable Gravatar
config.enable_federated_avatar = Enable Federated Avatars
config.open_with_editor_app_help = The "Open with" editors for the clone menu. If left empty, the default will be used. Expand to see the default.
config.repository_config = Repository Configuration
config.enable_size_limit = Enable Size Limit
config.repo_size_limit = Default Repository Size Limit
config.invalid_repo_size = Invalid repository size %s
config.save_repo_size_setting_failed = Failed to save global repository settings %s
config.repository_setting_success = Global repository setting has been updated
config.git_config = Git Configuration
config.git_disable_diff_highlight = Disable Diff Syntax Highlight
config.git_max_diff_lines = Max Diff Lines (for a single file)

View File

@ -258,6 +258,7 @@ func CreateUserRepo(ctx *context.APIContext, owner *user_model.User, opt api.Cre
TrustModel: repo_model.ToTrustModel(opt.TrustModel),
IsTemplate: opt.Template,
ObjectFormatName: opt.ObjectFormatName,
SizeLimit: opt.SizeLimit,
})
if err != nil {
if repo_model.IsErrRepoAlreadyExist(err) {
@ -743,6 +744,10 @@ func updateBasicProperties(ctx *context.APIContext, opts api.EditRepoOption) err
repo.DefaultBranch = *opts.DefaultBranch
}
if opts.SizeLimit != nil {
repo.SizeLimit = *opts.SizeLimit
}
if err := repo_service.UpdateRepository(ctx, repo, visibilityChanged); err != nil {
ctx.Error(http.StatusInternalServerError, "UpdateRepository", err)
return err

View File

@ -7,6 +7,11 @@ import (
"fmt"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"code.gitea.io/gitea/models"
asymkey_model "code.gitea.io/gitea/models/asymkey"
@ -16,6 +21,7 @@ import (
access_model "code.gitea.io/gitea/models/perm/access"
"code.gitea.io/gitea/models/unit"
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/private"
@ -101,8 +107,273 @@ func (ctx *preReceiveContext) AssertCreatePullRequest() bool {
return true
}
// calculateSizeOfObject calculates the size of one git object via git cat-file -s command
func calculateSizeOfObject(ctx *gitea_context.PrivateContext, opts *git.RunOpts, objectID string) (objectSize int64) {
objectSizeStr, _, err := git.NewCommand(ctx, "cat-file", "-s").AddDynamicArguments(objectID).RunStdString(opts)
if err != nil {
log.Trace("CalculateSizeOfRemovedObjects: Error during git cat-file -s on object: %s", objectID)
return objectSize
}
objectSize, _ = strconv.ParseInt(strings.TrimSpace(objectSizeStr), 10, 64)
if err != nil {
log.Trace("CalculateSizeOfRemovedObjects: Error during ParseInt on string '%s'", objectID)
return objectSize
}
return objectSize
}
// calculateSizeOfObjectsFromCache calculates the size of objects added and removed from the repository by new push
// it uses data that was cached about the repository for this run
func calculateSizeOfObjectsFromCache(newCommitObjects, oldCommitObjects, otherCommitObjects map[string]bool, commitObjectsSizes map[string]int64) (addedSize, removedSize int64) {
// Calculate size of objects that were added
for objectID := range newCommitObjects {
if _, exists := oldCommitObjects[objectID]; !exists {
// objectID is not referenced in the list of objects of old commit so it is a new object
// Calculate its size and add it to the addedSize
addedSize += commitObjectsSizes[objectID]
}
// We might check here if new object is not already in the rest of repo to be precise
// However our goal is to prevent growth of repository so on determination of addedSize
// We can skip this preciseness, addedSize will be more then real addedSize
// TODO - do not count size of object that is referenced in other part of repo but not referenced neither in old nor new commit
// git will not add the object twice
}
// Calculate size of objects that were removed
for objectID := range oldCommitObjects {
if _, exists := newCommitObjects[objectID]; !exists {
// objectID is not referenced in the list of new commit objects so it was possibly removed
if _, exists := otherCommitObjects[objectID]; !exists {
// objectID is not referenced in rest of the objects of the repository so it was removed
// Calculate its size and add it to the removedSize
removedSize += commitObjectsSizes[objectID]
}
}
}
return addedSize, removedSize
}
// convertObjectsToMap takes a newline-separated string of git objects and
// converts it into a map for efficient lookup.
func convertObjectsToMap(objects string) map[string]bool {
objectsMap := make(map[string]bool)
for _, object := range strings.Split(objects, "\n") {
if len(object) == 0 {
continue
}
objectID := strings.Split(object, " ")[0]
objectsMap[objectID] = true
}
return objectsMap
}
// convertObjectsToSlice converts a list of hashes in a string from the git rev-list --objects command to a slice of string objects
func convertObjectsToSlice(objects string) (objectIDs []string) {
for _, object := range strings.Split(objects, "\n") {
if len(object) == 0 {
continue
}
objectID := strings.Split(object, " ")[0]
objectIDs = append(objectIDs, objectID)
}
return objectIDs
}
// loadObjectSizesFromPack access all packs that this push or repo has
// and load compressed object size in bytes into objectSizes map
// using `git verify-pack -v` output
func loadObjectSizesFromPack(ctx *gitea_context.PrivateContext, opts *git.RunOpts, objectIDs []string, objectsSizes map[string]int64) error {
// Find the path from GIT_QUARANTINE_PATH environment variable (path to the pack file)
var packPath string
for _, envVar := range opts.Env {
split := strings.SplitN(envVar, "=", 2)
if split[0] == "GIT_QUARANTINE_PATH" {
packPath = split[1]
break
}
}
// if no quarantinPath determined we silently ignore
if packPath == "" {
log.Trace("GIT_QUARANTINE_PATH not found in the environment variables. Will read the pack files from main repo instead")
packPath = filepath.Join(ctx.Repo.Repository.RepoPath(), "./objects/")
}
log.Warn("packPath: %s", packPath)
// Find all pack files *.idx in the quarantine directory
packFiles, err := filepath.Glob(filepath.Join(packPath, "./pack/*.idx"))
// if pack file not found we silently ignore
if err != nil {
log.Trace("Error during finding pack files %s: %v", filepath.Join(packPath, "./pack/*.idx"), err)
}
// Loop over each pack file
i := 0
for _, packFile := range packFiles {
log.Trace("Processing packfile %s", packFile)
// Extract and store in cache objectsSizes the sizes of the object parsing output of the `git verify-pack` command
output, _, err := git.NewCommand(ctx, "verify-pack", "-v").AddDynamicArguments(packFile).RunStdString(opts)
if err != nil {
log.Trace("Error during git verify-pack on pack file: %s", packFile)
continue
}
// Parsing the output of the git verify-pack command
lines := strings.Split(output, "\n")
for _, line := range lines {
fields := strings.Fields(line)
if len(fields) < 4 {
continue
}
// Second field has object type
// If object type is not known filter it out and do not process
objectType := fields[1]
if objectType != "commit" && objectType != "tree" && objectType != "blob" && objectType != "tag" {
continue
}
// First field would have an object hash
objectID := fields[0]
// Forth field would have an object compressed size
size, err := strconv.ParseInt(fields[3], 10, 64)
if err != nil {
log.Trace("Failed to parse size for object %s: %v", objectID, err)
continue
}
i++
objectsSizes[objectID] = size
}
}
log.Trace("Loaded %d items from packfiles", i)
return nil
}
// loadObjectsSizesViaCatFile uses hashes from objectIDs and runs `git cat-file -s` in 10 workers to return each object sizes
// Objects for which size is already loaded are skipped
// can't use `git cat-file --batch-check` here as it only provides data from git DB before the commit applied and has no knowledge on new commit objects
func loadObjectsSizesViaCatFile(ctx *gitea_context.PrivateContext, opts *git.RunOpts, objectIDs []string, objectsSizes map[string]int64) error {
// This is the number of workers that will simultaneously process CalculateSizeOfObject.
const numWorkers = 10
var wg sync.WaitGroup
var mu sync.Mutex
// Prepare numWorker slices to store the work
reducedObjectIDs := make([][]string, numWorkers)
for i := 0; i < numWorkers; i++ {
reducedObjectIDs[i] = make([]string, 0, len(objectIDs)/numWorkers+1)
}
// Loop over all objectIDs and find which ones are missing size information
i := 0
for _, objectID := range objectIDs {
_, exists := objectsSizes[objectID]
// If object doesn't yet have size in objectsSizes add it for further processing
if !exists {
reducedObjectIDs[i%numWorkers] = append(reducedObjectIDs[i%numWorkers], objectID)
i++
}
}
// Start workers and determine size using `git cat-file -s` store in objectsSizes cache
for w := 1; w <= numWorkers; w++ {
wg.Add(1)
go func(reducedObjectIDs *[]string) {
defer wg.Done()
for _, objectID := range *reducedObjectIDs {
ctx := ctx
// Create a copy of opts to allow change of the Env property
tsopts := *opts
// Ensure that each worker has its own copy of the Env environment to prevent races
tsopts.Env = append([]string(nil), opts.Env...)
objectSize := calculateSizeOfObject(ctx, &tsopts, objectID)
mu.Lock() // Protecting shared resource
objectsSizes[objectID] = objectSize
mu.Unlock() // Releasing shared resource for other goroutines
}
}(&reducedObjectIDs[(w-1)%numWorkers])
}
// Wait for all workers to finish processing.
wg.Wait()
return nil
}
// loadObjectsSizesViaBatch uses hashes from objectIDs and uses pre-opened `git cat-file --batch-check` command to slice and return each object sizes
// This function can't be used for new commit objects.
// It speeds up loading object sizes from existing git database of the repository avoiding
// multiple `git cat-files -s`
func loadObjectsSizesViaBatch(ctx *gitea_context.PrivateContext, repoPath string, objectIDs []string, objectsSizes map[string]int64) error {
var i int32
reducedObjectIDs := make([]string, 0, len(objectIDs))
// Loop over all objectIDs and find which ones are missing size information
for _, objectID := range objectIDs {
_, exists := objectsSizes[objectID]
// If object doesn't yet have size in objectsSizes add it for further processing
if !exists {
reducedObjectIDs = append(reducedObjectIDs, objectID)
}
}
wr, rd, cancel := git.CatFileBatchCheck(ctx, repoPath)
defer cancel()
for _, commitID := range reducedObjectIDs {
_, err := wr.Write([]byte(commitID + "\n"))
if err != nil {
return err
}
i++
line, err := rd.ReadString('\n')
if err != nil {
return err
}
if len(line) == 1 {
line, err = rd.ReadString('\n')
if err != nil {
return err
}
}
fields := strings.Fields(line)
objectID := fields[0]
if len(fields) < 3 || len(fields) > 3 {
log.Trace("String '%s' does not contain size ignored %s: %v", line, objectID, err)
continue
}
sizeStr := fields[2]
size, err := parseSize(sizeStr)
if err != nil {
log.Trace("String '%s' Failed to parse size for object %s: %v", line, objectID, err)
continue
}
objectsSizes[objectID] = size
}
return nil
}
// parseSize parses the object size from a string
func parseSize(sizeStr string) (int64, error) {
size, err := strconv.ParseInt(sizeStr, 10, 64)
if err != nil {
return 0, fmt.Errorf("failed to parse object size: %w", err)
}
return size, nil
}
// HookPreReceive checks whether a individual commit is acceptable
func HookPreReceive(ctx *gitea_context.PrivateContext) {
startTime := time.Now()
opts := web.GetForm(ctx).(*private.HookOptions)
ourCtx := &preReceiveContext{
@ -111,12 +382,164 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) {
opts: opts,
}
repo := ourCtx.Repo.Repository
var addedSize int64
var removedSize int64
var isRepoOversized bool
var pushSize *git.CountObject
var repoSize *git.CountObject
var err error
var duration time.Duration
if repo.IsRepoSizeLimitEnabled() {
// Calculating total size of the repo using `git count-objects`
repoSize, err = git.CountObjects(ctx, repo.RepoPath())
if err != nil {
log.Error("Unable to get repository size with env %v: %s Error: %v", repo.RepoPath(), ourCtx.env, err)
ctx.JSON(http.StatusInternalServerError, map[string]interface{}{
"err": err.Error(),
})
return
}
// Calculating total size of the push using `git count-objects`
pushSize, err = git.CountObjectsWithEnv(ctx, repo.RepoPath(), ourCtx.env)
if err != nil {
log.Error("Unable to get push size with env %v: %s Error: %v", repo.RepoPath(), ourCtx.env, err)
ctx.JSON(http.StatusInternalServerError, map[string]interface{}{
"err": err.Error(),
})
return
}
// Cache whether the repository would breach the size limit after the operation
isRepoOversized = repo.IsRepoSizeOversized(pushSize.Size + pushSize.SizePack)
log.Warn("Push counts %+v", pushSize)
log.Warn("Repo counts %+v", repoSize)
}
// Iterate across the provided old commit IDs
for i := range opts.OldCommitIDs {
oldCommitID := opts.OldCommitIDs[i]
newCommitID := opts.NewCommitIDs[i]
refFullName := opts.RefFullNames[i]
log.Trace("Processing old commit: %s, new commit: %s, ref: %s", oldCommitID, newCommitID, refFullName)
// If operation is in potential breach of size limit prepare data for analysis
if isRepoOversized {
var gitObjects string
var error error
// Create cache of objects in old commit
// if oldCommitID all 0 then it's a fresh repository on gitea server and all git operations on such oldCommitID would fail
if oldCommitID != "0000000000000000000000000000000000000000" {
gitObjects, _, err = git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(oldCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env})
if err != nil {
log.Error("Unable to list objects in old commit: %s in %-v Error: %v", oldCommitID, repo, err)
ctx.JSON(http.StatusInternalServerError, private.Response{
Err: fmt.Sprintf("Fail to list objects in old commit: %v", err),
})
return
}
}
commitObjectsSizes := make(map[string]int64)
oldCommitObjects := convertObjectsToMap(gitObjects)
objectIDs := convertObjectsToSlice(gitObjects)
// Create cache of objects that are in the repository but not part of old or new commit
// if oldCommitID all 0 then it's a fresh repository on gitea server and all git operations on such oldCommitID would fail
if oldCommitID == "0000000000000000000000000000000000000000" {
gitObjects, _, err = git.NewCommand(ctx, "rev-list", "--objects", "--all").AddDynamicArguments("^" + newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env})
if err != nil {
log.Error("Unable to list objects in the repo that are missing from both old %s and new %s commits in %-v Error: %v", oldCommitID, newCommitID, repo, err)
ctx.JSON(http.StatusInternalServerError, private.Response{
Err: fmt.Sprintf("Fail to list objects missing from both old and new commits: %v", err),
})
return
}
} else {
gitObjects, _, err = git.NewCommand(ctx, "rev-list", "--objects", "--all").AddDynamicArguments("^"+oldCommitID, "^"+newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env})
if err != nil {
log.Error("Unable to list objects in the repo that are missing from both old %s and new %s commits in %-v Error: %v", oldCommitID, newCommitID, repo, err)
ctx.JSON(http.StatusInternalServerError, private.Response{
Err: fmt.Sprintf("Fail to list objects missing from both old and new commits: %v", err),
})
return
}
}
otherCommitObjects := convertObjectsToMap(gitObjects)
objectIDs = append(objectIDs, convertObjectsToSlice(gitObjects)...)
// Unfortunately `git cat-file --check-batch` shows full object size
// so we would load compressed sizes from pack file via `git verify-pack -v` if there are pack files in repo
// The result would still miss items that are loose as individual objects (not part of pack files)
if repoSize.InPack > 0 {
error = loadObjectSizesFromPack(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: nil}, objectIDs, commitObjectsSizes)
if error != nil {
log.Error("Unable to get sizes of objects from the pack in %-v Error: %v", repo, error)
ctx.JSON(http.StatusInternalServerError, private.Response{
Err: fmt.Sprintf("Fail to get sizes of objects in repo: %v", err),
})
return
}
}
// Load loose objects that are missing
error = loadObjectsSizesViaBatch(ctx, repo.RepoPath(), objectIDs, commitObjectsSizes)
if error != nil {
log.Error("Unable to get sizes of objects that are missing in both old %s and new commits %s in %-v Error: %v", oldCommitID, newCommitID, repo, error)
ctx.JSON(http.StatusInternalServerError, private.Response{
Err: fmt.Sprintf("Fail to get sizes of objects missing in both old and new commit and those in old commit: %v", err),
})
return
}
// Create cache of objects in new commit
gitObjects, _, err = git.NewCommand(ctx, "rev-list", "--objects").AddDynamicArguments(newCommitID).RunStdString(&git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env})
if err != nil {
log.Error("Unable to list objects in new commit %s in %-v Error: %v", newCommitID, repo, err)
ctx.JSON(http.StatusInternalServerError, private.Response{
Err: fmt.Sprintf("Fail to list objects in new commit: %v", err),
})
return
}
newCommitObjects := convertObjectsToMap(gitObjects)
objectIDs = convertObjectsToSlice(gitObjects)
// Unfortunately `git cat-file --check-batch` doesn't work on objects not yet accepted into git database
// so the sizes will be calculated through pack file `git verify-pack -v` if there are pack files
// The result would still miss items that were sent loose as individual objects (not part of pack files)
if pushSize.InPack > 0 {
error = loadObjectSizesFromPack(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}, objectIDs, commitObjectsSizes)
if error != nil {
log.Error("Unable to get sizes of objects from the pack in new commit %s in %-v Error: %v", newCommitID, repo, error)
ctx.JSON(http.StatusInternalServerError, private.Response{
Err: fmt.Sprintf("Fail to get sizes of objects in new commit: %v", err),
})
return
}
}
// After loading everything we could from pack file, objects could have been sent as loose bunch as well
// We need to load them individually with `git cat-file -s` on any object that is missing from accumulated size cache commitObjectsSizes
error = loadObjectsSizesViaCatFile(ctx, &git.RunOpts{Dir: repo.RepoPath(), Env: ourCtx.env}, objectIDs, commitObjectsSizes)
if error != nil {
log.Error("Unable to get sizes of objects in new commit %s in %-v Error: %v", newCommitID, repo, error)
ctx.JSON(http.StatusInternalServerError, private.Response{
Err: fmt.Sprintf("Fail to get sizes of objects in new commit: %v", err),
})
return
}
// Calculate size that was added and removed by the new commit
addedSize, removedSize = calculateSizeOfObjectsFromCache(newCommitObjects, oldCommitObjects, otherCommitObjects, commitObjectsSizes)
}
switch {
case refFullName.IsBranch():
preReceiveBranch(ourCtx, oldCommitID, newCommitID, refFullName)
@ -132,6 +555,20 @@ func HookPreReceive(ctx *gitea_context.PrivateContext) {
}
}
if repo.IsRepoSizeLimitEnabled() {
duration = time.Since(startTime)
log.Warn("During size checking - Addition in size is: %d, removal in size is: %d, limit size: %d, push size: %d, repo size: %d. Took %s seconds.", addedSize, removedSize, repo.GetActualSizeLimit(), pushSize.Size+pushSize.SizePack, repo.GitSize, duration)
}
// If total of commits add more size then they remove and we are in a potential breach of size limit -- abort
if (addedSize > removedSize) && isRepoOversized {
log.Warn("Forbidden: new repo size %s would be over limitation of %s. Push size: %s. Took %s seconds. addedSize: %s. removedSize: %s", base.FileSize(repo.GitSize+addedSize-removedSize), base.FileSize(repo.GetActualSizeLimit()), base.FileSize(pushSize.Size+pushSize.SizePack), duration, base.FileSize(addedSize), base.FileSize(removedSize))
ctx.JSON(http.StatusForbidden, private.Response{
UserMsg: fmt.Sprintf("New repository size is over limitation of %s", base.FileSize(repo.GetActualSizeLimit())),
})
return
}
ctx.PlainText(http.StatusOK, "ok")
}

View File

@ -16,8 +16,10 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/modules/web"
"code.gitea.io/gitea/routers/web/explore"
"code.gitea.io/gitea/services/context"
"code.gitea.io/gitea/services/forms"
repo_service "code.gitea.io/gitea/services/repository"
)
@ -31,6 +33,9 @@ func Repos(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("admin.repositories")
ctx.Data["PageIsAdminRepositories"] = true
ctx.Data["EnableSizeLimit"] = setting.EnableSizeLimit
ctx.Data["RepoSizeLimit"] = base.FileSize(setting.RepoSizeLimit)
explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{
Private: true,
PageSize: setting.UI.Admin.RepoPagingNum,
@ -39,6 +44,54 @@ func Repos(ctx *context.Context) {
})
}
func UpdateRepoPost(ctx *context.Context) {
temp := web.GetForm(ctx)
if temp == nil {
ctx.Data["Err_Repo_Size_Limit"] = ""
explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{
Private: true,
PageSize: setting.UI.Admin.RepoPagingNum,
TplName: tplRepos,
OnlyShowRelevant: false,
})
return
}
form := temp.(*forms.UpdateGlobalRepoFrom)
ctx.Data["Title"] = ctx.Tr("admin.repositories")
ctx.Data["PageIsAdminRepositories"] = true
repoSizeLimit, err := base.GetFileSize(form.RepoSizeLimit)
ctx.Data["EnableSizeLimit"] = form.EnableSizeLimit
ctx.Data["RepoSizeLimit"] = form.RepoSizeLimit
if err != nil {
ctx.Data["Err_Repo_Size_Limit"] = err.Error()
explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{
Private: true,
PageSize: setting.UI.Admin.RepoPagingNum,
TplName: tplRepos,
OnlyShowRelevant: false,
})
return
}
err = setting.SaveGlobalRepositorySetting(form.EnableSizeLimit, repoSizeLimit)
if err != nil {
ctx.Data["Err_Repo_Size_Save"] = err.Error()
explore.RenderRepoSearch(ctx, &explore.RepoSearchOptions{
Private: true,
PageSize: setting.UI.Admin.RepoPagingNum,
TplName: tplRepos,
OnlyShowRelevant: false,
})
return
}
ctx.Flash.Success(ctx.Tr("admin.config.repository_setting_success"))
ctx.Redirect(setting.AppSubURL + "/admin/repos")
}
// DeleteRepo delete one repository
func DeleteRepo(ctx *context.Context) {
repo, err := repo_model.GetRepositoryByID(ctx, ctx.FormInt64("id"))

View File

@ -0,0 +1,26 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package admin
import (
"testing"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/services/contexttest"
"github.com/stretchr/testify/assert"
)
func TestUpdateRepoPost(t *testing.T) {
unittest.PrepareTestEnv(t)
ctx, _ := contexttest.MockContext(t, "admin/repos")
contexttest.LoadUser(t, ctx, 1)
ctx.Req.Form.Set("enable_size_limit", "on")
ctx.Req.Form.Set("repo_size_limit", "222 kcmcm")
UpdateRepoPost(ctx)
assert.NotEmpty(t, ctx.Flash.ErrorMsg)
}

View File

@ -160,6 +160,18 @@ func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) {
}
ctx.Data["Page"] = pager
if ctx.Data["Err_Repo_Size_Limit"] != nil {
ctx.RenderWithErr(ctx.Tr("admin.config.invalid_repo_size", ctx.Data["Err_Repo_Size_Limit"].(string)),
opts.TplName, nil)
return
}
if ctx.Data["Err_Repo_Size_Save"] != nil {
ctx.RenderWithErr(ctx.Tr("admin.config.save_repo_size_setting_failed", ctx.Data["Err_Repo_Size_Save"].(string)),
opts.TplName, nil)
return
}
ctx.HTML(http.StatusOK, opts.TplName)
}

View File

@ -293,6 +293,7 @@ func CreatePost(ctx *context.Context) {
IsTemplate: form.Template,
TrustModel: repo_model.DefaultTrustModel,
ObjectFormatName: form.ObjectFormatName,
SizeLimit: form.SizeLimit,
})
if err == nil {
log.Trace("Repository created [%d]: %s/%s", repo.ID, ctxUser.Name, repo.Name)

View File

@ -61,6 +61,9 @@ func SettingsCtxData(ctx *context.Context) {
ctx.Data["DisableNewPushMirrors"] = setting.Mirror.DisableNewPush
ctx.Data["DefaultMirrorInterval"] = setting.Mirror.DefaultInterval
ctx.Data["MinimumMirrorInterval"] = setting.Mirror.MinInterval
ctx.Data["Err_RepoSize"] = ctx.Repo.Repository.IsRepoSizeOversized(ctx.Repo.Repository.GetActualSizeLimit() / 10) // less than 10% left
ctx.Data["ActualSizeLimit"] = ctx.Repo.Repository.GetActualSizeLimit()
ctx.Data["EnableSizeLimit"] = setting.EnableSizeLimit
signing, _ := asymkey_service.SigningKey(ctx, ctx.Repo.Repository.RepoPath())
ctx.Data["SigningKeyAvailable"] = len(signing) > 0
@ -113,6 +116,7 @@ func SettingsPost(ctx *context.Context) {
ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
repo := ctx.Repo.Repository
ctx.Data["Err_RepoSize"] = repo.IsRepoSizeOversized(repo.SizeLimit / 10) // less than 10% left
switch ctx.FormString("action") {
case "update":
@ -170,6 +174,19 @@ func SettingsPost(ctx *context.Context) {
form.Private = repo.BaseRepo.IsPrivate || repo.BaseRepo.Owner.Visibility == structs.VisibleTypePrivate
}
if form.RepoSizeLimit < 0 {
ctx.Data["Err_RepoSizeLimit"] = true
ctx.RenderWithErr(ctx.Tr("repo.form.repo_size_limit_negative"), tplSettingsOptions, &form)
return
}
if !ctx.Doer.IsAdmin && repo.SizeLimit != form.RepoSizeLimit {
ctx.Data["Err_RepoSizeLimit"] = true
ctx.RenderWithErr(ctx.Tr("repo.form.repo_size_limit_only_by_admins"), tplSettingsOptions, &form)
return
}
repo.SizeLimit = form.RepoSizeLimit
if err := repo_service.UpdateRepository(ctx, repo, false); err != nil {
ctx.ServerError("UpdateRepository", err)
return

View File

@ -738,6 +738,7 @@ func registerRoutes(m *web.Router) {
m.Get("", admin.Repos)
m.Combo("/unadopted").Get(admin.UnadoptedRepos).Post(admin.AdoptOrDeleteRepository)
m.Post("/delete", admin.DeleteRepo)
m.Post("", web.Bind(forms.UpdateGlobalRepoFrom{}), admin.UpdateRepoPost)
})
m.Group("/packages", func() {

View File

@ -21,6 +21,12 @@ import (
"gitea.com/go-chi/binding"
)
// UpdateGlobalRepoFrom for updating global repository setting
type UpdateGlobalRepoFrom struct {
RepoSizeLimit string
EnableSizeLimit bool
}
// CreateRepoForm form for creating repository
type CreateRepoForm struct {
UID int64 `binding:"Required"`
@ -46,6 +52,7 @@ type CreateRepoForm struct {
ForkSingleBranch string
ObjectFormatName string
SizeLimit int64
}
// Validate validates the fields
@ -131,6 +138,7 @@ type RepoSettingForm struct {
Private bool
Template bool
EnablePrune bool
RepoSizeLimit int64
// Advanced settings
EnableCode bool

View File

@ -45,6 +45,7 @@ type CreateRepoOptions struct {
TrustModel repo_model.TrustModelType
MirrorInterval string
ObjectFormatName string
SizeLimit int64
}
func prepareRepoCommit(ctx context.Context, repo *repo_model.Repository, tmpDir, repoPath string, opts CreateRepoOptions) error {
@ -234,6 +235,7 @@ func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opt
Status: opts.Status,
IsEmpty: !opts.AutoInit,
TrustModel: opts.TrustModel,
SizeLimit: opts.SizeLimit,
IsMirror: opts.IsMirror,
DefaultBranch: opts.DefaultBranch,
DefaultWikiBranch: setting.Repository.DefaultBranch,

View File

@ -1,5 +1,26 @@
{{template "admin/layout_head" (dict "ctxData" . "pageClass" "admin")}}
<div class="admin-setting-content">
<h4 class="ui top attached header">
{{.locale.Tr "admin.config.repository_config"}}
</h4>
<div class="ui attached segment">
<form class="ui form" action="{{.Link}}" method="post">
{{.CsrfTokenHtml}}
<div class="inline field">
<label>{{.locale.Tr "admin.config.enable_size_limit"}}</label>
<div class="ui checkbox">
<input name="enable_size_limit" type="checkbox" {{if .EnableSizeLimit}}checked{{end}}>
</div>
</div>
<div class="inline field {{if .Err_Repo_Size_Limit}}error{{end}}">
<label for="repo_size_limit">{{.locale.Tr "admin.config.repo_size_limit"}}</label>
<input id="repo_size_limit" name="repo_size_limit" value="{{.RepoSizeLimit}}">
</div>
<div class="field">
<button class="ui green button">{{$.locale.Tr "repo.settings.update_settings"}}</button>
</div>
</form>
</div>
<h4 class="ui top attached header">
{{ctx.Locale.Tr "admin.repos.repo_manage_panel"}} ({{ctx.Locale.Tr "admin.total" .Total}})
<div class="ui right">

View File

@ -13,8 +13,18 @@
<input name="repo_name" value="{{.Repository.Name}}" data-repo-name="{{.Repository.Name}}" autofocus required>
</div>
<div class="inline field">
<label>{{ctx.Locale.Tr "repo.repo_size"}}</label>
<span {{if not (eq .Repository.Size 0)}} data-tooltip-content="{{.Repository.SizeDetailsString}}"{{end}}>{{FileSize .Repository.Size}}</span>
<label>{{ctx.locale.Tr "repo.repo_size"}}</label>
<span {{if .Err_RepoSize}}class="ui text red"{{end}} {{if not (eq .Repository.Size 0)}} data-tooltip-content="{{.Repository.SizeDetailsString}}"{{end}}>{{FileSize .Repository.Size}}
{{if and .ActualSizeLimit .EnableSizeLimit}}
/{{FileSize .ActualSizeLimit}}
{{end}}
</span>
</div>
<div class="field {{if .Err_RepoSizeLimit}}error{{end}}" {{if not .IsAdmin}}style="display:none;"{{end}}>
<label for="repo_size_limit">{{.locale.Tr "repo.repo_size_limit"}}</label>
<input id="repo_size_limit" name="repo_size_limit"
{{if not .EnableSizeLimit}}class="ui text light grey"{{end}}
type="number" value="{{.Repository.SizeLimit}}" data-repo-size-limit="{{.Repository.SizeLimit}}">
</div>
<div class="inline field">
<label>{{ctx.Locale.Tr "repo.template"}}</label>

View File

@ -20188,6 +20188,12 @@
"type": "string",
"x-go-name": "Readme"
},
"size_limit": {
"description": "SizeLimit of the repository",
"type": "integer",
"format": "int64",
"x-go-name": "SizeLimit"
},
"template": {
"description": "Whether the repository is template",
"type": "boolean",
@ -21205,6 +21211,12 @@
"type": "string",
"x-go-name": "ProjectsMode"
},
"size_limit": {
"description": "SizeLimit of the repository.",
"type": "integer",
"format": "int64",
"x-go-name": "SizeLimit"
},
"template": {
"description": "either `true` to make this repository a template or `false` to make it a normal repository",
"type": "boolean",

View File

@ -133,6 +133,23 @@ func doAPIForkRepository(ctx APITestContext, username string, callback ...func(*
}
}
func doAPIGetRepositorySize(ctx APITestContext, owner, repo string) func(*testing.T) int64 {
return func(t *testing.T) int64 {
urlStr := fmt.Sprintf("/api/v1/repos/%s/%s", ctx.Username, ctx.Reponame)
req := NewRequest(t, "GET", urlStr).
AddTokenAuth(ctx.Token)
if ctx.ExpectedCode != 0 {
ctx.Session.MakeRequest(t, req, ctx.ExpectedCode)
}
resp := ctx.Session.MakeRequest(t, req, http.StatusOK)
var repository api.Repository
DecodeJSON(t, resp, &repository)
return int64(repository.Size)
}
}
func doAPIGetRepository(ctx APITestContext, callback ...func(*testing.T, api.Repository)) func(*testing.T) {
return func(t *testing.T) {
req := NewRequest(t, "GET", fmt.Sprintf("/api/v1/repos/%s/%s", ctx.Username, ctx.Reponame)).
@ -448,3 +465,18 @@ func doAPIAddRepoToOrganizationTeam(ctx APITestContext, teamID int64, orgName, r
ctx.Session.MakeRequest(t, req, http.StatusNoContent)
}
}
func doAPISetRepoSizeLimit(ctx APITestContext, owner, repo string, size int64) func(*testing.T) {
return func(t *testing.T) {
urlStr := fmt.Sprintf("/api/v1/repos/%s/%s",
owner, repo)
req := NewRequestWithJSON(t, http.MethodPatch, urlStr, &api.EditRepoOption{SizeLimit: &size}).
AddTokenAuth(ctx.Token)
if ctx.ExpectedCode != 0 {
ctx.Session.MakeRequest(t, req, ctx.ExpectedCode)
return
}
ctx.Session.MakeRequest(t, req, 200)
}
}

View File

@ -14,6 +14,7 @@ import (
"path"
"path/filepath"
"strconv"
"strings"
"testing"
"time"
@ -27,6 +28,7 @@ import (
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/lfs"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
api "code.gitea.io/gitea/modules/structs"
gitea_context "code.gitea.io/gitea/services/context"
@ -62,6 +64,8 @@ func testGit(t *testing.T, u *url.URL) {
dstPath := t.TempDir()
dstForkedPath := t.TempDir()
t.Run("CreateRepoInDifferentUser", doAPICreateRepository(forkedUserCtx, false))
t.Run("AddUserAsCollaborator", doAPIAddCollaborator(forkedUserCtx, httpContext.Username, perm.AccessModeRead))
@ -81,6 +85,39 @@ func testGit(t *testing.T, u *url.URL) {
rawTest(t, &httpContext, little, big, littleLFS, bigLFS)
mediaTest(t, &httpContext, little, big, littleLFS, bigLFS)
t.Run("SizeLimit", func(t *testing.T) {
setting.SaveGlobalRepositorySetting(true, 0)
t.Run("Under", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
doCommitAndPush(t, littleSize, dstPath, "data-file-")
})
t.Run("Over", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
u.Path = forkedUserCtx.GitPath()
u.User = url.UserPassword(forkedUserCtx.Username, userPassword)
t.Run("Clone", doGitClone(dstForkedPath, u))
t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, littleSize))
doCommitAndPushWithExpectedError(t, bigSize, dstForkedPath, "data-file-")
})
t.Run("UnderAfterResize", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
t.Run("APISetRepoSizeLimit", doAPISetRepoSizeLimit(forkedUserCtx, forkedUserCtx.Username, forkedUserCtx.Reponame, bigSize*10))
doCommitAndPush(t, littleSize, dstPath, "data-file-")
})
t.Run("Deletion", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
doCommitAndPush(t, littleSize, dstPath, "data-file-")
bigFileName := doCommitAndPush(t, bigSize, dstPath, "data-file-")
oldRepoSize := doGetRemoteRepoSizeViaAPI(t, forkedUserCtx)
lastCommitID := doGetAddCommitID(t, dstPath, bigFileName)
doDeleteAndPush(t, dstPath, bigFileName)
doRebaseCommitAndPush(t, dstPath, lastCommitID)
newRepoSize := doGetRemoteRepoSizeViaAPI(t, forkedUserCtx)
assert.LessOrEqual(t, newRepoSize, oldRepoSize)
setting.SaveGlobalRepositorySetting(false, 0)
})
})
t.Run("CreateAgitFlowPull", doCreateAgitFlowPull(dstPath, &httpContext, "master", "test/head"))
t.Run("CreateAgitFlowPull", doCreateAgitFlowPull(dstPath, &httpContext, "test/head"))
t.Run("BranchProtectMerge", doBranchProtectPRMerge(&httpContext, dstPath))
t.Run("AutoMerge", doAutoPRMerge(&httpContext, dstPath))
@ -92,6 +129,8 @@ func testGit(t *testing.T, u *url.URL) {
mediaTest(t, &forkedUserCtx, little, big, littleLFS, bigLFS)
})
u.Path = httpContext.GitPath()
u.User = url.UserPassword(username, userPassword)
t.Run("PushCreate", doPushCreate(httpContext, u))
})
t.Run("SSH", func(t *testing.T) {
@ -190,7 +229,9 @@ func commitAndPushTest(t *testing.T, dstPath, prefix string) (little, big string
defer tests.PrintCurrentTest(t)()
t.Run("Little", func(t *testing.T) {
defer tests.PrintCurrentTest(t)()
log.Error("before doCommitAndPush")
little = doCommitAndPush(t, littleSize, dstPath, prefix)
log.Error("after doCommitAndPush")
})
t.Run("Big", func(t *testing.T) {
if testing.Short() {
@ -291,6 +332,50 @@ func lockFileTest(t *testing.T, filename, repoPath string) {
assert.NoError(t, err)
}
func doGetRemoteRepoSizeViaAPI(t *testing.T, ctx APITestContext) int64 {
return doAPIGetRepositorySize(ctx, ctx.Username, ctx.Reponame)(t)
}
func doDeleteAndPush(t *testing.T, repoPath, filename string) {
_, _, err := git.NewCommand(git.DefaultContext, "rm").AddDashesAndList(filename).RunStdString(&git.RunOpts{Dir: repoPath}) // Delete
assert.NoError(t, err)
signature := git.Signature{
Email: "user2@example.com",
Name: "User Two",
When: time.Now(),
}
_, _, err = git.NewCommand(git.DefaultContext, "status").RunStdString(&git.RunOpts{Dir: repoPath})
assert.NoError(t, err)
err2 := git.CommitChanges(repoPath, git.CommitChangesOptions{ // Commit
Committer: &signature,
Author: &signature,
Message: "Delete Commit",
})
assert.NoError(t, err2)
_, _, err = git.NewCommand(git.DefaultContext, "status").RunStdString(&git.RunOpts{Dir: repoPath})
assert.NoError(t, err)
_, _, err = git.NewCommand(git.DefaultContext, "push", "origin", "master").RunStdString(&git.RunOpts{Dir: repoPath}) // Push
assert.NoError(t, err)
}
func doGetAddCommitID(t *testing.T, repoPath, filename string) string {
output, _, err := git.NewCommand(git.DefaultContext, "log", "origin", "master").RunStdString(&git.RunOpts{Dir: repoPath}) // Push
assert.NoError(t, err)
list := strings.Fields(output)
assert.LessOrEqual(t, 2, len(list))
return list[1]
}
func doRebaseCommitAndPush(t *testing.T, repoPath, commitID string) {
command := git.NewCommand(git.DefaultContext, "rebase", "--interactive").AddDashesAndList(commitID)
env := os.Environ()
env = append(env, "GIT_SEQUENCE_EDITOR=true")
_, _, err := command.RunStdString(&git.RunOpts{Dir: repoPath, Env: env}) // Push
assert.NoError(t, err)
_, _, err = git.NewCommand(git.DefaultContext, "push", "origin", "master", "-f").RunStdString(&git.RunOpts{Dir: repoPath}) // Push
assert.NoError(t, err)
}
func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string {
name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix)
assert.NoError(t, err)
@ -299,6 +384,14 @@ func doCommitAndPush(t *testing.T, size int, repoPath, prefix string) string {
return name
}
func doCommitAndPushWithExpectedError(t *testing.T, size int, repoPath, prefix string) string {
name, err := generateCommitWithNewData(size, repoPath, "user2@example.com", "User Two", prefix)
assert.NoError(t, err)
_, _, err = git.NewCommand(git.DefaultContext, "push", "origin", "master").RunStdString(&git.RunOpts{Dir: repoPath}) // Push
assert.Error(t, err)
return name
}
func generateCommitWithNewData(size int, repoPath, email, fullName, prefix string) (string, error) {
// Generate random file
bufSize := 4 * 1024