Move some repository methods from models to modules/repository (#9353)

* Move some repository methods from models to modules/repository

* fix test

cmd/admin.go

@@ -16,6 +16,7 @@ import (
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	pwd "code.gitea.io/gitea/modules/password"
+	"code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
 
 	"github.com/urfave/cli"
@@ -373,7 +374,7 @@ func runRepoSyncReleases(c *cli.Context) error {
 			}
 			log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum)
 
-			if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil {
+			if err = repository.SyncReleasesWithTags(repo, gitRepo); err != nil {
 				log.Warn(" SyncReleasesWithTags: %v", err)
 				gitRepo.Close()
 				continue

models/migrations/v39.go

@@ -10,6 +10,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/repository"
 
 	"xorm.io/xorm"
 )
@@ -44,7 +45,7 @@ func releaseAddColumnIsTagAndSyncTags(x *xorm.Engine) error {
 				continue
 			}
 
-			if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil {
+			if err = repository.SyncReleasesWithTags(repo, gitRepo); err != nil {
 				log.Warn("SyncReleasesWithTags: %v", err)
 			}
 			gitRepo.Close()

models/release.go

@@ -10,7 +10,6 @@ import (
 	"sort"
 	"strings"
 
-	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 	api "code.gitea.io/gitea/modules/structs"
@@ -318,49 +317,6 @@ func DeleteReleaseByID(id int64) error {
 	return err
 }
 
-// SyncReleasesWithTags synchronizes release table with repository tags
-func SyncReleasesWithTags(repo *Repository, gitRepo *git.Repository) error {
-	existingRelTags := make(map[string]struct{})
-	opts := FindReleasesOptions{IncludeDrafts: true, IncludeTags: true}
-	for page := 1; ; page++ {
-		rels, err := GetReleasesByRepoID(repo.ID, opts, page, 100)
-		if err != nil {
-			return fmt.Errorf("GetReleasesByRepoID: %v", err)
-		}
-		if len(rels) == 0 {
-			break
-		}
-		for _, rel := range rels {
-			if rel.IsDraft {
-				continue
-			}
-			commitID, err := gitRepo.GetTagCommitID(rel.TagName)
-			if err != nil && !git.IsErrNotExist(err) {
-				return fmt.Errorf("GetTagCommitID: %v", err)
-			}
-			if git.IsErrNotExist(err) || commitID != rel.Sha1 {
-				if err := PushUpdateDeleteTag(repo, rel.TagName); err != nil {
-					return fmt.Errorf("PushUpdateDeleteTag: %v", err)
-				}
-			} else {
-				existingRelTags[strings.ToLower(rel.TagName)] = struct{}{}
-			}
-		}
-	}
-	tags, err := gitRepo.GetTags()
-	if err != nil {
-		return fmt.Errorf("GetTags: %v", err)
-	}
-	for _, tagName := range tags {
-		if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok {
-			if err := PushUpdateAddTag(repo, gitRepo, tagName); err != nil {
-				return fmt.Errorf("pushUpdateAddTag: %v", err)
-			}
-		}
-	}
-	return nil
-}
-
 // UpdateReleasesMigrationsByType updates all migrated repositories' releases from gitServiceType to replace originalAuthorID to posterID
 func UpdateReleasesMigrationsByType(gitServiceType structs.GitServiceType, originalAuthorID string, posterID int64) error {
 	_, err := x.Table("release").

models/repo.go

@@ -39,7 +39,6 @@ import (
 
 	"github.com/mcuadros/go-version"
 	"github.com/unknwon/com"
-	ini "gopkg.in/ini.v1"
 	"xorm.io/builder"
 )
 
@@ -941,25 +940,6 @@ func (repo *Repository) CloneLink() (cl *CloneLink) {
 	return repo.cloneLink(x, false)
 }
 
-/*
-	GitHub, GitLab, Gogs: *.wiki.git
-	BitBucket: *.git/wiki
-*/
-var commonWikiURLSuffixes = []string{".wiki.git", ".git/wiki"}
-
-// wikiRemoteURL returns accessible repository URL for wiki if exists.
-// Otherwise, it returns an empty string.
-func wikiRemoteURL(remote string) string {
-	remote = strings.TrimSuffix(remote, ".git")
-	for _, suffix := range commonWikiURLSuffixes {
-		wikiURL := remote + suffix
-		if git.IsRepoURLAccessible(wikiURL) {
-			return wikiURL
-		}
-	}
-	return ""
-}
-
 // CheckCreateRepository check if could created a repository
 func CheckCreateRepository(doer, u *User, name string) error {
 	if !doer.CanCreateRepo() {
@@ -979,118 +959,9 @@ func CheckCreateRepository(doer, u *User, name string) error {
 	return nil
 }
 
-// MigrateRepositoryGitData starts migrating git related data after created migrating repository
+// CreateDelegateHooks creates all the hooks scripts for the repo
-func MigrateRepositoryGitData(doer, u *User, repo *Repository, opts api.MigrateRepoOption) (*Repository, error) {
+func CreateDelegateHooks(repoPath string) error {
-	repoPath := RepoPath(u.Name, opts.RepoName)
+	return createDelegateHooks(repoPath)
-
-	if u.IsOrganization() {
-		t, err := u.GetOwnerTeam()
-		if err != nil {
-			return nil, err
-		}
-		repo.NumWatches = t.NumMembers
-	} else {
-		repo.NumWatches = 1
-	}
-
-	migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second
-
-	var err error
-	if err = os.RemoveAll(repoPath); err != nil {
-		return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err)
-	}
-
-	if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{
-		Mirror:  true,
-		Quiet:   true,
-		Timeout: migrateTimeout,
-	}); err != nil {
-		return repo, fmt.Errorf("Clone: %v", err)
-	}
-
-	if opts.Wiki {
-		wikiPath := WikiPath(u.Name, opts.RepoName)
-		wikiRemotePath := wikiRemoteURL(opts.CloneAddr)
-		if len(wikiRemotePath) > 0 {
-			if err := os.RemoveAll(wikiPath); err != nil {
-				return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
-			}
-
-			if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
-				Mirror:  true,
-				Quiet:   true,
-				Timeout: migrateTimeout,
-				Branch:  "master",
-			}); err != nil {
-				log.Warn("Clone wiki: %v", err)
-				if err := os.RemoveAll(wikiPath); err != nil {
-					return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
-				}
-			}
-		}
-	}
-
-	gitRepo, err := git.OpenRepository(repoPath)
-	if err != nil {
-		return repo, fmt.Errorf("OpenRepository: %v", err)
-	}
-	defer gitRepo.Close()
-
-	repo.IsEmpty, err = gitRepo.IsEmpty()
-	if err != nil {
-		return repo, fmt.Errorf("git.IsEmpty: %v", err)
-	}
-
-	if !opts.Releases && !repo.IsEmpty {
-		// Try to get HEAD branch and set it as default branch.
-		headBranch, err := gitRepo.GetHEADBranch()
-		if err != nil {
-			return repo, fmt.Errorf("GetHEADBranch: %v", err)
-		}
-		if headBranch != nil {
-			repo.DefaultBranch = headBranch.Name
-		}
-
-		if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
-			log.Error("Failed to synchronize tags to releases for repository: %v", err)
-		}
-	}
-
-	if err = repo.UpdateSize(); err != nil {
-		log.Error("Failed to update size for repository: %v", err)
-	}
-
-	if opts.Mirror {
-		if _, err = x.InsertOne(&Mirror{
-			RepoID:         repo.ID,
-			Interval:       setting.Mirror.DefaultInterval,
-			EnablePrune:    true,
-			NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval),
-		}); err != nil {
-			return repo, fmt.Errorf("InsertOne: %v", err)
-		}
-
-		repo.IsMirror = true
-		err = UpdateRepository(repo, false)
-	} else {
-		repo, err = CleanUpMigrateInfo(repo)
-	}
-
-	return repo, err
-}
-
-// cleanUpMigrateGitConfig removes mirror info which prevents "push --all".
-// This also removes possible user credentials.
-func cleanUpMigrateGitConfig(configPath string) error {
-	cfg, err := ini.Load(configPath)
-	if err != nil {
-		return fmt.Errorf("open config file: %v", err)
-	}
-	cfg.DeleteSection("remote \"origin\"")
-	if err = cfg.SaveToIndent(configPath, "\t"); err != nil {
-		return fmt.Errorf("save config file: %v", err)
-	}
-	return nil
 }
 
 // createDelegateHooks creates all the hooks scripts for the repo
@@ -1132,32 +1003,6 @@ func createDelegateHooks(repoPath string) (err error) {
 	return nil
 }
 
-// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
-func CleanUpMigrateInfo(repo *Repository) (*Repository, error) {
-	repoPath := repo.RepoPath()
-	if err := createDelegateHooks(repoPath); err != nil {
-		return repo, fmt.Errorf("createDelegateHooks: %v", err)
-	}
-	if repo.HasWiki() {
-		if err := createDelegateHooks(repo.WikiPath()); err != nil {
-			return repo, fmt.Errorf("createDelegateHooks.(wiki): %v", err)
-		}
-	}
-
-	_, err := git.NewCommand("remote", "rm", "origin").RunInDir(repoPath)
-	if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") {
-		return repo, fmt.Errorf("CleanUpMigrateInfo: %v", err)
-	}
-
-	if repo.HasWiki() {
-		if err := cleanUpMigrateGitConfig(path.Join(repo.WikiPath(), "config")); err != nil {
-			return repo, fmt.Errorf("cleanUpMigrateGitConfig (wiki): %v", err)
-		}
-	}
-
-	return repo, UpdateRepository(repo, false)
-}
-
 // initRepoCommit temporarily changes with work directory.
 func initRepoCommit(tmpPath string, u *User) (err error) {
 	commitTimeStr := time.Now().Format(time.RFC3339)

models/repo_mirror.go

@@ -97,3 +97,9 @@ func MirrorsIterate(f func(idx int, bean interface{}) error) error {
 		And("next_update_unix!=0").
 		Iterate(new(Mirror), f)
 }
+
+// InsertMirror inserts a mirror to database
+func InsertMirror(mirror *Mirror) error {
+	_, err := x.Insert(mirror)
+	return err
+}

modules/migrations/gitea.go

@@ -21,6 +21,7 @@ import (
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
@@ -111,7 +112,7 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
 		return err
 	}
 
-	r, err = models.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{
+	r, err = repository.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{
 		RepoName:       g.repoName,
 		Description:    repo.Description,
 		OriginalURL:    repo.OriginalURL,
@@ -294,7 +295,7 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
 
 // SyncTags syncs releases with tags in the database
 func (g *GiteaLocalUploader) SyncTags() error {
-	return models.SyncReleasesWithTags(g.repo, g.gitRepo)
+	return repository.SyncReleasesWithTags(g.repo, g.gitRepo)
 }
 
 // CreateIssues creates issues

modules/repository/repo.go

@@ -0,0 +1,223 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package repository
+
+import (
+	"fmt"
+	"os"
+	"path"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	api "code.gitea.io/gitea/modules/structs"
+	"code.gitea.io/gitea/modules/timeutil"
+	"gopkg.in/ini.v1"
+)
+
+/*
+	GitHub, GitLab, Gogs: *.wiki.git
+	BitBucket: *.git/wiki
+*/
+var commonWikiURLSuffixes = []string{".wiki.git", ".git/wiki"}
+
+// wikiRemoteURL returns accessible repository URL for wiki if exists.
+// Otherwise, it returns an empty string.
+func wikiRemoteURL(remote string) string {
+	remote = strings.TrimSuffix(remote, ".git")
+	for _, suffix := range commonWikiURLSuffixes {
+		wikiURL := remote + suffix
+		if git.IsRepoURLAccessible(wikiURL) {
+			return wikiURL
+		}
+	}
+	return ""
+}
+
+// MigrateRepositoryGitData starts migrating git related data after created migrating repository
+func MigrateRepositoryGitData(doer, u *models.User, repo *models.Repository, opts api.MigrateRepoOption) (*models.Repository, error) {
+	repoPath := models.RepoPath(u.Name, opts.RepoName)
+
+	if u.IsOrganization() {
+		t, err := u.GetOwnerTeam()
+		if err != nil {
+			return nil, err
+		}
+		repo.NumWatches = t.NumMembers
+	} else {
+		repo.NumWatches = 1
+	}
+
+	migrateTimeout := time.Duration(setting.Git.Timeout.Migrate) * time.Second
+
+	var err error
+	if err = os.RemoveAll(repoPath); err != nil {
+		return repo, fmt.Errorf("Failed to remove %s: %v", repoPath, err)
+	}
+
+	if err = git.Clone(opts.CloneAddr, repoPath, git.CloneRepoOptions{
+		Mirror:  true,
+		Quiet:   true,
+		Timeout: migrateTimeout,
+	}); err != nil {
+		return repo, fmt.Errorf("Clone: %v", err)
+	}
+
+	if opts.Wiki {
+		wikiPath := models.WikiPath(u.Name, opts.RepoName)
+		wikiRemotePath := wikiRemoteURL(opts.CloneAddr)
+		if len(wikiRemotePath) > 0 {
+			if err := os.RemoveAll(wikiPath); err != nil {
+				return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
+			}
+
+			if err = git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
+				Mirror:  true,
+				Quiet:   true,
+				Timeout: migrateTimeout,
+				Branch:  "master",
+			}); err != nil {
+				log.Warn("Clone wiki: %v", err)
+				if err := os.RemoveAll(wikiPath); err != nil {
+					return repo, fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
+				}
+			}
+		}
+	}
+
+	gitRepo, err := git.OpenRepository(repoPath)
+	if err != nil {
+		return repo, fmt.Errorf("OpenRepository: %v", err)
+	}
+	defer gitRepo.Close()
+
+	repo.IsEmpty, err = gitRepo.IsEmpty()
+	if err != nil {
+		return repo, fmt.Errorf("git.IsEmpty: %v", err)
+	}
+
+	if !opts.Releases && !repo.IsEmpty {
+		// Try to get HEAD branch and set it as default branch.
+		headBranch, err := gitRepo.GetHEADBranch()
+		if err != nil {
+			return repo, fmt.Errorf("GetHEADBranch: %v", err)
+		}
+		if headBranch != nil {
+			repo.DefaultBranch = headBranch.Name
+		}
+
+		if err = SyncReleasesWithTags(repo, gitRepo); err != nil {
+			log.Error("Failed to synchronize tags to releases for repository: %v", err)
+		}
+	}
+
+	if err = repo.UpdateSize(); err != nil {
+		log.Error("Failed to update size for repository: %v", err)
+	}
+
+	if opts.Mirror {
+		if err = models.InsertMirror(&models.Mirror{
+			RepoID:         repo.ID,
+			Interval:       setting.Mirror.DefaultInterval,
+			EnablePrune:    true,
+			NextUpdateUnix: timeutil.TimeStampNow().AddDuration(setting.Mirror.DefaultInterval),
+		}); err != nil {
+			return repo, fmt.Errorf("InsertOne: %v", err)
+		}
+
+		repo.IsMirror = true
+		err = models.UpdateRepository(repo, false)
+	} else {
+		repo, err = CleanUpMigrateInfo(repo)
+	}
+
+	return repo, err
+}
+
+// cleanUpMigrateGitConfig removes mirror info which prevents "push --all".
+// This also removes possible user credentials.
+func cleanUpMigrateGitConfig(configPath string) error {
+	cfg, err := ini.Load(configPath)
+	if err != nil {
+		return fmt.Errorf("open config file: %v", err)
+	}
+	cfg.DeleteSection("remote \"origin\"")
+	if err = cfg.SaveToIndent(configPath, "\t"); err != nil {
+		return fmt.Errorf("save config file: %v", err)
+	}
+	return nil
+}
+
+// CleanUpMigrateInfo finishes migrating repository and/or wiki with things that don't need to be done for mirrors.
+func CleanUpMigrateInfo(repo *models.Repository) (*models.Repository, error) {
+	repoPath := repo.RepoPath()
+	if err := models.CreateDelegateHooks(repoPath); err != nil {
+		return repo, fmt.Errorf("createDelegateHooks: %v", err)
+	}
+	if repo.HasWiki() {
+		if err := models.CreateDelegateHooks(repo.WikiPath()); err != nil {
+			return repo, fmt.Errorf("createDelegateHooks.(wiki): %v", err)
+		}
+	}
+
+	_, err := git.NewCommand("remote", "rm", "origin").RunInDir(repoPath)
+	if err != nil && !strings.HasPrefix(err.Error(), "exit status 128 - fatal: No such remote ") {
+		return repo, fmt.Errorf("CleanUpMigrateInfo: %v", err)
+	}
+
+	if repo.HasWiki() {
+		if err := cleanUpMigrateGitConfig(path.Join(repo.WikiPath(), "config")); err != nil {
+			return repo, fmt.Errorf("cleanUpMigrateGitConfig (wiki): %v", err)
+		}
+	}
+
+	return repo, models.UpdateRepository(repo, false)
+}
+
+// SyncReleasesWithTags synchronizes release table with repository tags
+func SyncReleasesWithTags(repo *models.Repository, gitRepo *git.Repository) error {
+	existingRelTags := make(map[string]struct{})
+	opts := models.FindReleasesOptions{IncludeDrafts: true, IncludeTags: true}
+	for page := 1; ; page++ {
+		rels, err := models.GetReleasesByRepoID(repo.ID, opts, page, 100)
+		if err != nil {
+			return fmt.Errorf("GetReleasesByRepoID: %v", err)
+		}
+		if len(rels) == 0 {
+			break
+		}
+		for _, rel := range rels {
+			if rel.IsDraft {
+				continue
+			}
+			commitID, err := gitRepo.GetTagCommitID(rel.TagName)
+			if err != nil && !git.IsErrNotExist(err) {
+				return fmt.Errorf("GetTagCommitID: %v", err)
+			}
+			if git.IsErrNotExist(err) || commitID != rel.Sha1 {
+				if err := models.PushUpdateDeleteTag(repo, rel.TagName); err != nil {
+					return fmt.Errorf("PushUpdateDeleteTag: %v", err)
+				}
+			} else {
+				existingRelTags[strings.ToLower(rel.TagName)] = struct{}{}
+			}
+		}
+	}
+	tags, err := gitRepo.GetTags()
+	if err != nil {
+		return fmt.Errorf("GetTags: %v", err)
+	}
+	for _, tagName := range tags {
+		if _, ok := existingRelTags[strings.ToLower(tagName)]; !ok {
+			if err := models.PushUpdateAddTag(repo, gitRepo, tagName); err != nil {
+				return fmt.Errorf("pushUpdateAddTag: %v", err)
+			}
+		}
+	}
+	return nil
+}

routers/repo/setting.go

@@ -20,6 +20,7 @@ import (
 	"code.gitea.io/gitea/modules/context"
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/validation"
@@ -338,7 +339,7 @@ func SettingsPost(ctx *context.Context, form auth.RepoSettingForm) {
 		}
 		repo.IsMirror = false
 
-		if _, err := models.CleanUpMigrateInfo(repo); err != nil {
+		if _, err := repository.CleanUpMigrateInfo(repo); err != nil {
 			ctx.ServerError("CleanUpMigrateInfo", err)
 			return
 		} else if err = models.DeleteMirrorByRepoID(ctx.Repo.Repository.ID); err != nil {

services/mirror/mirror.go

@@ -15,6 +15,7 @@ import (
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/notification"
+	"code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/sync"
 	"code.gitea.io/gitea/modules/timeutil"
@@ -207,7 +208,7 @@ func runSync(m *models.Mirror) ([]*mirrorSyncResult, bool) {
 		log.Error("OpenRepository: %v", err)
 		return nil, false
 	}
-	if err = models.SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
+	if err = repository.SyncReleasesWithTags(m.Repo, gitRepo); err != nil {
 		gitRepo.Close()
 		log.Error("Failed to synchronize tags to releases for repository: %v", err)
 	}

services/mirror/mirror_test.go

@@ -10,6 +10,7 @@ import (
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/repository"
 	"code.gitea.io/gitea/modules/structs"
 	release_service "code.gitea.io/gitea/services/release"
 
@@ -46,7 +47,7 @@ func TestRelease_MirrorDelete(t *testing.T) {
 	})
 	assert.NoError(t, err)
 
-	mirror, err := models.MigrateRepositoryGitData(user, user, mirrorRepo, opts)
+	mirror, err := repository.MigrateRepositoryGitData(user, user, mirrorRepo, opts)
 	assert.NoError(t, err)
 
 	gitRepo, err := git.OpenRepository(repoPath)