MarcoBuster/gitea
1
0
Fork 0
mirror of https://github.com/go-gitea/gitea synced 2024-10-19 20:10:10 +02:00
Code Issues Packages Projects Releases Wiki Activity

move to services

Browse Source
This commit is contained in:
yp05327 2024-03-05 08:19:27 +00:00
parent 2f1af68517
commit f87f8578cf
20 changed files with 332 additions and 329 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

210
modules/repository/license.go
View File

@ -6,144 +6,13 @@ package repository
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"regexp"
"strings"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/util"
licenseclassifier "github.com/google/licenseclassifier/v2"
)
var (
classifier *licenseclassifier.Classifier
licenseAliases map[string]string
// licenseUpdaterQueue represents a queue to handle update repo licenses
licenseUpdaterQueue *queue.WorkerPoolQueue[*LicenseUpdaterOptions]
)
func Init() error {
licenseUpdaterQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "repo_license_updater", repoLicenseUpdater)
if licenseUpdaterQueue == nil {
return fmt.Errorf("unable to create repo_license_updater queue")
}
go graceful.GetManager().RunWithCancel(licenseUpdaterQueue)
return nil
}
type LicenseUpdaterOptions struct {
RepoID int64
}
func repoLicenseUpdater(items ...*LicenseUpdaterOptions) []*LicenseUpdaterOptions {
ctx := graceful.GetManager().ShutdownContext()
for _, opts := range items {
repo, err := repo_model.GetRepositoryByID(ctx, opts.RepoID)
if err != nil {
log.Error("repoLicenseUpdater [%d] failed: GetRepositoryByID: %v", opts.RepoID, err)
continue
}
if repo.IsEmpty {
continue
}
gitRepo, err := gitrepo.OpenRepository(ctx, repo)
if err != nil {
log.Error("repoLicenseUpdater [%d] failed: OpenRepository: %v", opts.RepoID, err)
continue
}
defer gitRepo.Close()
commit, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
if err != nil {
log.Error("repoLicenseUpdater [%d] failed: GetBranchCommit: %v", opts.RepoID, err)
continue
}
if err = updateRepoLicenses(ctx, repo, commit); err != nil {
log.Error("repoLicenseUpdater [%d] failed: updateRepoLicenses: %v", opts.RepoID, err)
}
}
return nil
}
func AddRepoToLicenseUpdaterQueue(opts *LicenseUpdaterOptions) error {
if opts == nil {
return nil
}
return licenseUpdaterQueue.Push(opts)
}
func loadLicenseAliases() error {
if licenseAliases != nil {
return nil
}
data, err := options.AssetFS().ReadFile("", "license-aliases.json")
if err != nil {
return err
}
err = json.Unmarshal(data, &licenseAliases)
if err != nil {
return err
}
return nil
}
func ConvertLicenseName(name string) string {
if err := loadLicenseAliases(); err != nil {
return name
}
v, ok := licenseAliases[name]
if ok {
return v
}
return name
}
func initClassifier() error {
if classifier != nil {
return nil
}
// threshold should be 0.84~0.86 or the test will be failed
classifier = licenseclassifier.NewClassifier(.85)
licenseFiles, err := options.AssetFS().ListFiles("license", true)
if err != nil {
return err
}
existLicense := make(container.Set[string])
if len(licenseFiles) > 0 {
for _, licenseFile := range licenseFiles {
licenseName := ConvertLicenseName(licenseFile)
if existLicense.Contains(licenseName) {
continue
}
existLicense.Add(licenseName)
data, err := options.License(licenseFile)
if err != nil {
return err
}
classifier.AddContent("License", licenseFile, licenseName, data)
}
}
return nil
}
type LicenseValues struct {
Owner string
Email string
@ -242,82 +111,3 @@ func getLicensePlaceholder(name string) *licensePlaceholder {
}
return ret
}
// updateRepoLicenses will update repository licenses col if license file exists
func updateRepoLicenses(ctx context.Context, repo *repo_model.Repository, commit *git.Commit) error {
if commit == nil {
return nil
}
_, licenseFile, err := findLicenseFile(commit)
if err != nil {
return fmt.Errorf("findLicenseFile: %w", err)
}
licenses := make([]string, 0)
if licenseFile != nil {
r, err := licenseFile.Blob().DataAsync()
if err != nil {
return err
}
defer r.Close()
licenses, err = detectLicense(r)
if err != nil {
return fmt.Errorf("detectLicense: %w", err)
}
}
return repo_model.UpdateRepoLicenses(ctx, repo, commit.ID.String(), licenses)
}
// GetDetectedLicenseFileName returns license file name in the repository if it exists
func GetDetectedLicenseFileName(ctx context.Context, repo *repo_model.Repository, commit *git.Commit) (string, error) {
if commit == nil {
return "", nil
}
_, licenseFile, err := findLicenseFile(commit)
if err != nil {
return "", fmt.Errorf("findLicenseFile: %w", err)
}
if licenseFile != nil {
return licenseFile.Name(), nil
}
return "", nil
}
// findLicenseFile returns the entry of license file in the repository if it exists
func findLicenseFile(commit *git.Commit) (string, *git.TreeEntry, error) {
if commit == nil {
return "", nil, nil
}
entries, err := commit.ListEntries()
if err != nil {
return "", nil, fmt.Errorf("ListEntries: %w", err)
}
return FindFileInEntries(util.FileTypeLicense, entries, "", "", false)
}
// detectLicense returns the licenses detected by the given content buff
func detectLicense(r io.Reader) ([]string, error) {
if r == nil {
return nil, nil
}
if err := initClassifier(); err != nil {
return nil, err
}
matches, err := classifier.MatchFrom(r)
if err != nil {
return nil, err
}
if len(matches.Matches) > 0 {
results := make(container.Set[string], len(matches.Matches))
for _, r := range matches.Matches {
if r.MatchType == "License" && !results.Contains(r.Variant) {
results.Add(r.Variant)
}
}
return results.Values(), nil
}
return nil, nil
}

60
modules/repository/license_test.go
View File

@ -5,7 +5,6 @@ package repository
import (
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@ -180,62 +179,3 @@ Copyright (C) 2023 by Gitea teabot@gitea.io
})
}
}
func Test_detectLicense(t *testing.T) {
type DetectLicenseTest struct {
name string
arg string
want []string
}
tests := []DetectLicenseTest{
{
name: "empty",
arg: "",
want: nil,
},
{
name: "no detected license",
arg: "Copyright (c) 2023 Gitea",
want: nil,
},
}
LoadRepoConfig()
err := loadLicenseAliases()
assert.NoError(t, err)
for _, licenseName := range Licenses {
license, err := GetLicense(licenseName, &LicenseValues{
Owner: "Gitea",
Email: "teabot@gitea.io",
Repo: "gitea",
Year: "2024",
})
assert.NoError(t, err)
tests = append(tests, DetectLicenseTest{
name: fmt.Sprintf("single license test: %s", licenseName),
arg: string(license),
want: []string{ConvertLicenseName(licenseName)},
})
}
err = initClassifier()
assert.NoError(t, err)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
license, err := detectLicense(strings.NewReader(tt.arg))
assert.NoError(t, err)
assert.Equal(t, tt.want, license)
})
}
result, err := detectLicense(strings.NewReader(tests[2].arg + tests[3].arg + tests[4].arg))
assert.NoError(t, err)
t.Run("multiple licenses test", func(t *testing.T) {
assert.Equal(t, 3, len(result))
assert.Contains(t, result, tests[2].want[0])
assert.Contains(t, result, tests[3].want[0])
assert.Contains(t, result, tests[4].want[0])
})
}

2
routers/api/v1/repo/repo.go
View File

@ -749,7 +749,7 @@ func updateBasicProperties(ctx *context.APIContext, opts api.EditRepoOption) err
}
if updateRepoLicense {
if err := repo_module.AddRepoToLicenseUpdaterQueue(&repo_module.LicenseUpdaterOptions{
if err := repo_service.AddRepoToLicenseUpdaterQueue(&repo_service.LicenseUpdaterOptions{
RepoID: ctx.Repo.Repository.ID,
}); err != nil {
ctx.Error(http.StatusInternalServerError, "AddRepoToLicenseUpdaterQueue", err)

3
routers/init.go
View File

@ -17,7 +17,6 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/markup/external"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/ssh"
"code.gitea.io/gitea/modules/storage"
@ -167,8 +166,6 @@ func InitWebInstalled(ctx context.Context) {
actions_service.Init()
mustInit(repository.Init)
// Finally start up the cron
cron.NewContext(ctx)
}

4
routers/private/default_branch.go
View File

@ -10,8 +10,8 @@ import (
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/private"
repo_module "code.gitea.io/gitea/modules/repository"
gitea_context "code.gitea.io/gitea/services/context"
repo_service "code.gitea.io/gitea/services/repository"
)
// SetDefaultBranch updates the default branch
@ -37,7 +37,7 @@ func SetDefaultBranch(ctx *gitea_context.PrivateContext) {
return
}
if err := repo_module.AddRepoToLicenseUpdaterQueue(&repo_module.LicenseUpdaterOptions{
if err := repo_service.AddRepoToLicenseUpdaterQueue(&repo_service.LicenseUpdaterOptions{
RepoID: ctx.Repo.Repository.ID,
}); err != nil {
ctx.JSON(http.StatusInternalServerError, private.Response{

2
routers/private/hook_post_receive.go
View File

@ -195,7 +195,7 @@ func HookPostReceive(ctx *gitea_context.PrivateContext) {
branch := refFullName.BranchName()
if branch == baseRepo.DefaultBranch {
if err := repo_module.AddRepoToLicenseUpdaterQueue(&repo_module.LicenseUpdaterOptions{
if err := repo_service.AddRepoToLicenseUpdaterQueue(&repo_service.LicenseUpdaterOptions{
RepoID: repo.ID,
}); err != nil {
ctx.JSON(http.StatusInternalServerError, private.Response{Err: err.Error()})

2
routers/web/repo/branch.go
View File

@ -85,7 +85,7 @@ func Branches(ctx *context.Context) {
pager.SetDefaultParams(ctx)
ctx.Data["Page"] = pager
ctx.Data["DetectedLicenseFileName"], err = repo_module.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit)
ctx.Data["DetectedLicenseFileName"], err = repo_service.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit)
if err != nil {
ctx.ServerError("GetDetectedLicenseFileName", err)
return

8
routers/web/repo/commit.go
View File

@ -24,12 +24,12 @@ import (
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/markup"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/services/context"
"code.gitea.io/gitea/services/gitdiff"
git_service "code.gitea.io/gitea/services/repository"
repo_service "code.gitea.io/gitea/services/repository"
)
const (
@ -91,7 +91,7 @@ func Commits(ctx *context.Context) {
pager := context.NewPagination(int(commitsCount), pageSize, page, 5)
pager.SetDefaultParams(ctx)
ctx.Data["Page"] = pager
ctx.Data["DetectedLicenseFileName"], err = repo_module.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit)
ctx.Data["DetectedLicenseFileName"], err = repo_service.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit)
if err != nil {
ctx.ServerError("GetDetectedLicenseFileName", err)
return
@ -213,7 +213,7 @@ func SearchCommits(ctx *context.Context) {
ctx.Data["Username"] = ctx.Repo.Owner.Name
ctx.Data["Reponame"] = ctx.Repo.Repository.Name
ctx.Data["RefName"] = ctx.Repo.RefName
ctx.Data["DetectedLicenseFileName"], err = repo_module.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit)
ctx.Data["DetectedLicenseFileName"], err = repo_service.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit)
if err != nil {
ctx.ServerError("GetDetectedLicenseFileName", err)
return
@ -266,7 +266,7 @@ func FileHistory(ctx *context.Context) {
pager.SetDefaultParams(ctx)
ctx.Data["Page"] = pager
ctx.Data["DetectedLicenseFileName"], err = repo_module.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit)
ctx.Data["DetectedLicenseFileName"], err = repo_service.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit)
if err != nil {
ctx.ServerError("GetDetectedLicenseFileName", err)
return

5
routers/web/repo/view.go
View File

@ -49,6 +49,7 @@ import (
"code.gitea.io/gitea/routers/web/feed"
"code.gitea.io/gitea/services/context"
issue_service "code.gitea.io/gitea/services/issue"
repo_service "code.gitea.io/gitea/services/repository"
files_service "code.gitea.io/gitea/services/repository/files"
"github.com/nektos/act/pkg/model"
@ -77,7 +78,7 @@ func renderDirectory(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("repo.file.title", ctx.Repo.Repository.Name+"/"+path.Base(ctx.Repo.TreePath), ctx.Repo.RefName)
}
subfolder, readmeFile, err := repo_module.FindFileInEntries(util.FileTypeReadme, entries, ctx.Repo.TreePath, ctx.Locale.Language(), true)
subfolder, readmeFile, err := repo_service.FindFileInEntries(util.FileTypeReadme, entries, ctx.Repo.TreePath, ctx.Locale.Language(), true)
if err != nil {
ctx.ServerError("findFileInEntries", err)
return
@ -946,7 +947,7 @@ func renderHomeCode(ctx *context.Context) {
ctx.Data["TreeLink"] = treeLink
ctx.Data["TreeNames"] = treeNames
ctx.Data["BranchLink"] = branchLink
ctx.Data["DetectedLicenseFileName"], err = repo_module.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit)
ctx.Data["DetectedLicenseFileName"], err = repo_service.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit)
if err != nil {
ctx.ServerError("GetDetectedLicenseFileName", err)
return

6
services/migrations/gitea_uploader_test.go
View File

@ -24,7 +24,6 @@ import (
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/optional"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/test"
@ -231,16 +230,13 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) {
func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) {
unittest.PrepareTestEnv(t)
err := repository.Init()
assert.NoError(t, err)
//
// fromRepo master
//
fromRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1})
baseRef := "master"
assert.NoError(t, git.InitRepository(git.DefaultContext, fromRepo.RepoPath(), false, fromRepo.ObjectFormatName))
err = git.NewCommand(git.DefaultContext, "symbolic-ref").AddDynamicArguments("HEAD", git.BranchPrefix+baseRef).Run(&git.RunOpts{Dir: fromRepo.RepoPath()})
err := git.NewCommand(git.DefaultContext, "symbolic-ref").AddDynamicArguments("HEAD", git.BranchPrefix+baseRef).Run(&git.RunOpts{Dir: fromRepo.RepoPath()})
assert.NoError(t, err)
assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte(fmt.Sprintf("# Testing Repository\n\nOriginally created in: %s", fromRepo.RepoPath())), 0o644))
assert.NoError(t, git.AddChanges(fromRepo.RepoPath(), true))

3
services/mirror/mirror_pull.go
View File

@ -23,6 +23,7 @@ import (
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
notify_service "code.gitea.io/gitea/services/notify"
repo_service "code.gitea.io/gitea/services/repository"
)
// gitShortEmptySha Git short empty SHA
@ -550,7 +551,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool {
}
// Update License
if err = repo_module.AddRepoToLicenseUpdaterQueue(&repo_module.LicenseUpdaterOptions{
if err = repo_service.AddRepoToLicenseUpdaterQueue(&repo_service.LicenseUpdaterOptions{
RepoID: m.Repo.ID,
}); err != nil {
log.Error("SyncMirrors [repo: %-v]: unable to add repo to license updater queue: %v", m.Repo, err)

2
services/repository/branch.go
View File

@ -504,7 +504,7 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, gitR
}
if !repo.IsEmpty {
if err := repo_module.AddRepoToLicenseUpdaterQueue(&repo_module.LicenseUpdaterOptions{
if err := AddRepoToLicenseUpdaterQueue(&LicenseUpdaterOptions{
RepoID: repo.ID,
}); err != nil {
log.Error("AddRepoToLicenseUpdaterQueue: %v", err)

2
services/repository/create.go
View File

@ -310,7 +310,7 @@ func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opt
// update licenses
var licenses []string
if len(opts.License) > 0 {
licenses = append(licenses, repo_module.ConvertLicenseName(opts.License))
licenses = append(licenses, ConvertLicenseName(opts.License))
stdout, _, err := git.NewCommand(ctx, "rev-parse", "HEAD").
SetDescription(fmt.Sprintf("CreateRepository(git rev-parse HEAD): %s", repoPath)).

0
modules/repository/file.go → services/repository/file.go
View File

0
modules/repository/file_test.go → services/repository/file_test.go
View File

235
services/repository/license.go Normal file
View File

@ -0,0 +1,235 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package repository
import (
"context"
"encoding/json"
"fmt"
"io"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/container"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/gitrepo"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/queue"
"code.gitea.io/gitea/modules/util"
licenseclassifier "github.com/google/licenseclassifier/v2"
)
var (
classifier *licenseclassifier.Classifier
licenseAliases map[string]string
// licenseUpdaterQueue represents a queue to handle update repo licenses
licenseUpdaterQueue *queue.WorkerPoolQueue[*LicenseUpdaterOptions]
)
func AddRepoToLicenseUpdaterQueue(opts *LicenseUpdaterOptions) error {
if opts == nil {
return nil
}
return licenseUpdaterQueue.Push(opts)
}
func loadLicenseAliases() error {
if licenseAliases != nil {
return nil
}
data, err := options.AssetFS().ReadFile("", "license-aliases.json")
if err != nil {
return err
}
err = json.Unmarshal(data, &licenseAliases)
if err != nil {
return err
}
return nil
}
func ConvertLicenseName(name string) string {
if err := loadLicenseAliases(); err != nil {
return name
}
v, ok := licenseAliases[name]
if ok {
return v
}
return name
}
func initClassifier() error {
if classifier != nil {
return nil
}
// threshold should be 0.84~0.86 or the test will be failed
classifier = licenseclassifier.NewClassifier(.85)
licenseFiles, err := options.AssetFS().ListFiles("license", true)
if err != nil {
return err
}
existLicense := make(container.Set[string])
if len(licenseFiles) > 0 {
for _, licenseFile := range licenseFiles {
licenseName := ConvertLicenseName(licenseFile)
if existLicense.Contains(licenseName) {
continue
}
existLicense.Add(licenseName)
data, err := options.License(licenseFile)
if err != nil {
return err
}
classifier.AddContent("License", licenseFile, licenseName, data)
}
}
return nil
}
type LicenseUpdaterOptions struct {
RepoID int64
}
func repoLicenseUpdater(items ...*LicenseUpdaterOptions) []*LicenseUpdaterOptions {
ctx := graceful.GetManager().ShutdownContext()
for _, opts := range items {
repo, err := repo_model.GetRepositoryByID(ctx, opts.RepoID)
if err != nil {
log.Error("repoLicenseUpdater [%d] failed: GetRepositoryByID: %v", opts.RepoID, err)
continue
}
if repo.IsEmpty {
continue
}
gitRepo, err := gitrepo.OpenRepository(ctx, repo)
if err != nil {
log.Error("repoLicenseUpdater [%d] failed: OpenRepository: %v", opts.RepoID, err)
continue
}
defer gitRepo.Close()
commit, err := gitRepo.GetBranchCommit(repo.DefaultBranch)
if err != nil {
log.Error("repoLicenseUpdater [%d] failed: GetBranchCommit: %v", opts.RepoID, err)
continue
}
if err = UpdateRepoLicenses(ctx, repo, commit); err != nil {
log.Error("repoLicenseUpdater [%d] failed: updateRepoLicenses: %v", opts.RepoID, err)
}
}
return nil
}
func SyncRepoLicenses(ctx context.Context) error {
log.Trace("Doing: SyncRepoLicenses")
if err := db.Iterate(
ctx,
nil,
func(ctx context.Context, repo *repo_model.Repository) error {
select {
case <-ctx.Done():
return db.ErrCancelledf("before sync repo licenses for %s", repo.FullName())
default:
}
return AddRepoToLicenseUpdaterQueue(&LicenseUpdaterOptions{RepoID: repo.ID})
},
); err != nil {
log.Trace("Error: SyncRepoLicenses: %v", err)
return err
}
log.Trace("Finished: SyncReposLicenses")
return nil
}
// UpdateRepoLicenses will update repository licenses col if license file exists
func UpdateRepoLicenses(ctx context.Context, repo *repo_model.Repository, commit *git.Commit) error {
if commit == nil {
return nil
}
_, licenseFile, err := findLicenseFile(commit)
if err != nil {
return fmt.Errorf("findLicenseFile: %w", err)
}
licenses := make([]string, 0)
if licenseFile != nil {
r, err := licenseFile.Blob().DataAsync()
if err != nil {
return err
}
defer r.Close()
licenses, err = detectLicense(r)
if err != nil {
return fmt.Errorf("detectLicense: %w", err)
}
}
return repo_model.UpdateRepoLicenses(ctx, repo, commit.ID.String(), licenses)
}
// GetDetectedLicenseFileName returns license file name in the repository if it exists
func GetDetectedLicenseFileName(ctx context.Context, repo *repo_model.Repository, commit *git.Commit) (string, error) {
if commit == nil {
return "", nil
}
_, licenseFile, err := findLicenseFile(commit)
if err != nil {
return "", fmt.Errorf("findLicenseFile: %w", err)
}
if licenseFile != nil {
return licenseFile.Name(), nil
}
return "", nil
}
// findLicenseFile returns the entry of license file in the repository if it exists
func findLicenseFile(commit *git.Commit) (string, *git.TreeEntry, error) {
if commit == nil {
return "", nil, nil
}
entries, err := commit.ListEntries()
if err != nil {
return "", nil, fmt.Errorf("ListEntries: %w", err)
}
return FindFileInEntries(util.FileTypeLicense, entries, "", "", false)
}
// detectLicense returns the licenses detected by the given content buff
func detectLicense(r io.Reader) ([]string, error) {
if r == nil {
return nil, nil
}
if err := initClassifier(); err != nil {
return nil, err
}
matches, err := classifier.MatchFrom(r)
if err != nil {
return nil, err
}
if len(matches.Matches) > 0 {
results := make(container.Set[string], len(matches.Matches))
for _, r := range matches.Matches {
if r.MatchType == "License" && !results.Contains(r.Variant) {
results.Add(r.Variant)
}
}
return results.Values(), nil
}
return nil, nil
}

72
services/repository/license_test.go Normal file
View File

@ -0,0 +1,72 @@
// Copyright 2023 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package repository
import (
"fmt"
"strings"
"testing"
repo_module "code.gitea.io/gitea/modules/repository"
"github.com/stretchr/testify/assert"
)
func Test_detectLicense(t *testing.T) {
type DetectLicenseTest struct {
name string
arg string
want []string
}
tests := []DetectLicenseTest{
{
name: "empty",
arg: "",
want: nil,
},
{
name: "no detected license",
arg: "Copyright (c) 2023 Gitea",
want: nil,
},
}
repo_module.LoadRepoConfig()
err := loadLicenseAliases()
assert.NoError(t, err)
for _, licenseName := range repo_module.Licenses {
license, err := repo_module.GetLicense(licenseName, &repo_module.LicenseValues{
Owner: "Gitea",
Email: "teabot@gitea.io",
Repo: "gitea",
Year: "2024",
})
assert.NoError(t, err)
tests = append(tests, DetectLicenseTest{
name: fmt.Sprintf("single license test: %s", licenseName),
arg: string(license),
want: []string{ConvertLicenseName(licenseName)},
})
}
err = initClassifier()
assert.NoError(t, err)
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
license, err := detectLicense(strings.NewReader(tt.arg))
assert.NoError(t, err)
assert.Equal(t, tt.want, license)
})
}
result, err := detectLicense(strings.NewReader(tests[2].arg + tests[3].arg + tests[4].arg))
assert.NoError(t, err)
t.Run("multiple licenses test", func(t *testing.T) {
assert.Equal(t, 3, len(result))
assert.Contains(t, result, tests[2].want[0])
assert.Contains(t, result, tests[3].want[0])
assert.Contains(t, result, tests[4].want[0])
})
}

36
services/repository/licenses.go
View File

@ -1,36 +0,0 @@
// Copyright 2024 The Gitea Authors. All rights reserved.
// SPDX-License-Identifier: MIT
package repository
import (
"context"
"code.gitea.io/gitea/models/db"
repo_model "code.gitea.io/gitea/models/repo"
"code.gitea.io/gitea/modules/log"
repo_module "code.gitea.io/gitea/modules/repository"
)
func SyncRepoLicenses(ctx context.Context) error {
log.Trace("Doing: SyncRepoLicenses")
if err := db.Iterate(
ctx,
nil,
func(ctx context.Context, repo *repo_model.Repository) error {
select {
case <-ctx.Done():
return db.ErrCancelledf("before sync repo licenses for %s", repo.FullName())
default:
}
return repo_module.AddRepoToLicenseUpdaterQueue(&repo_module.LicenseUpdaterOptions{RepoID: repo.ID})
},
); err != nil {
log.Trace("Error: SyncRepoLicenses: %v", err)
return err
}
log.Trace("Finished: SyncReposLicenses")
return nil
}

2
services/repository/migrate.go
View File

@ -152,7 +152,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User,
}
// Update repo license
if err := repo_module.AddRepoToLicenseUpdaterQueue(&repo_module.LicenseUpdaterOptions{RepoID: repo.ID}); err != nil {
if err := AddRepoToLicenseUpdaterQueue(&LicenseUpdaterOptions{RepoID: repo.ID}); err != nil {
log.Error("Failed to add repo to license updater queue: %v", err)
}
}

7
services/repository/repository.go
View File

@ -18,6 +18,7 @@ import (
user_model "code.gitea.io/gitea/models/user"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/queue"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
@ -96,6 +97,12 @@ func PushCreateRepo(ctx context.Context, authUser, owner *user_model.User, repoN
// Init start repository service
func Init(ctx context.Context) error {
licenseUpdaterQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "repo_license_updater", repoLicenseUpdater)
if licenseUpdaterQueue == nil {
return fmt.Errorf("unable to create repo_license_updater queue")
}
go graceful.GetManager().RunWithCancel(licenseUpdaterQueue)
if err := repo_module.LoadRepoConfig(); err != nil {
return err
}