From f87f8578cf7e2d6550b32ae628718f02bea5336f Mon Sep 17 00:00:00 2001 From: yp05327 <576951401@qq.com> Date: Tue, 5 Mar 2024 08:19:27 +0000 Subject: [PATCH] move to services --- modules/repository/license.go | 210 ---------------- modules/repository/license_test.go | 60 ----- routers/api/v1/repo/repo.go | 2 +- routers/init.go | 3 - routers/private/default_branch.go | 4 +- routers/private/hook_post_receive.go | 2 +- routers/web/repo/branch.go | 2 +- routers/web/repo/commit.go | 8 +- routers/web/repo/view.go | 5 +- services/migrations/gitea_uploader_test.go | 6 +- services/mirror/mirror_pull.go | 3 +- services/repository/branch.go | 2 +- services/repository/create.go | 2 +- {modules => services}/repository/file.go | 0 {modules => services}/repository/file_test.go | 0 services/repository/license.go | 235 ++++++++++++++++++ services/repository/license_test.go | 72 ++++++ services/repository/licenses.go | 36 --- services/repository/migrate.go | 2 +- services/repository/repository.go | 7 + 20 files changed, 332 insertions(+), 329 deletions(-) rename {modules => services}/repository/file.go (100%) rename {modules => services}/repository/file_test.go (100%) create mode 100644 services/repository/license.go create mode 100644 services/repository/license_test.go delete mode 100644 services/repository/licenses.go diff --git a/modules/repository/license.go b/modules/repository/license.go index 083566a938..9da3af84f8 100644 --- a/modules/repository/license.go +++ b/modules/repository/license.go @@ -6,144 +6,13 @@ package repository import ( "bufio" "bytes" - "context" "fmt" - "io" "regexp" "strings" - repo_model "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/modules/container" - "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" - "code.gitea.io/gitea/modules/graceful" - "code.gitea.io/gitea/modules/json" - "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/options" - "code.gitea.io/gitea/modules/queue" - "code.gitea.io/gitea/modules/util" - - licenseclassifier "github.com/google/licenseclassifier/v2" ) -var ( - classifier *licenseclassifier.Classifier - licenseAliases map[string]string - - // licenseUpdaterQueue represents a queue to handle update repo licenses - licenseUpdaterQueue *queue.WorkerPoolQueue[*LicenseUpdaterOptions] -) - -func Init() error { - licenseUpdaterQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "repo_license_updater", repoLicenseUpdater) - if licenseUpdaterQueue == nil { - return fmt.Errorf("unable to create repo_license_updater queue") - } - go graceful.GetManager().RunWithCancel(licenseUpdaterQueue) - return nil -} - -type LicenseUpdaterOptions struct { - RepoID int64 -} - -func repoLicenseUpdater(items ...*LicenseUpdaterOptions) []*LicenseUpdaterOptions { - ctx := graceful.GetManager().ShutdownContext() - - for _, opts := range items { - repo, err := repo_model.GetRepositoryByID(ctx, opts.RepoID) - if err != nil { - log.Error("repoLicenseUpdater [%d] failed: GetRepositoryByID: %v", opts.RepoID, err) - continue - } - if repo.IsEmpty { - continue - } - - gitRepo, err := gitrepo.OpenRepository(ctx, repo) - if err != nil { - log.Error("repoLicenseUpdater [%d] failed: OpenRepository: %v", opts.RepoID, err) - continue - } - defer gitRepo.Close() - - commit, err := gitRepo.GetBranchCommit(repo.DefaultBranch) - if err != nil { - log.Error("repoLicenseUpdater [%d] failed: GetBranchCommit: %v", opts.RepoID, err) - continue - } - if err = updateRepoLicenses(ctx, repo, commit); err != nil { - log.Error("repoLicenseUpdater [%d] failed: updateRepoLicenses: %v", opts.RepoID, err) - } - } - return nil -} - -func AddRepoToLicenseUpdaterQueue(opts *LicenseUpdaterOptions) error { - if opts == nil { - return nil - } - return licenseUpdaterQueue.Push(opts) -} - -func loadLicenseAliases() error { - if licenseAliases != nil { - return nil - } - - data, err := options.AssetFS().ReadFile("", "license-aliases.json") - if err != nil { - return err - } - err = json.Unmarshal(data, &licenseAliases) - if err != nil { - return err - } - return nil -} - -func ConvertLicenseName(name string) string { - if err := loadLicenseAliases(); err != nil { - return name - } - - v, ok := licenseAliases[name] - if ok { - return v - } - return name -} - -func initClassifier() error { - if classifier != nil { - return nil - } - - // threshold should be 0.84~0.86 or the test will be failed - classifier = licenseclassifier.NewClassifier(.85) - licenseFiles, err := options.AssetFS().ListFiles("license", true) - if err != nil { - return err - } - - existLicense := make(container.Set[string]) - if len(licenseFiles) > 0 { - for _, licenseFile := range licenseFiles { - licenseName := ConvertLicenseName(licenseFile) - if existLicense.Contains(licenseName) { - continue - } - existLicense.Add(licenseName) - data, err := options.License(licenseFile) - if err != nil { - return err - } - classifier.AddContent("License", licenseFile, licenseName, data) - } - } - return nil -} - type LicenseValues struct { Owner string Email string @@ -242,82 +111,3 @@ func getLicensePlaceholder(name string) *licensePlaceholder { } return ret } - -// updateRepoLicenses will update repository licenses col if license file exists -func updateRepoLicenses(ctx context.Context, repo *repo_model.Repository, commit *git.Commit) error { - if commit == nil { - return nil - } - - _, licenseFile, err := findLicenseFile(commit) - if err != nil { - return fmt.Errorf("findLicenseFile: %w", err) - } - - licenses := make([]string, 0) - if licenseFile != nil { - r, err := licenseFile.Blob().DataAsync() - if err != nil { - return err - } - defer r.Close() - - licenses, err = detectLicense(r) - if err != nil { - return fmt.Errorf("detectLicense: %w", err) - } - } - return repo_model.UpdateRepoLicenses(ctx, repo, commit.ID.String(), licenses) -} - -// GetDetectedLicenseFileName returns license file name in the repository if it exists -func GetDetectedLicenseFileName(ctx context.Context, repo *repo_model.Repository, commit *git.Commit) (string, error) { - if commit == nil { - return "", nil - } - _, licenseFile, err := findLicenseFile(commit) - if err != nil { - return "", fmt.Errorf("findLicenseFile: %w", err) - } - if licenseFile != nil { - return licenseFile.Name(), nil - } - return "", nil -} - -// findLicenseFile returns the entry of license file in the repository if it exists -func findLicenseFile(commit *git.Commit) (string, *git.TreeEntry, error) { - if commit == nil { - return "", nil, nil - } - entries, err := commit.ListEntries() - if err != nil { - return "", nil, fmt.Errorf("ListEntries: %w", err) - } - return FindFileInEntries(util.FileTypeLicense, entries, "", "", false) -} - -// detectLicense returns the licenses detected by the given content buff -func detectLicense(r io.Reader) ([]string, error) { - if r == nil { - return nil, nil - } - if err := initClassifier(); err != nil { - return nil, err - } - - matches, err := classifier.MatchFrom(r) - if err != nil { - return nil, err - } - if len(matches.Matches) > 0 { - results := make(container.Set[string], len(matches.Matches)) - for _, r := range matches.Matches { - if r.MatchType == "License" && !results.Contains(r.Variant) { - results.Add(r.Variant) - } - } - return results.Values(), nil - } - return nil, nil -} diff --git a/modules/repository/license_test.go b/modules/repository/license_test.go index 38cda74454..3b0cfa1eed 100644 --- a/modules/repository/license_test.go +++ b/modules/repository/license_test.go @@ -5,7 +5,6 @@ package repository import ( "fmt" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -180,62 +179,3 @@ Copyright (C) 2023 by Gitea teabot@gitea.io }) } } - -func Test_detectLicense(t *testing.T) { - type DetectLicenseTest struct { - name string - arg string - want []string - } - - tests := []DetectLicenseTest{ - { - name: "empty", - arg: "", - want: nil, - }, - { - name: "no detected license", - arg: "Copyright (c) 2023 Gitea", - want: nil, - }, - } - - LoadRepoConfig() - err := loadLicenseAliases() - assert.NoError(t, err) - for _, licenseName := range Licenses { - license, err := GetLicense(licenseName, &LicenseValues{ - Owner: "Gitea", - Email: "teabot@gitea.io", - Repo: "gitea", - Year: "2024", - }) - assert.NoError(t, err) - - tests = append(tests, DetectLicenseTest{ - name: fmt.Sprintf("single license test: %s", licenseName), - arg: string(license), - want: []string{ConvertLicenseName(licenseName)}, - }) - } - - err = initClassifier() - assert.NoError(t, err) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - license, err := detectLicense(strings.NewReader(tt.arg)) - assert.NoError(t, err) - assert.Equal(t, tt.want, license) - }) - } - - result, err := detectLicense(strings.NewReader(tests[2].arg + tests[3].arg + tests[4].arg)) - assert.NoError(t, err) - t.Run("multiple licenses test", func(t *testing.T) { - assert.Equal(t, 3, len(result)) - assert.Contains(t, result, tests[2].want[0]) - assert.Contains(t, result, tests[3].want[0]) - assert.Contains(t, result, tests[4].want[0]) - }) -} diff --git a/routers/api/v1/repo/repo.go b/routers/api/v1/repo/repo.go index 6e1c29b620..2b8264f1db 100644 --- a/routers/api/v1/repo/repo.go +++ b/routers/api/v1/repo/repo.go @@ -749,7 +749,7 @@ func updateBasicProperties(ctx *context.APIContext, opts api.EditRepoOption) err } if updateRepoLicense { - if err := repo_module.AddRepoToLicenseUpdaterQueue(&repo_module.LicenseUpdaterOptions{ + if err := repo_service.AddRepoToLicenseUpdaterQueue(&repo_service.LicenseUpdaterOptions{ RepoID: ctx.Repo.Repository.ID, }); err != nil { ctx.Error(http.StatusInternalServerError, "AddRepoToLicenseUpdaterQueue", err) diff --git a/routers/init.go b/routers/init.go index cfc77528e0..aaf95920c2 100644 --- a/routers/init.go +++ b/routers/init.go @@ -17,7 +17,6 @@ import ( "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/markup/external" - "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/ssh" "code.gitea.io/gitea/modules/storage" @@ -167,8 +166,6 @@ func InitWebInstalled(ctx context.Context) { actions_service.Init() - mustInit(repository.Init) - // Finally start up the cron cron.NewContext(ctx) } diff --git a/routers/private/default_branch.go b/routers/private/default_branch.go index 0ffa56801c..72e97db344 100644 --- a/routers/private/default_branch.go +++ b/routers/private/default_branch.go @@ -10,8 +10,8 @@ import ( repo_model "code.gitea.io/gitea/models/repo" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/private" - repo_module "code.gitea.io/gitea/modules/repository" gitea_context "code.gitea.io/gitea/services/context" + repo_service "code.gitea.io/gitea/services/repository" ) // SetDefaultBranch updates the default branch @@ -37,7 +37,7 @@ func SetDefaultBranch(ctx *gitea_context.PrivateContext) { return } - if err := repo_module.AddRepoToLicenseUpdaterQueue(&repo_module.LicenseUpdaterOptions{ + if err := repo_service.AddRepoToLicenseUpdaterQueue(&repo_service.LicenseUpdaterOptions{ RepoID: ctx.Repo.Repository.ID, }); err != nil { ctx.JSON(http.StatusInternalServerError, private.Response{ diff --git a/routers/private/hook_post_receive.go b/routers/private/hook_post_receive.go index 2e332fbff8..3eba36b2e9 100644 --- a/routers/private/hook_post_receive.go +++ b/routers/private/hook_post_receive.go @@ -195,7 +195,7 @@ func HookPostReceive(ctx *gitea_context.PrivateContext) { branch := refFullName.BranchName() if branch == baseRepo.DefaultBranch { - if err := repo_module.AddRepoToLicenseUpdaterQueue(&repo_module.LicenseUpdaterOptions{ + if err := repo_service.AddRepoToLicenseUpdaterQueue(&repo_service.LicenseUpdaterOptions{ RepoID: repo.ID, }); err != nil { ctx.JSON(http.StatusInternalServerError, private.Response{Err: err.Error()}) diff --git a/routers/web/repo/branch.go b/routers/web/repo/branch.go index 1995823867..a2f8830105 100644 --- a/routers/web/repo/branch.go +++ b/routers/web/repo/branch.go @@ -85,7 +85,7 @@ func Branches(ctx *context.Context) { pager.SetDefaultParams(ctx) ctx.Data["Page"] = pager - ctx.Data["DetectedLicenseFileName"], err = repo_module.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit) + ctx.Data["DetectedLicenseFileName"], err = repo_service.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit) if err != nil { ctx.ServerError("GetDetectedLicenseFileName", err) return diff --git a/routers/web/repo/commit.go b/routers/web/repo/commit.go index 020fa4aea2..f3c9df9d3c 100644 --- a/routers/web/repo/commit.go +++ b/routers/web/repo/commit.go @@ -24,12 +24,12 @@ import ( "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/markup" - repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "code.gitea.io/gitea/services/context" "code.gitea.io/gitea/services/gitdiff" git_service "code.gitea.io/gitea/services/repository" + repo_service "code.gitea.io/gitea/services/repository" ) const ( @@ -91,7 +91,7 @@ func Commits(ctx *context.Context) { pager := context.NewPagination(int(commitsCount), pageSize, page, 5) pager.SetDefaultParams(ctx) ctx.Data["Page"] = pager - ctx.Data["DetectedLicenseFileName"], err = repo_module.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit) + ctx.Data["DetectedLicenseFileName"], err = repo_service.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit) if err != nil { ctx.ServerError("GetDetectedLicenseFileName", err) return @@ -213,7 +213,7 @@ func SearchCommits(ctx *context.Context) { ctx.Data["Username"] = ctx.Repo.Owner.Name ctx.Data["Reponame"] = ctx.Repo.Repository.Name ctx.Data["RefName"] = ctx.Repo.RefName - ctx.Data["DetectedLicenseFileName"], err = repo_module.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit) + ctx.Data["DetectedLicenseFileName"], err = repo_service.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit) if err != nil { ctx.ServerError("GetDetectedLicenseFileName", err) return @@ -266,7 +266,7 @@ func FileHistory(ctx *context.Context) { pager.SetDefaultParams(ctx) ctx.Data["Page"] = pager - ctx.Data["DetectedLicenseFileName"], err = repo_module.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit) + ctx.Data["DetectedLicenseFileName"], err = repo_service.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit) if err != nil { ctx.ServerError("GetDetectedLicenseFileName", err) return diff --git a/routers/web/repo/view.go b/routers/web/repo/view.go index d498a5dd07..fdaf6dee92 100644 --- a/routers/web/repo/view.go +++ b/routers/web/repo/view.go @@ -49,6 +49,7 @@ import ( "code.gitea.io/gitea/routers/web/feed" "code.gitea.io/gitea/services/context" issue_service "code.gitea.io/gitea/services/issue" + repo_service "code.gitea.io/gitea/services/repository" files_service "code.gitea.io/gitea/services/repository/files" "github.com/nektos/act/pkg/model" @@ -77,7 +78,7 @@ func renderDirectory(ctx *context.Context) { ctx.Data["Title"] = ctx.Tr("repo.file.title", ctx.Repo.Repository.Name+"/"+path.Base(ctx.Repo.TreePath), ctx.Repo.RefName) } - subfolder, readmeFile, err := repo_module.FindFileInEntries(util.FileTypeReadme, entries, ctx.Repo.TreePath, ctx.Locale.Language(), true) + subfolder, readmeFile, err := repo_service.FindFileInEntries(util.FileTypeReadme, entries, ctx.Repo.TreePath, ctx.Locale.Language(), true) if err != nil { ctx.ServerError("findFileInEntries", err) return @@ -946,7 +947,7 @@ func renderHomeCode(ctx *context.Context) { ctx.Data["TreeLink"] = treeLink ctx.Data["TreeNames"] = treeNames ctx.Data["BranchLink"] = branchLink - ctx.Data["DetectedLicenseFileName"], err = repo_module.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit) + ctx.Data["DetectedLicenseFileName"], err = repo_service.GetDetectedLicenseFileName(ctx, ctx.Repo.Repository, ctx.Repo.Commit) if err != nil { ctx.ServerError("GetDetectedLicenseFileName", err) return diff --git a/services/migrations/gitea_uploader_test.go b/services/migrations/gitea_uploader_test.go index c5ba62c27e..c9b9248098 100644 --- a/services/migrations/gitea_uploader_test.go +++ b/services/migrations/gitea_uploader_test.go @@ -24,7 +24,6 @@ import ( "code.gitea.io/gitea/modules/log" base "code.gitea.io/gitea/modules/migration" "code.gitea.io/gitea/modules/optional" - "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/test" @@ -231,16 +230,13 @@ func TestGiteaUploadRemapExternalUser(t *testing.T) { func TestGiteaUploadUpdateGitForPullRequest(t *testing.T) { unittest.PrepareTestEnv(t) - err := repository.Init() - assert.NoError(t, err) - // // fromRepo master // fromRepo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) baseRef := "master" assert.NoError(t, git.InitRepository(git.DefaultContext, fromRepo.RepoPath(), false, fromRepo.ObjectFormatName)) - err = git.NewCommand(git.DefaultContext, "symbolic-ref").AddDynamicArguments("HEAD", git.BranchPrefix+baseRef).Run(&git.RunOpts{Dir: fromRepo.RepoPath()}) + err := git.NewCommand(git.DefaultContext, "symbolic-ref").AddDynamicArguments("HEAD", git.BranchPrefix+baseRef).Run(&git.RunOpts{Dir: fromRepo.RepoPath()}) assert.NoError(t, err) assert.NoError(t, os.WriteFile(filepath.Join(fromRepo.RepoPath(), "README.md"), []byte(fmt.Sprintf("# Testing Repository\n\nOriginally created in: %s", fromRepo.RepoPath())), 0o644)) assert.NoError(t, git.AddChanges(fromRepo.RepoPath(), true)) diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go index a26e571f74..84e7657980 100644 --- a/services/mirror/mirror_pull.go +++ b/services/mirror/mirror_pull.go @@ -23,6 +23,7 @@ import ( "code.gitea.io/gitea/modules/timeutil" "code.gitea.io/gitea/modules/util" notify_service "code.gitea.io/gitea/services/notify" + repo_service "code.gitea.io/gitea/services/repository" ) // gitShortEmptySha Git short empty SHA @@ -550,7 +551,7 @@ func SyncPullMirror(ctx context.Context, repoID int64) bool { } // Update License - if err = repo_module.AddRepoToLicenseUpdaterQueue(&repo_module.LicenseUpdaterOptions{ + if err = repo_service.AddRepoToLicenseUpdaterQueue(&repo_service.LicenseUpdaterOptions{ RepoID: m.Repo.ID, }); err != nil { log.Error("SyncMirrors [repo: %-v]: unable to add repo to license updater queue: %v", m.Repo, err) diff --git a/services/repository/branch.go b/services/repository/branch.go index cb2c212989..8292faa448 100644 --- a/services/repository/branch.go +++ b/services/repository/branch.go @@ -504,7 +504,7 @@ func SetRepoDefaultBranch(ctx context.Context, repo *repo_model.Repository, gitR } if !repo.IsEmpty { - if err := repo_module.AddRepoToLicenseUpdaterQueue(&repo_module.LicenseUpdaterOptions{ + if err := AddRepoToLicenseUpdaterQueue(&LicenseUpdaterOptions{ RepoID: repo.ID, }); err != nil { log.Error("AddRepoToLicenseUpdaterQueue: %v", err) diff --git a/services/repository/create.go b/services/repository/create.go index bdfaf09404..79e7d01605 100644 --- a/services/repository/create.go +++ b/services/repository/create.go @@ -310,7 +310,7 @@ func CreateRepositoryDirectly(ctx context.Context, doer, u *user_model.User, opt // update licenses var licenses []string if len(opts.License) > 0 { - licenses = append(licenses, repo_module.ConvertLicenseName(opts.License)) + licenses = append(licenses, ConvertLicenseName(opts.License)) stdout, _, err := git.NewCommand(ctx, "rev-parse", "HEAD"). SetDescription(fmt.Sprintf("CreateRepository(git rev-parse HEAD): %s", repoPath)). diff --git a/modules/repository/file.go b/services/repository/file.go similarity index 100% rename from modules/repository/file.go rename to services/repository/file.go diff --git a/modules/repository/file_test.go b/services/repository/file_test.go similarity index 100% rename from modules/repository/file_test.go rename to services/repository/file_test.go diff --git a/services/repository/license.go b/services/repository/license.go new file mode 100644 index 0000000000..b62ef0009b --- /dev/null +++ b/services/repository/license.go @@ -0,0 +1,235 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "context" + "encoding/json" + "fmt" + "io" + + "code.gitea.io/gitea/models/db" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/modules/container" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + "code.gitea.io/gitea/modules/graceful" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/options" + "code.gitea.io/gitea/modules/queue" + "code.gitea.io/gitea/modules/util" + + licenseclassifier "github.com/google/licenseclassifier/v2" +) + +var ( + classifier *licenseclassifier.Classifier + licenseAliases map[string]string + + // licenseUpdaterQueue represents a queue to handle update repo licenses + licenseUpdaterQueue *queue.WorkerPoolQueue[*LicenseUpdaterOptions] +) + +func AddRepoToLicenseUpdaterQueue(opts *LicenseUpdaterOptions) error { + if opts == nil { + return nil + } + return licenseUpdaterQueue.Push(opts) +} + +func loadLicenseAliases() error { + if licenseAliases != nil { + return nil + } + + data, err := options.AssetFS().ReadFile("", "license-aliases.json") + if err != nil { + return err + } + err = json.Unmarshal(data, &licenseAliases) + if err != nil { + return err + } + return nil +} + +func ConvertLicenseName(name string) string { + if err := loadLicenseAliases(); err != nil { + return name + } + + v, ok := licenseAliases[name] + if ok { + return v + } + return name +} + +func initClassifier() error { + if classifier != nil { + return nil + } + + // threshold should be 0.84~0.86 or the test will be failed + classifier = licenseclassifier.NewClassifier(.85) + licenseFiles, err := options.AssetFS().ListFiles("license", true) + if err != nil { + return err + } + + existLicense := make(container.Set[string]) + if len(licenseFiles) > 0 { + for _, licenseFile := range licenseFiles { + licenseName := ConvertLicenseName(licenseFile) + if existLicense.Contains(licenseName) { + continue + } + existLicense.Add(licenseName) + data, err := options.License(licenseFile) + if err != nil { + return err + } + classifier.AddContent("License", licenseFile, licenseName, data) + } + } + return nil +} + +type LicenseUpdaterOptions struct { + RepoID int64 +} + +func repoLicenseUpdater(items ...*LicenseUpdaterOptions) []*LicenseUpdaterOptions { + ctx := graceful.GetManager().ShutdownContext() + + for _, opts := range items { + repo, err := repo_model.GetRepositoryByID(ctx, opts.RepoID) + if err != nil { + log.Error("repoLicenseUpdater [%d] failed: GetRepositoryByID: %v", opts.RepoID, err) + continue + } + if repo.IsEmpty { + continue + } + + gitRepo, err := gitrepo.OpenRepository(ctx, repo) + if err != nil { + log.Error("repoLicenseUpdater [%d] failed: OpenRepository: %v", opts.RepoID, err) + continue + } + defer gitRepo.Close() + + commit, err := gitRepo.GetBranchCommit(repo.DefaultBranch) + if err != nil { + log.Error("repoLicenseUpdater [%d] failed: GetBranchCommit: %v", opts.RepoID, err) + continue + } + if err = UpdateRepoLicenses(ctx, repo, commit); err != nil { + log.Error("repoLicenseUpdater [%d] failed: updateRepoLicenses: %v", opts.RepoID, err) + } + } + return nil +} + +func SyncRepoLicenses(ctx context.Context) error { + log.Trace("Doing: SyncRepoLicenses") + + if err := db.Iterate( + ctx, + nil, + func(ctx context.Context, repo *repo_model.Repository) error { + select { + case <-ctx.Done(): + return db.ErrCancelledf("before sync repo licenses for %s", repo.FullName()) + default: + } + return AddRepoToLicenseUpdaterQueue(&LicenseUpdaterOptions{RepoID: repo.ID}) + }, + ); err != nil { + log.Trace("Error: SyncRepoLicenses: %v", err) + return err + } + + log.Trace("Finished: SyncReposLicenses") + return nil +} + +// UpdateRepoLicenses will update repository licenses col if license file exists +func UpdateRepoLicenses(ctx context.Context, repo *repo_model.Repository, commit *git.Commit) error { + if commit == nil { + return nil + } + + _, licenseFile, err := findLicenseFile(commit) + if err != nil { + return fmt.Errorf("findLicenseFile: %w", err) + } + + licenses := make([]string, 0) + if licenseFile != nil { + r, err := licenseFile.Blob().DataAsync() + if err != nil { + return err + } + defer r.Close() + + licenses, err = detectLicense(r) + if err != nil { + return fmt.Errorf("detectLicense: %w", err) + } + } + return repo_model.UpdateRepoLicenses(ctx, repo, commit.ID.String(), licenses) +} + +// GetDetectedLicenseFileName returns license file name in the repository if it exists +func GetDetectedLicenseFileName(ctx context.Context, repo *repo_model.Repository, commit *git.Commit) (string, error) { + if commit == nil { + return "", nil + } + _, licenseFile, err := findLicenseFile(commit) + if err != nil { + return "", fmt.Errorf("findLicenseFile: %w", err) + } + if licenseFile != nil { + return licenseFile.Name(), nil + } + return "", nil +} + +// findLicenseFile returns the entry of license file in the repository if it exists +func findLicenseFile(commit *git.Commit) (string, *git.TreeEntry, error) { + if commit == nil { + return "", nil, nil + } + entries, err := commit.ListEntries() + if err != nil { + return "", nil, fmt.Errorf("ListEntries: %w", err) + } + return FindFileInEntries(util.FileTypeLicense, entries, "", "", false) +} + +// detectLicense returns the licenses detected by the given content buff +func detectLicense(r io.Reader) ([]string, error) { + if r == nil { + return nil, nil + } + if err := initClassifier(); err != nil { + return nil, err + } + + matches, err := classifier.MatchFrom(r) + if err != nil { + return nil, err + } + if len(matches.Matches) > 0 { + results := make(container.Set[string], len(matches.Matches)) + for _, r := range matches.Matches { + if r.MatchType == "License" && !results.Contains(r.Variant) { + results.Add(r.Variant) + } + } + return results.Values(), nil + } + return nil, nil +} diff --git a/services/repository/license_test.go b/services/repository/license_test.go new file mode 100644 index 0000000000..297cea57ff --- /dev/null +++ b/services/repository/license_test.go @@ -0,0 +1,72 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package repository + +import ( + "fmt" + "strings" + "testing" + + repo_module "code.gitea.io/gitea/modules/repository" + "github.com/stretchr/testify/assert" +) + +func Test_detectLicense(t *testing.T) { + type DetectLicenseTest struct { + name string + arg string + want []string + } + + tests := []DetectLicenseTest{ + { + name: "empty", + arg: "", + want: nil, + }, + { + name: "no detected license", + arg: "Copyright (c) 2023 Gitea", + want: nil, + }, + } + + repo_module.LoadRepoConfig() + err := loadLicenseAliases() + assert.NoError(t, err) + for _, licenseName := range repo_module.Licenses { + license, err := repo_module.GetLicense(licenseName, &repo_module.LicenseValues{ + Owner: "Gitea", + Email: "teabot@gitea.io", + Repo: "gitea", + Year: "2024", + }) + assert.NoError(t, err) + + tests = append(tests, DetectLicenseTest{ + name: fmt.Sprintf("single license test: %s", licenseName), + arg: string(license), + want: []string{ConvertLicenseName(licenseName)}, + }) + } + + err = initClassifier() + assert.NoError(t, err) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + license, err := detectLicense(strings.NewReader(tt.arg)) + assert.NoError(t, err) + assert.Equal(t, tt.want, license) + }) + } + + result, err := detectLicense(strings.NewReader(tests[2].arg + tests[3].arg + tests[4].arg)) + assert.NoError(t, err) + t.Run("multiple licenses test", func(t *testing.T) { + assert.Equal(t, 3, len(result)) + assert.Contains(t, result, tests[2].want[0]) + assert.Contains(t, result, tests[3].want[0]) + assert.Contains(t, result, tests[4].want[0]) + }) +} diff --git a/services/repository/licenses.go b/services/repository/licenses.go deleted file mode 100644 index 8b751c2807..0000000000 --- a/services/repository/licenses.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2024 The Gitea Authors. All rights reserved. -// SPDX-License-Identifier: MIT - -package repository - -import ( - "context" - - "code.gitea.io/gitea/models/db" - repo_model "code.gitea.io/gitea/models/repo" - "code.gitea.io/gitea/modules/log" - repo_module "code.gitea.io/gitea/modules/repository" -) - -func SyncRepoLicenses(ctx context.Context) error { - log.Trace("Doing: SyncRepoLicenses") - - if err := db.Iterate( - ctx, - nil, - func(ctx context.Context, repo *repo_model.Repository) error { - select { - case <-ctx.Done(): - return db.ErrCancelledf("before sync repo licenses for %s", repo.FullName()) - default: - } - return repo_module.AddRepoToLicenseUpdaterQueue(&repo_module.LicenseUpdaterOptions{RepoID: repo.ID}) - }, - ); err != nil { - log.Trace("Error: SyncRepoLicenses: %v", err) - return err - } - - log.Trace("Finished: SyncReposLicenses") - return nil -} diff --git a/services/repository/migrate.go b/services/repository/migrate.go index e934abbe24..38737c93da 100644 --- a/services/repository/migrate.go +++ b/services/repository/migrate.go @@ -152,7 +152,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *user_model.User, } // Update repo license - if err := repo_module.AddRepoToLicenseUpdaterQueue(&repo_module.LicenseUpdaterOptions{RepoID: repo.ID}); err != nil { + if err := AddRepoToLicenseUpdaterQueue(&LicenseUpdaterOptions{RepoID: repo.ID}); err != nil { log.Error("Failed to add repo to license updater queue: %v", err) } } diff --git a/services/repository/repository.go b/services/repository/repository.go index d28200c0ad..ed8b8abdc2 100644 --- a/services/repository/repository.go +++ b/services/repository/repository.go @@ -18,6 +18,7 @@ import ( user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/graceful" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/queue" repo_module "code.gitea.io/gitea/modules/repository" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" @@ -96,6 +97,12 @@ func PushCreateRepo(ctx context.Context, authUser, owner *user_model.User, repoN // Init start repository service func Init(ctx context.Context) error { + licenseUpdaterQueue = queue.CreateUniqueQueue(graceful.GetManager().ShutdownContext(), "repo_license_updater", repoLicenseUpdater) + if licenseUpdaterQueue == nil { + return fmt.Errorf("unable to create repo_license_updater queue") + } + go graceful.GetManager().RunWithCancel(licenseUpdaterQueue) + if err := repo_module.LoadRepoConfig(); err != nil { return err }