mirror of
https://github.com/go-gitea/gitea
synced 2024-09-10 08:27:04 +02:00
Add support for linguist-detectable
and linguist-documentation
(#29267)
Add support for `linguist-detectable` and `linguist-documentation` Add tests for the attributes https://github.com/github-linguist/linguist/blob/master/docs/overrides.md#detectable https://github.com/github-linguist/linguist/blob/master/docs/overrides.md#documentation
This commit is contained in:
parent
7d0903bf90
commit
2a278b996f
@ -11,6 +11,7 @@ import (
|
||||
"os"
|
||||
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/optional"
|
||||
)
|
||||
|
||||
// CheckAttributeOpts represents the possible options to CheckAttribute
|
||||
@ -291,7 +292,7 @@ func (repo *Repository) CheckAttributeReader(commitID string) (*CheckAttributeRe
|
||||
}
|
||||
|
||||
checker := &CheckAttributeReader{
|
||||
Attributes: []string{"linguist-vendored", "linguist-generated", "linguist-language", "gitlab-language"},
|
||||
Attributes: []string{"linguist-vendored", "linguist-generated", "linguist-language", "gitlab-language", "linguist-documentation", "linguist-detectable"},
|
||||
Repo: repo,
|
||||
IndexFile: indexFilename,
|
||||
WorkTree: worktree,
|
||||
@ -316,3 +317,23 @@ func (repo *Repository) CheckAttributeReader(commitID string) (*CheckAttributeRe
|
||||
|
||||
return checker, deferable
|
||||
}
|
||||
|
||||
// true if "set"/"true", false if "unset"/"false", none otherwise
|
||||
func attributeToBool(attr map[string]string, name string) optional.Option[bool] {
|
||||
if value, has := attr[name]; has && value != "unspecified" {
|
||||
switch value {
|
||||
case "set", "true":
|
||||
return optional.Some(true)
|
||||
case "unset", "false":
|
||||
return optional.Some(false)
|
||||
}
|
||||
}
|
||||
return optional.None[bool]()
|
||||
}
|
||||
|
||||
func attributeToString(attr map[string]string, name string) optional.Option[string] {
|
||||
if value, has := attr[name]; has && value != "unspecified" {
|
||||
return optional.Some(value)
|
||||
}
|
||||
return optional.None[string]()
|
||||
}
|
||||
|
@ -11,6 +11,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/modules/analyze"
|
||||
"code.gitea.io/gitea/modules/optional"
|
||||
|
||||
"github.com/go-enry/go-enry/v2"
|
||||
"github.com/go-git/go-git/v5"
|
||||
@ -57,25 +58,47 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
|
||||
return nil
|
||||
}
|
||||
|
||||
notVendored := false
|
||||
notGenerated := false
|
||||
isVendored := optional.None[bool]()
|
||||
isGenerated := optional.None[bool]()
|
||||
isDocumentation := optional.None[bool]()
|
||||
isDetectable := optional.None[bool]()
|
||||
|
||||
if checker != nil {
|
||||
attrs, err := checker.CheckPath(f.Name)
|
||||
if err == nil {
|
||||
if vendored, has := attrs["linguist-vendored"]; has {
|
||||
if vendored == "set" || vendored == "true" {
|
||||
return nil
|
||||
}
|
||||
notVendored = vendored == "false"
|
||||
isVendored = attributeToBool(attrs, "linguist-vendored")
|
||||
if isVendored.ValueOrDefault(false) {
|
||||
return nil
|
||||
}
|
||||
if generated, has := attrs["linguist-generated"]; has {
|
||||
if generated == "set" || generated == "true" {
|
||||
return nil
|
||||
}
|
||||
notGenerated = generated == "false"
|
||||
|
||||
isGenerated = attributeToBool(attrs, "linguist-generated")
|
||||
if isGenerated.ValueOrDefault(false) {
|
||||
return nil
|
||||
}
|
||||
if language, has := attrs["linguist-language"]; has && language != "unspecified" && language != "" {
|
||||
|
||||
isDocumentation = attributeToBool(attrs, "linguist-documentation")
|
||||
if isDocumentation.ValueOrDefault(false) {
|
||||
return nil
|
||||
}
|
||||
|
||||
isDetectable = attributeToBool(attrs, "linguist-detectable")
|
||||
if !isDetectable.ValueOrDefault(true) {
|
||||
return nil
|
||||
}
|
||||
|
||||
hasLanguage := attributeToString(attrs, "linguist-language")
|
||||
if hasLanguage.Value() == "" {
|
||||
hasLanguage = attributeToString(attrs, "gitlab-language")
|
||||
if hasLanguage.Has() {
|
||||
language := hasLanguage.Value()
|
||||
if idx := strings.IndexByte(language, '?'); idx >= 0 {
|
||||
hasLanguage = optional.Some(language[:idx])
|
||||
}
|
||||
}
|
||||
}
|
||||
if hasLanguage.Value() != "" {
|
||||
language := hasLanguage.Value()
|
||||
|
||||
// group languages, such as Pug -> HTML; SCSS -> CSS
|
||||
group := enry.GetLanguageGroup(language)
|
||||
if len(group) != 0 {
|
||||
@ -85,28 +108,14 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
|
||||
// this language will always be added to the size
|
||||
sizes[language] += f.Size
|
||||
return nil
|
||||
} else if language, has := attrs["gitlab-language"]; has && language != "unspecified" && language != "" {
|
||||
// strip off a ? if present
|
||||
if idx := strings.IndexByte(language, '?'); idx >= 0 {
|
||||
language = language[:idx]
|
||||
}
|
||||
if len(language) != 0 {
|
||||
// group languages, such as Pug -> HTML; SCSS -> CSS
|
||||
group := enry.GetLanguageGroup(language)
|
||||
if len(group) != 0 {
|
||||
language = group
|
||||
}
|
||||
|
||||
// this language will always be added to the size
|
||||
sizes[language] += f.Size
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!notVendored && analyze.IsVendor(f.Name)) || enry.IsDotFile(f.Name) ||
|
||||
enry.IsDocumentation(f.Name) || enry.IsConfiguration(f.Name) {
|
||||
if (!isVendored.Has() && analyze.IsVendor(f.Name)) ||
|
||||
enry.IsDotFile(f.Name) ||
|
||||
(!isDocumentation.Has() && enry.IsDocumentation(f.Name)) ||
|
||||
enry.IsConfiguration(f.Name) {
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -115,12 +124,10 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
|
||||
if f.Size <= bigFileSize {
|
||||
content, _ = readFile(f, fileSizeLimit)
|
||||
}
|
||||
if !notGenerated && enry.IsGenerated(f.Name, content) {
|
||||
if !isGenerated.Has() && enry.IsGenerated(f.Name, content) {
|
||||
return nil
|
||||
}
|
||||
|
||||
// TODO: Use .gitattributes file for linguist overrides
|
||||
|
||||
language := analyze.GetCodeLanguage(f.Name, content)
|
||||
if language == enry.OtherLanguage || language == "" {
|
||||
return nil
|
||||
@ -138,7 +145,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
|
||||
included = langtype == enry.Programming || langtype == enry.Markup
|
||||
includedLanguage[language] = included
|
||||
}
|
||||
if included {
|
||||
if included || isDetectable.ValueOrDefault(false) {
|
||||
sizes[language] += f.Size
|
||||
} else if len(sizes) == 0 && (firstExcludedLanguage == "" || firstExcludedLanguage == language) {
|
||||
firstExcludedLanguage = language
|
||||
|
@ -12,6 +12,7 @@ import (
|
||||
|
||||
"code.gitea.io/gitea/modules/analyze"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/optional"
|
||||
|
||||
"github.com/go-enry/go-enry/v2"
|
||||
)
|
||||
@ -88,25 +89,47 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
|
||||
continue
|
||||
}
|
||||
|
||||
notVendored := false
|
||||
notGenerated := false
|
||||
isVendored := optional.None[bool]()
|
||||
isGenerated := optional.None[bool]()
|
||||
isDocumentation := optional.None[bool]()
|
||||
isDetectable := optional.None[bool]()
|
||||
|
||||
if checker != nil {
|
||||
attrs, err := checker.CheckPath(f.Name())
|
||||
if err == nil {
|
||||
if vendored, has := attrs["linguist-vendored"]; has {
|
||||
if vendored == "set" || vendored == "true" {
|
||||
continue
|
||||
}
|
||||
notVendored = vendored == "false"
|
||||
isVendored = attributeToBool(attrs, "linguist-vendored")
|
||||
if isVendored.ValueOrDefault(false) {
|
||||
continue
|
||||
}
|
||||
if generated, has := attrs["linguist-generated"]; has {
|
||||
if generated == "set" || generated == "true" {
|
||||
continue
|
||||
}
|
||||
notGenerated = generated == "false"
|
||||
|
||||
isGenerated = attributeToBool(attrs, "linguist-generated")
|
||||
if isGenerated.ValueOrDefault(false) {
|
||||
continue
|
||||
}
|
||||
if language, has := attrs["linguist-language"]; has && language != "unspecified" && language != "" {
|
||||
|
||||
isDocumentation = attributeToBool(attrs, "linguist-documentation")
|
||||
if isDocumentation.ValueOrDefault(false) {
|
||||
continue
|
||||
}
|
||||
|
||||
isDetectable = attributeToBool(attrs, "linguist-detectable")
|
||||
if !isDetectable.ValueOrDefault(true) {
|
||||
continue
|
||||
}
|
||||
|
||||
hasLanguage := attributeToString(attrs, "linguist-language")
|
||||
if hasLanguage.Value() == "" {
|
||||
hasLanguage = attributeToString(attrs, "gitlab-language")
|
||||
if hasLanguage.Has() {
|
||||
language := hasLanguage.Value()
|
||||
if idx := strings.IndexByte(language, '?'); idx >= 0 {
|
||||
hasLanguage = optional.Some(language[:idx])
|
||||
}
|
||||
}
|
||||
}
|
||||
if hasLanguage.Value() != "" {
|
||||
language := hasLanguage.Value()
|
||||
|
||||
// group languages, such as Pug -> HTML; SCSS -> CSS
|
||||
group := enry.GetLanguageGroup(language)
|
||||
if len(group) != 0 {
|
||||
@ -116,29 +139,14 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
|
||||
// this language will always be added to the size
|
||||
sizes[language] += f.Size()
|
||||
continue
|
||||
} else if language, has := attrs["gitlab-language"]; has && language != "unspecified" && language != "" {
|
||||
// strip off a ? if present
|
||||
if idx := strings.IndexByte(language, '?'); idx >= 0 {
|
||||
language = language[:idx]
|
||||
}
|
||||
if len(language) != 0 {
|
||||
// group languages, such as Pug -> HTML; SCSS -> CSS
|
||||
group := enry.GetLanguageGroup(language)
|
||||
if len(group) != 0 {
|
||||
language = group
|
||||
}
|
||||
|
||||
// this language will always be added to the size
|
||||
sizes[language] += f.Size()
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if (!notVendored && analyze.IsVendor(f.Name())) || enry.IsDotFile(f.Name()) ||
|
||||
enry.IsDocumentation(f.Name()) || enry.IsConfiguration(f.Name()) {
|
||||
if (!isVendored.Has() && analyze.IsVendor(f.Name())) ||
|
||||
enry.IsDotFile(f.Name()) ||
|
||||
(!isDocumentation.Has() && enry.IsDocumentation(f.Name())) ||
|
||||
enry.IsConfiguration(f.Name()) {
|
||||
continue
|
||||
}
|
||||
|
||||
@ -170,7 +178,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if !notGenerated && enry.IsGenerated(f.Name(), content) {
|
||||
if !isGenerated.Has() && enry.IsGenerated(f.Name(), content) {
|
||||
continue
|
||||
}
|
||||
|
||||
@ -193,13 +201,12 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
|
||||
included = langType == enry.Programming || langType == enry.Markup
|
||||
includedLanguage[language] = included
|
||||
}
|
||||
if included {
|
||||
if included || isDetectable.ValueOrDefault(false) {
|
||||
sizes[language] += f.Size()
|
||||
} else if len(sizes) == 0 && (firstExcludedLanguage == "" || firstExcludedLanguage == language) {
|
||||
firstExcludedLanguage = language
|
||||
firstExcludedLanguageSize += f.Size()
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// If there are no included languages add the first excluded language
|
||||
|
259
tests/integration/linguist_test.go
Normal file
259
tests/integration/linguist_test.go
Normal file
@ -0,0 +1,259 @@
|
||||
// Copyright 2024 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package integration
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/models/db"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
"code.gitea.io/gitea/models/unittest"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/git"
|
||||
"code.gitea.io/gitea/modules/indexer/stats"
|
||||
"code.gitea.io/gitea/modules/queue"
|
||||
repo_service "code.gitea.io/gitea/services/repository"
|
||||
files_service "code.gitea.io/gitea/services/repository/files"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestLinguist(t *testing.T) {
|
||||
onGiteaRun(t, func(t *testing.T, _ *url.URL) {
|
||||
user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2})
|
||||
|
||||
cppContent := "#include <iostream>\nint main() {\nstd::cout << \"Hello Gitea!\";\nreturn 0;\n}"
|
||||
pyContent := "print(\"Hello Gitea!\")"
|
||||
phpContent := "<?php\necho 'Hallo Welt';\n?>"
|
||||
lockContent := "# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand."
|
||||
mdContent := "markdown"
|
||||
|
||||
cases := []struct {
|
||||
GitAttributesContent string
|
||||
FilesToAdd []*files_service.ChangeRepoFile
|
||||
ExpectedLanguageOrder []string
|
||||
}{
|
||||
// case 0
|
||||
{
|
||||
ExpectedLanguageOrder: []string{},
|
||||
},
|
||||
// case 1
|
||||
{
|
||||
FilesToAdd: []*files_service.ChangeRepoFile{
|
||||
{
|
||||
TreePath: "cplusplus.cpp",
|
||||
ContentReader: strings.NewReader(cppContent),
|
||||
},
|
||||
{
|
||||
TreePath: "python.py",
|
||||
ContentReader: strings.NewReader(pyContent),
|
||||
},
|
||||
{
|
||||
TreePath: "php.php",
|
||||
ContentReader: strings.NewReader(phpContent),
|
||||
},
|
||||
},
|
||||
ExpectedLanguageOrder: []string{"C++", "PHP", "Python"},
|
||||
},
|
||||
// case 2
|
||||
{
|
||||
FilesToAdd: []*files_service.ChangeRepoFile{
|
||||
{
|
||||
TreePath: ".cplusplus.cpp",
|
||||
ContentReader: strings.NewReader(cppContent),
|
||||
},
|
||||
{
|
||||
TreePath: "python.py",
|
||||
ContentReader: strings.NewReader(pyContent),
|
||||
},
|
||||
{
|
||||
TreePath: "vendor/php.php",
|
||||
ContentReader: strings.NewReader(phpContent),
|
||||
},
|
||||
},
|
||||
ExpectedLanguageOrder: []string{"Python"},
|
||||
},
|
||||
// case 3
|
||||
{
|
||||
GitAttributesContent: "*.cpp linguist-language=Go",
|
||||
FilesToAdd: []*files_service.ChangeRepoFile{
|
||||
{
|
||||
TreePath: "cplusplus.cpp",
|
||||
ContentReader: strings.NewReader(cppContent),
|
||||
},
|
||||
},
|
||||
ExpectedLanguageOrder: []string{"Go"},
|
||||
},
|
||||
// case 4
|
||||
{
|
||||
GitAttributesContent: "*.cpp gitlab-language=Go?parent=json",
|
||||
FilesToAdd: []*files_service.ChangeRepoFile{
|
||||
{
|
||||
TreePath: "cplusplus.cpp",
|
||||
ContentReader: strings.NewReader(cppContent),
|
||||
},
|
||||
},
|
||||
ExpectedLanguageOrder: []string{"Go"},
|
||||
},
|
||||
// case 5
|
||||
{
|
||||
GitAttributesContent: "*.cpp linguist-language=HTML gitlab-language=Go?parent=json",
|
||||
FilesToAdd: []*files_service.ChangeRepoFile{
|
||||
{
|
||||
TreePath: "cplusplus.cpp",
|
||||
ContentReader: strings.NewReader(cppContent),
|
||||
},
|
||||
},
|
||||
ExpectedLanguageOrder: []string{"HTML"},
|
||||
},
|
||||
// case 6
|
||||
{
|
||||
GitAttributesContent: "vendor/** linguist-vendored=false",
|
||||
FilesToAdd: []*files_service.ChangeRepoFile{
|
||||
{
|
||||
TreePath: "vendor/php.php",
|
||||
ContentReader: strings.NewReader(phpContent),
|
||||
},
|
||||
},
|
||||
ExpectedLanguageOrder: []string{"PHP"},
|
||||
},
|
||||
// case 7
|
||||
{
|
||||
GitAttributesContent: "*.cpp linguist-vendored=true\n*.py linguist-vendored\nvendor/** -linguist-vendored",
|
||||
FilesToAdd: []*files_service.ChangeRepoFile{
|
||||
{
|
||||
TreePath: "cplusplus.cpp",
|
||||
ContentReader: strings.NewReader(cppContent),
|
||||
},
|
||||
{
|
||||
TreePath: "python.py",
|
||||
ContentReader: strings.NewReader(pyContent),
|
||||
},
|
||||
{
|
||||
TreePath: "vendor/php.php",
|
||||
ContentReader: strings.NewReader(phpContent),
|
||||
},
|
||||
},
|
||||
ExpectedLanguageOrder: []string{"PHP"},
|
||||
},
|
||||
// case 8
|
||||
{
|
||||
GitAttributesContent: "poetry.lock linguist-language=Go",
|
||||
FilesToAdd: []*files_service.ChangeRepoFile{
|
||||
{
|
||||
TreePath: "poetry.lock",
|
||||
ContentReader: strings.NewReader(lockContent),
|
||||
},
|
||||
},
|
||||
ExpectedLanguageOrder: []string{"Go"},
|
||||
},
|
||||
// case 9
|
||||
{
|
||||
GitAttributesContent: "poetry.lock linguist-generated=false",
|
||||
FilesToAdd: []*files_service.ChangeRepoFile{
|
||||
{
|
||||
TreePath: "poetry.lock",
|
||||
ContentReader: strings.NewReader(lockContent),
|
||||
},
|
||||
},
|
||||
ExpectedLanguageOrder: []string{"TOML"},
|
||||
},
|
||||
// case 10
|
||||
{
|
||||
GitAttributesContent: "*.cpp -linguist-detectable",
|
||||
FilesToAdd: []*files_service.ChangeRepoFile{
|
||||
{
|
||||
TreePath: "cplusplus.cpp",
|
||||
ContentReader: strings.NewReader(cppContent),
|
||||
},
|
||||
},
|
||||
ExpectedLanguageOrder: []string{},
|
||||
},
|
||||
// case 11
|
||||
{
|
||||
GitAttributesContent: "*.md linguist-detectable",
|
||||
FilesToAdd: []*files_service.ChangeRepoFile{
|
||||
{
|
||||
TreePath: "test.md",
|
||||
ContentReader: strings.NewReader(mdContent),
|
||||
},
|
||||
},
|
||||
ExpectedLanguageOrder: []string{"Markdown"},
|
||||
},
|
||||
// case 12
|
||||
{
|
||||
GitAttributesContent: "test2.md linguist-detectable",
|
||||
FilesToAdd: []*files_service.ChangeRepoFile{
|
||||
{
|
||||
TreePath: "cplusplus.cpp",
|
||||
ContentReader: strings.NewReader(cppContent),
|
||||
},
|
||||
{
|
||||
TreePath: "test.md",
|
||||
ContentReader: strings.NewReader(mdContent),
|
||||
},
|
||||
{
|
||||
TreePath: "test2.md",
|
||||
ContentReader: strings.NewReader(mdContent),
|
||||
},
|
||||
},
|
||||
ExpectedLanguageOrder: []string{"C++", "Markdown"},
|
||||
},
|
||||
// case 13
|
||||
{
|
||||
GitAttributesContent: "README.md linguist-documentation=false",
|
||||
FilesToAdd: []*files_service.ChangeRepoFile{
|
||||
{
|
||||
TreePath: "README.md",
|
||||
ContentReader: strings.NewReader(mdContent),
|
||||
},
|
||||
},
|
||||
ExpectedLanguageOrder: []string{"Markdown"},
|
||||
},
|
||||
}
|
||||
|
||||
for i, c := range cases {
|
||||
repo, err := repo_service.CreateRepository(db.DefaultContext, user, user, repo_service.CreateRepoOptions{
|
||||
Name: "linguist-test",
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
files := []*files_service.ChangeRepoFile{
|
||||
{
|
||||
TreePath: ".gitattributes",
|
||||
ContentReader: strings.NewReader(c.GitAttributesContent),
|
||||
},
|
||||
}
|
||||
files = append(files, c.FilesToAdd...)
|
||||
for _, f := range files {
|
||||
f.Operation = "create"
|
||||
}
|
||||
|
||||
_, err = files_service.ChangeRepoFiles(git.DefaultContext, repo, user, &files_service.ChangeRepoFilesOptions{
|
||||
Files: files,
|
||||
OldBranch: repo.DefaultBranch,
|
||||
NewBranch: repo.DefaultBranch,
|
||||
})
|
||||
assert.NoError(t, err)
|
||||
|
||||
assert.NoError(t, stats.UpdateRepoIndexer(repo))
|
||||
assert.NoError(t, queue.GetManager().FlushAll(context.Background(), 10*time.Second))
|
||||
|
||||
stats, err := repo_model.GetTopLanguageStats(db.DefaultContext, repo, len(c.FilesToAdd))
|
||||
assert.NoError(t, err)
|
||||
|
||||
languages := make([]string, 0, len(stats))
|
||||
for _, s := range stats {
|
||||
languages = append(languages, s.Language)
|
||||
}
|
||||
assert.Equal(t, c.ExpectedLanguageOrder, languages, "case %d: unexpected language stats", i)
|
||||
|
||||
assert.NoError(t, repo_service.DeleteRepository(db.DefaultContext, user, repo, false))
|
||||
}
|
||||
})
|
||||
}
|
Loading…
Reference in New Issue
Block a user