From 580e21dd2e9dfb3a3f86f51c4eb188c1bbfa8b11 Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 12 Nov 2024 10:38:22 +0800 Subject: [PATCH 1/4] Refactor LFS SSH and internal routers (#32473) Gitea instance keeps reporting a lot of errors like "LFS SSH transfer connection denied, pure SSH protocol is disabled". When starting debugging the problem, there are more problems found. Try to address most of them: * avoid unnecessary server side error logs (change `fail()` to not log them) * figure out the broken tests/user2/lfs.git (added comments) * avoid `migratePushMirrors` failure when a repository doesn't exist (ignore them) * avoid "Authorization" (internal&lfs) header conflicts, remove the tricky "swapAuth" and use "X-Gitea-Internal-Auth" * make internal token comparing constant time (it wasn't a serous problem because in a real world it's nearly impossible to timing-attack the token, but good to fix and backport) * avoid duplicate routers (introduce AddOwnerRepoGitLFSRoutes) * avoid "internal (private)" routes using session/web context (they should use private context) * fix incorrect "path" usages (use "filepath") * fix incorrect mocked route point handling (need to check func nil correctly) * split some tests from "git general tests" to "git misc tests" (to keep "git_general_test.go" simple) Still no correct result for Git LFS SSH tests. So the code is kept there (`tests/integration/git_lfs_ssh_test.go`) and a FIXME explains the details. --- cmd/serv.go | 14 +- models/fixtures/lfs_meta_object.yml | 18 +- models/migrations/v1_21/v276.go | 5 +- modules/git/batch_reader.go | 5 +- modules/lfstransfer/backend/backend.go | 48 ++--- modules/lfstransfer/backend/lock.go | 38 ++-- modules/lfstransfer/backend/util.go | 10 +- modules/private/internal.go | 2 +- modules/web/route.go | 13 +- routers/common/lfs.go | 29 +++ routers/private/internal.go | 55 ++---- routers/web/web.go | 18 +- tests/integration/api_repo_file_get_test.go | 2 +- .../{git_test.go => git_general_test.go} | 173 ++++-------------- tests/integration/git_lfs_ssh_test.go | 61 ++++++ tests/integration/git_misc_test.go | 138 ++++++++++++++ tests/test_utils.go | 11 +- 17 files changed, 376 insertions(+), 264 deletions(-) create mode 100644 routers/common/lfs.go rename tests/integration/{git_test.go => git_general_test.go} (85%) create mode 100644 tests/integration/git_lfs_ssh_test.go create mode 100644 tests/integration/git_misc_test.go diff --git a/cmd/serv.go b/cmd/serv.go index 2d2df8aa23b..d2271b68d29 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -111,12 +111,10 @@ func fail(ctx context.Context, userMessage, logMsgFmt string, args ...any) error if !setting.IsProd { _, _ = fmt.Fprintln(os.Stderr, "Gitea:", logMsg) } - if userMessage != "" { - if unicode.IsPunct(rune(userMessage[len(userMessage)-1])) { - logMsg = userMessage + " " + logMsg - } else { - logMsg = userMessage + ". " + logMsg - } + if unicode.IsPunct(rune(userMessage[len(userMessage)-1])) { + logMsg = userMessage + " " + logMsg + } else { + logMsg = userMessage + ". " + logMsg } _ = private.SSHLog(ctx, true, logMsg) } @@ -288,10 +286,10 @@ func runServ(c *cli.Context) error { if allowedCommands.Contains(verb) { if allowedCommandsLfs.Contains(verb) { if !setting.LFS.StartServer { - return fail(ctx, "Unknown git command", "LFS authentication request over SSH denied, LFS support is disabled") + return fail(ctx, "LFS Server is not enabled", "") } if verb == verbLfsTransfer && !setting.LFS.AllowPureSSH { - return fail(ctx, "Unknown git command", "LFS SSH transfer connection denied, pure SSH protocol is disabled") + return fail(ctx, "LFS SSH transfer is not enabled", "") } if len(words) > 2 { lfsVerb = words[2] diff --git a/models/fixtures/lfs_meta_object.yml b/models/fixtures/lfs_meta_object.yml index 1c29e02d44d..5430506d70a 100644 --- a/models/fixtures/lfs_meta_object.yml +++ b/models/fixtures/lfs_meta_object.yml @@ -1,4 +1,11 @@ # These are the LFS objects in user2/lfs.git +# user2/lfs is an INVALID repository +# +# commit e9c32647bab825977942598c0efa415de300304b (HEAD -> master) +# Author: Rowan Bohde +# Date: Thu Aug 1 14:38:23 2024 -0500 +# +# add invalid lfs file - id: 1 @@ -11,7 +18,7 @@ id: 2 oid: 2eccdb43825d2a49d99d542daa20075cff1d97d9d2349a8977efe9c03661737c - size: 107 + size: 107 # real size is 2048 repository_id: 54 created_unix: 1671607299 @@ -30,3 +37,12 @@ size: 25 repository_id: 54 created_unix: 1671607299 + +# this file is missing +# - +# +# id: 5 +# oid: 9d178b5f15046343fd32f451df93acc2bdd9e6373be478b968e4cad6b6647351 +# size: 25 +# repository_id: 54 +# created_unix: 1671607299 diff --git a/models/migrations/v1_21/v276.go b/models/migrations/v1_21/v276.go index ed1bc3bda52..15177bf0406 100644 --- a/models/migrations/v1_21/v276.go +++ b/models/migrations/v1_21/v276.go @@ -12,6 +12,7 @@ import ( "code.gitea.io/gitea/modules/git" giturl "code.gitea.io/gitea/modules/git/url" "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/util" "xorm.io/xorm" ) @@ -163,7 +164,9 @@ func migratePushMirrors(x *xorm.Engine) error { func getRemoteAddress(ownerName, repoName, remoteName string) (string, error) { repoPath := filepath.Join(setting.RepoRootPath, strings.ToLower(ownerName), strings.ToLower(repoName)+".git") - + if exist, _ := util.IsExist(repoPath); !exist { + return "", nil + } remoteURL, err := git.GetRemoteAddress(context.Background(), repoPath, remoteName) if err != nil { return "", fmt.Errorf("get remote %s's address of %s/%s failed: %v", remoteName, ownerName, repoName, err) diff --git a/modules/git/batch_reader.go b/modules/git/batch_reader.go index 3b1a466b2ea..7dfda721554 100644 --- a/modules/git/batch_reader.go +++ b/modules/git/batch_reader.go @@ -146,9 +146,8 @@ func catFileBatch(ctx context.Context, repoPath string) (WriteCloserError, *bufi } // ReadBatchLine reads the header line from cat-file --batch -// We expect: -// SP SP LF -// sha is a hex encoded here +// We expect: SP SP LF +// then leaving the rest of the stream " LF" to be read func ReadBatchLine(rd *bufio.Reader) (sha []byte, typ string, size int64, err error) { typ, err = rd.ReadString('\n') if err != nil { diff --git a/modules/lfstransfer/backend/backend.go b/modules/lfstransfer/backend/backend.go index d4523e1abfa..2b1fe49fdab 100644 --- a/modules/lfstransfer/backend/backend.go +++ b/modules/lfstransfer/backend/backend.go @@ -33,12 +33,12 @@ var _ transfer.Backend = &GiteaBackend{} // GiteaBackend is an adapter between git-lfs-transfer library and Gitea's internal LFS API type GiteaBackend struct { - ctx context.Context - server *url.URL - op string - token string - itoken string - logger transfer.Logger + ctx context.Context + server *url.URL + op string + authToken string + internalAuth string + logger transfer.Logger } func New(ctx context.Context, repo, op, token string, logger transfer.Logger) (transfer.Backend, error) { @@ -48,7 +48,7 @@ func New(ctx context.Context, repo, op, token string, logger transfer.Logger) (t return nil, err } server = server.JoinPath("api/internal/repo", repo, "info/lfs") - return &GiteaBackend{ctx: ctx, server: server, op: op, token: token, itoken: fmt.Sprintf("Bearer %s", setting.InternalToken), logger: logger}, nil + return &GiteaBackend{ctx: ctx, server: server, op: op, authToken: token, internalAuth: fmt.Sprintf("Bearer %s", setting.InternalToken), logger: logger}, nil } // Batch implements transfer.Backend @@ -73,10 +73,10 @@ func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args trans } url := g.server.JoinPath("objects/batch").String() headers := map[string]string{ - headerAuthorisation: g.itoken, - headerAuthX: g.token, - headerAccept: mimeGitLFS, - headerContentType: mimeGitLFS, + headerAuthorization: g.authToken, + headerGiteaInternalAuth: g.internalAuth, + headerAccept: mimeGitLFS, + headerContentType: mimeGitLFS, } req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) resp, err := req.Response() @@ -119,7 +119,7 @@ func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args trans } idMapStr := base64.StdEncoding.EncodeToString(idMapBytes) item.Args[argID] = idMapStr - if authHeader, ok := action.Header[headerAuthorisation]; ok { + if authHeader, ok := action.Header[headerAuthorization]; ok { authHeaderB64 := base64.StdEncoding.EncodeToString([]byte(authHeader)) item.Args[argToken] = authHeaderB64 } @@ -142,7 +142,7 @@ func (g *GiteaBackend) Batch(_ string, pointers []transfer.BatchItem, args trans } idMapStr := base64.StdEncoding.EncodeToString(idMapBytes) item.Args[argID] = idMapStr - if authHeader, ok := action.Header[headerAuthorisation]; ok { + if authHeader, ok := action.Header[headerAuthorization]; ok { authHeaderB64 := base64.StdEncoding.EncodeToString([]byte(authHeader)) item.Args[argToken] = authHeaderB64 } @@ -183,9 +183,9 @@ func (g *GiteaBackend) Download(oid string, args transfer.Args) (io.ReadCloser, } url := action.Href headers := map[string]string{ - headerAuthorisation: g.itoken, - headerAuthX: g.token, - headerAccept: mimeOctetStream, + headerAuthorization: g.authToken, + headerGiteaInternalAuth: g.internalAuth, + headerAccept: mimeOctetStream, } req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil) resp, err := req.Response() @@ -229,10 +229,10 @@ func (g *GiteaBackend) Upload(oid string, size int64, r io.Reader, args transfer } url := action.Href headers := map[string]string{ - headerAuthorisation: g.itoken, - headerAuthX: g.token, - headerContentType: mimeOctetStream, - headerContentLength: strconv.FormatInt(size, 10), + headerAuthorization: g.authToken, + headerGiteaInternalAuth: g.internalAuth, + headerContentType: mimeOctetStream, + headerContentLength: strconv.FormatInt(size, 10), } reqBytes, err := io.ReadAll(r) if err != nil { @@ -279,10 +279,10 @@ func (g *GiteaBackend) Verify(oid string, size int64, args transfer.Args) (trans } url := action.Href headers := map[string]string{ - headerAuthorisation: g.itoken, - headerAuthX: g.token, - headerAccept: mimeGitLFS, - headerContentType: mimeGitLFS, + headerAuthorization: g.authToken, + headerGiteaInternalAuth: g.internalAuth, + headerAccept: mimeGitLFS, + headerContentType: mimeGitLFS, } req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) resp, err := req.Response() diff --git a/modules/lfstransfer/backend/lock.go b/modules/lfstransfer/backend/lock.go index f72ffd5b6f9..f094cce1db6 100644 --- a/modules/lfstransfer/backend/lock.go +++ b/modules/lfstransfer/backend/lock.go @@ -21,17 +21,17 @@ import ( var _ transfer.LockBackend = &giteaLockBackend{} type giteaLockBackend struct { - ctx context.Context - g *GiteaBackend - server *url.URL - token string - itoken string - logger transfer.Logger + ctx context.Context + g *GiteaBackend + server *url.URL + authToken string + internalAuth string + logger transfer.Logger } func newGiteaLockBackend(g *GiteaBackend) transfer.LockBackend { server := g.server.JoinPath("locks") - return &giteaLockBackend{ctx: g.ctx, g: g, server: server, token: g.token, itoken: g.itoken, logger: g.logger} + return &giteaLockBackend{ctx: g.ctx, g: g, server: server, authToken: g.authToken, internalAuth: g.internalAuth, logger: g.logger} } // Create implements transfer.LockBackend @@ -45,10 +45,10 @@ func (g *giteaLockBackend) Create(path, refname string) (transfer.Lock, error) { } url := g.server.String() headers := map[string]string{ - headerAuthorisation: g.itoken, - headerAuthX: g.token, - headerAccept: mimeGitLFS, - headerContentType: mimeGitLFS, + headerAuthorization: g.authToken, + headerGiteaInternalAuth: g.internalAuth, + headerAccept: mimeGitLFS, + headerContentType: mimeGitLFS, } req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) resp, err := req.Response() @@ -97,10 +97,10 @@ func (g *giteaLockBackend) Unlock(lock transfer.Lock) error { } url := g.server.JoinPath(lock.ID(), "unlock").String() headers := map[string]string{ - headerAuthorisation: g.itoken, - headerAuthX: g.token, - headerAccept: mimeGitLFS, - headerContentType: mimeGitLFS, + headerAuthorization: g.authToken, + headerGiteaInternalAuth: g.internalAuth, + headerAccept: mimeGitLFS, + headerContentType: mimeGitLFS, } req := newInternalRequest(g.ctx, url, http.MethodPost, headers, bodyBytes) resp, err := req.Response() @@ -180,10 +180,10 @@ func (g *giteaLockBackend) queryLocks(v url.Values) ([]transfer.Lock, string, er urlq.RawQuery = v.Encode() url := urlq.String() headers := map[string]string{ - headerAuthorisation: g.itoken, - headerAuthX: g.token, - headerAccept: mimeGitLFS, - headerContentType: mimeGitLFS, + headerAuthorization: g.authToken, + headerGiteaInternalAuth: g.internalAuth, + headerAccept: mimeGitLFS, + headerContentType: mimeGitLFS, } req := newInternalRequest(g.ctx, url, http.MethodGet, headers, nil) resp, err := req.Response() diff --git a/modules/lfstransfer/backend/util.go b/modules/lfstransfer/backend/util.go index 126ac001753..cffefef375c 100644 --- a/modules/lfstransfer/backend/util.go +++ b/modules/lfstransfer/backend/util.go @@ -20,11 +20,11 @@ import ( // HTTP headers const ( - headerAccept = "Accept" - headerAuthorisation = "Authorization" - headerAuthX = "X-Auth" - headerContentType = "Content-Type" - headerContentLength = "Content-Length" + headerAccept = "Accept" + headerAuthorization = "Authorization" + headerGiteaInternalAuth = "X-Gitea-Internal-Auth" + headerContentType = "Content-Type" + headerContentLength = "Content-Length" ) // MIME types diff --git a/modules/private/internal.go b/modules/private/internal.go index 9c330a24a86..c7e7773524d 100644 --- a/modules/private/internal.go +++ b/modules/private/internal.go @@ -43,7 +43,7 @@ Ensure you are running in the correct environment or set the correct configurati req := httplib.NewRequest(url, method). SetContext(ctx). Header("X-Real-IP", getClientIP()). - Header("Authorization", fmt.Sprintf("Bearer %s", setting.InternalToken)). + Header("X-Gitea-Internal-Auth", fmt.Sprintf("Bearer %s", setting.InternalToken)). SetTLSClientConfig(&tls.Config{ InsecureSkipVerify: true, ServerName: setting.Domain, diff --git a/modules/web/route.go b/modules/web/route.go index b02f66802ee..77c411a97b9 100644 --- a/modules/web/route.go +++ b/modules/web/route.go @@ -6,6 +6,7 @@ package web import ( "net/http" "net/url" + "reflect" "strings" "code.gitea.io/gitea/modules/setting" @@ -82,15 +83,23 @@ func (r *Router) getPattern(pattern string) string { return strings.TrimSuffix(newPattern, "/") } +func isNilOrFuncNil(v any) bool { + if v == nil { + return true + } + r := reflect.ValueOf(v) + return r.Kind() == reflect.Func && r.IsNil() +} + func (r *Router) wrapMiddlewareAndHandler(h []any) ([]func(http.Handler) http.Handler, http.HandlerFunc) { handlerProviders := make([]func(http.Handler) http.Handler, 0, len(r.curMiddlewares)+len(h)+1) for _, m := range r.curMiddlewares { - if m != nil { + if !isNilOrFuncNil(m) { handlerProviders = append(handlerProviders, toHandlerProvider(m)) } } for _, m := range h { - if h != nil { + if !isNilOrFuncNil(m) { handlerProviders = append(handlerProviders, toHandlerProvider(m)) } } diff --git a/routers/common/lfs.go b/routers/common/lfs.go new file mode 100644 index 00000000000..ba6e1163f1b --- /dev/null +++ b/routers/common/lfs.go @@ -0,0 +1,29 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package common + +import ( + "net/http" + + "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/services/lfs" +) + +func AddOwnerRepoGitLFSRoutes(m *web.Router, middlewares ...any) { + // shared by web and internal routers + m.Group("/{username}/{reponame}/info/lfs", func() { + m.Post("/objects/batch", lfs.CheckAcceptMediaType, lfs.BatchHandler) + m.Put("/objects/{oid}/{size}", lfs.UploadHandler) + m.Get("/objects/{oid}/{filename}", lfs.DownloadHandler) + m.Get("/objects/{oid}", lfs.DownloadHandler) + m.Post("/verify", lfs.CheckAcceptMediaType, lfs.VerifyHandler) + m.Group("/locks", func() { + m.Get("/", lfs.GetListLockHandler) + m.Post("/", lfs.PostLockHandler) + m.Post("/verify", lfs.VerifyLockHandler) + m.Post("/{lid}/unlock", lfs.UnLockHandler) + }, lfs.CheckAcceptMediaType) + m.Any("/*", http.NotFound) + }, middlewares...) +} diff --git a/routers/private/internal.go b/routers/private/internal.go index f9adff388cf..db074238c6e 100644 --- a/routers/private/internal.go +++ b/routers/private/internal.go @@ -5,6 +5,7 @@ package private import ( + "crypto/subtle" "net/http" "strings" @@ -14,28 +15,30 @@ import ( "code.gitea.io/gitea/modules/web" "code.gitea.io/gitea/routers/common" "code.gitea.io/gitea/services/context" - "code.gitea.io/gitea/services/lfs" "gitea.com/go-chi/binding" chi_middleware "github.com/go-chi/chi/v5/middleware" ) -// CheckInternalToken check internal token is set -func CheckInternalToken(next http.Handler) http.Handler { +const RouterMockPointInternalLFS = "internal-lfs" + +func authInternal(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - tokens := req.Header.Get("Authorization") - fields := strings.SplitN(tokens, " ", 2) if setting.InternalToken == "" { log.Warn(`The INTERNAL_TOKEN setting is missing from the configuration file: %q, internal API can't work.`, setting.CustomConf) http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) return } - if len(fields) != 2 || fields[0] != "Bearer" || fields[1] != setting.InternalToken { + + tokens := req.Header.Get("X-Gitea-Internal-Auth") // TODO: use something like JWT or HMAC to avoid passing the token in the clear + after, found := strings.CutPrefix(tokens, "Bearer ") + authSucceeded := found && subtle.ConstantTimeCompare([]byte(after), []byte(setting.InternalToken)) == 1 + if !authSucceeded { log.Debug("Forbidden attempt to access internal url: Authorization header: %s", tokens) http.Error(w, http.StatusText(http.StatusForbidden), http.StatusForbidden) - } else { - next.ServeHTTP(w, req) + return } + next.ServeHTTP(w, req) }) } @@ -48,20 +51,12 @@ func bind[T any](_ T) any { } } -// SwapAuthToken swaps Authorization header with X-Auth header -func swapAuthToken(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - req.Header.Set("Authorization", req.Header.Get("X-Auth")) - next.ServeHTTP(w, req) - }) -} - // Routes registers all internal APIs routes to web application. // These APIs will be invoked by internal commands for example `gitea serv` and etc. func Routes() *web.Router { r := web.NewRouter() r.Use(context.PrivateContexter()) - r.Use(CheckInternalToken) + r.Use(authInternal) // Log the real ip address of the request from SSH is really helpful for diagnosing sometimes. // Since internal API will be sent only from Gitea sub commands and it's under control (checked by InternalToken), we can trust the headers. r.Use(chi_middleware.RealIP) @@ -90,25 +85,13 @@ func Routes() *web.Router { r.Post("/restore_repo", RestoreRepo) r.Post("/actions/generate_actions_runner_token", GenerateActionsRunnerToken) - r.Group("/repo/{username}/{reponame}", func() { - r.Group("/info/lfs", func() { - r.Post("/objects/batch", lfs.CheckAcceptMediaType, lfs.BatchHandler) - r.Put("/objects/{oid}/{size}", lfs.UploadHandler) - r.Get("/objects/{oid}/{filename}", lfs.DownloadHandler) - r.Get("/objects/{oid}", lfs.DownloadHandler) - r.Post("/verify", lfs.CheckAcceptMediaType, lfs.VerifyHandler) - r.Group("/locks", func() { - r.Get("/", lfs.GetListLockHandler) - r.Post("/", lfs.PostLockHandler) - r.Post("/verify", lfs.VerifyLockHandler) - r.Post("/{lid}/unlock", lfs.UnLockHandler) - }, lfs.CheckAcceptMediaType) - r.Any("/*", func(ctx *context.Context) { - ctx.NotFound("", nil) - }) - }, swapAuthToken) - }, common.Sessioner(), context.Contexter()) - // end "/repo/{username}/{reponame}": git (LFS) API mirror + r.Group("/repo", func() { + // FIXME: it is not right to use context.Contexter here because all routes here should use PrivateContext + common.AddOwnerRepoGitLFSRoutes(r, func(ctx *context.PrivateContext) { + webContext := &context.Context{Base: ctx.Base} + ctx.AppendContextValue(context.WebContextKey, webContext) + }, web.RouterMockPoint(RouterMockPointInternalLFS)) + }) return r } diff --git a/routers/web/web.go b/routers/web/web.go index 907bf88f6f7..e0915e6a6ef 100644 --- a/routers/web/web.go +++ b/routers/web/web.go @@ -44,7 +44,6 @@ import ( auth_service "code.gitea.io/gitea/services/auth" "code.gitea.io/gitea/services/context" "code.gitea.io/gitea/services/forms" - "code.gitea.io/gitea/services/lfs" _ "code.gitea.io/gitea/modules/session" // to registers all internal adapters @@ -1598,23 +1597,8 @@ func registerRoutes(m *web.Router) { m.Post("/action/{action}", reqSignIn, repo.Action) }, ignSignIn, context.RepoAssignment, context.RepoRef()) + common.AddOwnerRepoGitLFSRoutes(m, ignSignInAndCsrf, lfsServerEnabled) m.Group("/{username}/{reponame}", func() { - m.Group("/info/lfs", func() { - m.Post("/objects/batch", lfs.CheckAcceptMediaType, lfs.BatchHandler) - m.Put("/objects/{oid}/{size}", lfs.UploadHandler) - m.Get("/objects/{oid}/{filename}", lfs.DownloadHandler) - m.Get("/objects/{oid}", lfs.DownloadHandler) - m.Post("/verify", lfs.CheckAcceptMediaType, lfs.VerifyHandler) - m.Group("/locks", func() { - m.Get("/", lfs.GetListLockHandler) - m.Post("/", lfs.PostLockHandler) - m.Post("/verify", lfs.VerifyLockHandler) - m.Post("/{lid}/unlock", lfs.UnLockHandler) - }, lfs.CheckAcceptMediaType) - m.Any("/*", func(ctx *context.Context) { - ctx.NotFound("", nil) - }) - }, ignSignInAndCsrf, lfsServerEnabled) gitHTTPRouters(m) }) // end "/{username}/{reponame}.git": git support diff --git a/tests/integration/api_repo_file_get_test.go b/tests/integration/api_repo_file_get_test.go index 4649babad1d..27bc9e25bfc 100644 --- a/tests/integration/api_repo_file_get_test.go +++ b/tests/integration/api_repo_file_get_test.go @@ -39,7 +39,7 @@ func TestAPIGetRawFileOrLFS(t *testing.T) { t.Run("Partial Clone", doPartialGitClone(dstPath2, u)) - lfs, _ := lfsCommitAndPushTest(t, dstPath) + lfs := lfsCommitAndPushTest(t, dstPath, littleSize)[0] reqLFS := NewRequest(t, "GET", "/api/v1/repos/user2/repo1/media/"+lfs) respLFS := MakeRequestNilResponseRecorder(t, reqLFS, http.StatusOK) diff --git a/tests/integration/git_test.go b/tests/integration/git_general_test.go similarity index 85% rename from tests/integration/git_test.go rename to tests/integration/git_general_test.go index 76db3c69324..7fd19e7edde 100644 --- a/tests/integration/git_test.go +++ b/tests/integration/git_general_test.go @@ -4,8 +4,6 @@ package integration import ( - "bytes" - "context" "crypto/rand" "encoding/hex" "fmt" @@ -26,27 +24,25 @@ import ( "code.gitea.io/gitea/models/unittest" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/git" - "code.gitea.io/gitea/modules/gitrepo" "code.gitea.io/gitea/modules/lfs" "code.gitea.io/gitea/modules/setting" api "code.gitea.io/gitea/modules/structs" gitea_context "code.gitea.io/gitea/services/context" - files_service "code.gitea.io/gitea/services/repository/files" "code.gitea.io/gitea/tests" "github.com/stretchr/testify/assert" ) const ( - littleSize = 1024 // 1ko - bigSize = 128 * 1024 * 1024 // 128Mo + littleSize = 1024 // 1K + bigSize = 128 * 1024 * 1024 // 128M ) -func TestGit(t *testing.T) { - onGiteaRun(t, testGit) +func TestGitGeneral(t *testing.T) { + onGiteaRun(t, testGitGeneral) } -func testGit(t *testing.T, u *url.URL) { +func testGitGeneral(t *testing.T, u *url.URL) { username := "user2" baseAPITestContext := NewAPITestContext(t, username, "repo1", auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) @@ -77,10 +73,10 @@ func testGit(t *testing.T, u *url.URL) { t.Run("Partial Clone", doPartialGitClone(dstPath2, u)) - little, big := standardCommitAndPushTest(t, dstPath) - littleLFS, bigLFS := lfsCommitAndPushTest(t, dstPath) - rawTest(t, &httpContext, little, big, littleLFS, bigLFS) - mediaTest(t, &httpContext, little, big, littleLFS, bigLFS) + pushedFilesStandard := standardCommitAndPushTest(t, dstPath, littleSize, bigSize) + pushedFilesLFS := lfsCommitAndPushTest(t, dstPath, littleSize, bigSize) + rawTest(t, &httpContext, pushedFilesStandard[0], pushedFilesStandard[1], pushedFilesLFS[0], pushedFilesLFS[1]) + mediaTest(t, &httpContext, pushedFilesStandard[0], pushedFilesStandard[1], pushedFilesLFS[0], pushedFilesLFS[1]) t.Run("CreateAgitFlowPull", doCreateAgitFlowPull(dstPath, &httpContext, "test/head")) t.Run("BranchProtectMerge", doBranchProtectPRMerge(&httpContext, dstPath)) @@ -89,8 +85,8 @@ func testGit(t *testing.T, u *url.URL) { t.Run("MergeFork", func(t *testing.T) { defer tests.PrintCurrentTest(t)() t.Run("CreatePRAndMerge", doMergeFork(httpContext, forkedUserCtx, "master", httpContext.Username+":master")) - rawTest(t, &forkedUserCtx, little, big, littleLFS, bigLFS) - mediaTest(t, &forkedUserCtx, little, big, littleLFS, bigLFS) + rawTest(t, &forkedUserCtx, pushedFilesStandard[0], pushedFilesStandard[1], pushedFilesLFS[0], pushedFilesLFS[1]) + mediaTest(t, &forkedUserCtx, pushedFilesStandard[0], pushedFilesStandard[1], pushedFilesLFS[0], pushedFilesLFS[1]) }) t.Run("PushCreate", doPushCreate(httpContext, u)) @@ -118,18 +114,18 @@ func testGit(t *testing.T, u *url.URL) { t.Run("Clone", doGitClone(dstPath, sshURL)) - little, big := standardCommitAndPushTest(t, dstPath) - littleLFS, bigLFS := lfsCommitAndPushTest(t, dstPath) - rawTest(t, &sshContext, little, big, littleLFS, bigLFS) - mediaTest(t, &sshContext, little, big, littleLFS, bigLFS) + pushedFilesStandard := standardCommitAndPushTest(t, dstPath, littleSize, bigSize) + pushedFilesLFS := lfsCommitAndPushTest(t, dstPath, littleSize, bigSize) + rawTest(t, &sshContext, pushedFilesStandard[0], pushedFilesStandard[1], pushedFilesLFS[0], pushedFilesLFS[1]) + mediaTest(t, &sshContext, pushedFilesStandard[0], pushedFilesStandard[1], pushedFilesLFS[0], pushedFilesLFS[1]) t.Run("CreateAgitFlowPull", doCreateAgitFlowPull(dstPath, &sshContext, "test/head2")) t.Run("BranchProtectMerge", doBranchProtectPRMerge(&sshContext, dstPath)) t.Run("MergeFork", func(t *testing.T) { defer tests.PrintCurrentTest(t)() t.Run("CreatePRAndMerge", doMergeFork(sshContext, forkedUserCtx, "master", sshContext.Username+":master")) - rawTest(t, &forkedUserCtx, little, big, littleLFS, bigLFS) - mediaTest(t, &forkedUserCtx, little, big, littleLFS, bigLFS) + rawTest(t, &forkedUserCtx, pushedFilesStandard[0], pushedFilesStandard[1], pushedFilesLFS[0], pushedFilesLFS[1]) + mediaTest(t, &forkedUserCtx, pushedFilesStandard[0], pushedFilesStandard[1], pushedFilesLFS[0], pushedFilesLFS[1]) }) t.Run("PushCreate", doPushCreate(sshContext, sshURL)) @@ -142,16 +138,16 @@ func ensureAnonymousClone(t *testing.T, u *url.URL) { t.Run("CloneAnonymous", doGitClone(dstLocalPath, u)) } -func standardCommitAndPushTest(t *testing.T, dstPath string) (little, big string) { - t.Run("Standard", func(t *testing.T) { +func standardCommitAndPushTest(t *testing.T, dstPath string, sizes ...int) (pushedFiles []string) { + t.Run("CommitAndPushStandard", func(t *testing.T) { defer tests.PrintCurrentTest(t)() - little, big = commitAndPushTest(t, dstPath, "data-file-") + pushedFiles = commitAndPushTest(t, dstPath, "data-file-", sizes...) }) - return little, big + return pushedFiles } -func lfsCommitAndPushTest(t *testing.T, dstPath string) (littleLFS, bigLFS string) { - t.Run("LFS", func(t *testing.T) { +func lfsCommitAndPushTest(t *testing.T, dstPath string, sizes ...int) (pushedFiles []string) { + t.Run("CommitAndPushLFS", func(t *testing.T) { defer tests.PrintCurrentTest(t)() prefix := "lfs-data-file-" err := git.NewCommand(git.DefaultContext, "lfs").AddArguments("install").Run(&git.RunOpts{Dir: dstPath}) @@ -176,33 +172,23 @@ func lfsCommitAndPushTest(t *testing.T, dstPath string) (littleLFS, bigLFS strin }) assert.NoError(t, err) - littleLFS, bigLFS = commitAndPushTest(t, dstPath, prefix) - + pushedFiles = commitAndPushTest(t, dstPath, prefix, sizes...) t.Run("Locks", func(t *testing.T) { defer tests.PrintCurrentTest(t)() lockTest(t, dstPath) }) }) - return littleLFS, bigLFS + return pushedFiles } -func commitAndPushTest(t *testing.T, dstPath, prefix string) (little, big string) { - t.Run("PushCommit", func(t *testing.T) { - defer tests.PrintCurrentTest(t)() - t.Run("Little", func(t *testing.T) { +func commitAndPushTest(t *testing.T, dstPath, prefix string, sizes ...int) (pushedFiles []string) { + for _, size := range sizes { + t.Run("PushCommit Size-"+strconv.Itoa(size), func(t *testing.T) { defer tests.PrintCurrentTest(t)() - little = doCommitAndPush(t, littleSize, dstPath, prefix) + pushedFiles = append(pushedFiles, doCommitAndPush(t, size, dstPath, prefix)) }) - t.Run("Big", func(t *testing.T) { - if testing.Short() { - t.Skip("Skipping test in short mode.") - return - } - defer tests.PrintCurrentTest(t)() - big = doCommitAndPush(t, bigSize, dstPath, prefix) - }) - }) - return little, big + } + return pushedFiles } func rawTest(t *testing.T, ctx *APITestContext, little, big, littleLFS, bigLFS string) { @@ -903,100 +889,3 @@ func doCreateAgitFlowPull(dstPath string, ctx *APITestContext, headBranch string t.Run("CheckoutMasterAgain", doGitCheckoutBranch(dstPath, "master")) } } - -func TestDataAsync_Issue29101(t *testing.T) { - onGiteaRun(t, func(t *testing.T, u *url.URL) { - user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) - repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) - - resp, err := files_service.ChangeRepoFiles(db.DefaultContext, repo, user, &files_service.ChangeRepoFilesOptions{ - Files: []*files_service.ChangeRepoFile{ - { - Operation: "create", - TreePath: "test.txt", - ContentReader: bytes.NewReader(make([]byte, 10000)), - }, - }, - OldBranch: repo.DefaultBranch, - NewBranch: repo.DefaultBranch, - }) - assert.NoError(t, err) - - sha := resp.Commit.SHA - - gitRepo, err := gitrepo.OpenRepository(db.DefaultContext, repo) - assert.NoError(t, err) - - commit, err := gitRepo.GetCommit(sha) - assert.NoError(t, err) - - entry, err := commit.GetTreeEntryByPath("test.txt") - assert.NoError(t, err) - - b := entry.Blob() - - r, err := b.DataAsync() - assert.NoError(t, err) - defer r.Close() - - r2, err := b.DataAsync() - assert.NoError(t, err) - defer r2.Close() - }) -} - -func TestAgitPullPush(t *testing.T) { - onGiteaRun(t, func(t *testing.T, u *url.URL) { - baseAPITestContext := NewAPITestContext(t, "user2", "repo1", auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) - - u.Path = baseAPITestContext.GitPath() - u.User = url.UserPassword("user2", userPassword) - - dstPath := t.TempDir() - doGitClone(dstPath, u)(t) - - gitRepo, err := git.OpenRepository(context.Background(), dstPath) - assert.NoError(t, err) - defer gitRepo.Close() - - doGitCreateBranch(dstPath, "test-agit-push") - - // commit 1 - _, err = generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-") - assert.NoError(t, err) - - // push to create an agit pull request - err = git.NewCommand(git.DefaultContext, "push", "origin", - "-o", "title=test-title", "-o", "description=test-description", - "HEAD:refs/for/master/test-agit-push", - ).Run(&git.RunOpts{Dir: dstPath}) - assert.NoError(t, err) - - // check pull request exist - pr := unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{BaseRepoID: 1, Flow: issues_model.PullRequestFlowAGit, HeadBranch: "user2/test-agit-push"}) - assert.NoError(t, pr.LoadIssue(db.DefaultContext)) - assert.Equal(t, "test-title", pr.Issue.Title) - assert.Equal(t, "test-description", pr.Issue.Content) - - // commit 2 - _, err = generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-2-") - assert.NoError(t, err) - - // push 2 - err = git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push").Run(&git.RunOpts{Dir: dstPath}) - assert.NoError(t, err) - - // reset to first commit - err = git.NewCommand(git.DefaultContext, "reset", "--hard", "HEAD~1").Run(&git.RunOpts{Dir: dstPath}) - assert.NoError(t, err) - - // test force push without confirm - _, stderr, err := git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push").RunStdString(&git.RunOpts{Dir: dstPath}) - assert.Error(t, err) - assert.Contains(t, stderr, "[remote rejected] HEAD -> refs/for/master/test-agit-push (request `force-push` push option)") - - // test force push with confirm - err = git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push", "-o", "force-push").Run(&git.RunOpts{Dir: dstPath}) - assert.NoError(t, err) - }) -} diff --git a/tests/integration/git_lfs_ssh_test.go b/tests/integration/git_lfs_ssh_test.go new file mode 100644 index 00000000000..33c2fba6208 --- /dev/null +++ b/tests/integration/git_lfs_ssh_test.go @@ -0,0 +1,61 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "net/url" + "sync" + "testing" + + auth_model "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/modules/setting" + "code.gitea.io/gitea/modules/web" + "code.gitea.io/gitea/routers/private" + "code.gitea.io/gitea/services/context" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGitLFSSSH(t *testing.T) { + onGiteaRun(t, func(t *testing.T, u *url.URL) { + dstPath := t.TempDir() + apiTestContext := NewAPITestContext(t, "user2", "repo1", auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) + + var mu sync.Mutex + var routerCalls []string + web.RouteMock(private.RouterMockPointInternalLFS, func(ctx *context.PrivateContext) { + mu.Lock() + routerCalls = append(routerCalls, ctx.Req.Method+" "+ctx.Req.URL.Path) + mu.Unlock() + }) + + withKeyFile(t, "my-testing-key", func(keyFile string) { + t.Run("CreateUserKey", doAPICreateUserKey(apiTestContext, "test-key", keyFile)) + cloneURL := createSSHUrl(apiTestContext.GitPath(), u) + t.Run("Clone", doGitClone(dstPath, cloneURL)) + + cfg, err := setting.CfgProvider.PrepareSaving() + require.NoError(t, err) + cfg.Section("server").Key("LFS_ALLOW_PURE_SSH").SetValue("true") + setting.LFS.AllowPureSSH = true + require.NoError(t, cfg.Save()) + + // do LFS SSH transfer? + lfsCommitAndPushTest(t, dstPath, 10) + }) + + // FIXME: Here we only see the following calls, but actually there should be calls to "PUT"? + // 0 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks" + // 1 = {string} "POST /api/internal/repo/user2/repo1.git/info/lfs/objects/batch" + // 2 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks" + // 3 = {string} "POST /api/internal/repo/user2/repo1.git/info/lfs/locks" + // 4 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks" + // 5 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks" + // 6 = {string} "GET /api/internal/repo/user2/repo1.git/info/lfs/locks" + // 7 = {string} "POST /api/internal/repo/user2/repo1.git/info/lfs/locks/24/unlock" + assert.NotEmpty(t, routerCalls) + // assert.Contains(t, routerCalls, "PUT /api/internal/repo/user2/repo1.git/info/lfs/objects/....") + }) +} diff --git a/tests/integration/git_misc_test.go b/tests/integration/git_misc_test.go new file mode 100644 index 00000000000..82ab184bb05 --- /dev/null +++ b/tests/integration/git_misc_test.go @@ -0,0 +1,138 @@ +// Copyright 2024 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package integration + +import ( + "bytes" + "context" + "io" + "net/url" + "sync" + "testing" + + auth_model "code.gitea.io/gitea/models/auth" + "code.gitea.io/gitea/models/db" + issues_model "code.gitea.io/gitea/models/issues" + repo_model "code.gitea.io/gitea/models/repo" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + "code.gitea.io/gitea/modules/git" + "code.gitea.io/gitea/modules/gitrepo" + files_service "code.gitea.io/gitea/services/repository/files" + + "github.com/stretchr/testify/assert" +) + +func TestDataAsyncDoubleRead_Issue29101(t *testing.T) { + onGiteaRun(t, func(t *testing.T, u *url.URL) { + user := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 2}) + repo := unittest.AssertExistsAndLoadBean(t, &repo_model.Repository{ID: 1}) + + testContent := bytes.Repeat([]byte{'a'}, 10000) + resp, err := files_service.ChangeRepoFiles(db.DefaultContext, repo, user, &files_service.ChangeRepoFilesOptions{ + Files: []*files_service.ChangeRepoFile{ + { + Operation: "create", + TreePath: "test.txt", + ContentReader: bytes.NewReader(testContent), + }, + }, + OldBranch: repo.DefaultBranch, + NewBranch: repo.DefaultBranch, + }) + assert.NoError(t, err) + + sha := resp.Commit.SHA + + gitRepo, err := gitrepo.OpenRepository(db.DefaultContext, repo) + assert.NoError(t, err) + + commit, err := gitRepo.GetCommit(sha) + assert.NoError(t, err) + + entry, err := commit.GetTreeEntryByPath("test.txt") + assert.NoError(t, err) + + b := entry.Blob() + r1, err := b.DataAsync() + assert.NoError(t, err) + defer r1.Close() + r2, err := b.DataAsync() + assert.NoError(t, err) + defer r2.Close() + + var data1, data2 []byte + wg := sync.WaitGroup{} + wg.Add(2) + go func() { + data1, _ = io.ReadAll(r1) + assert.NoError(t, err) + wg.Done() + }() + go func() { + data2, _ = io.ReadAll(r2) + assert.NoError(t, err) + wg.Done() + }() + wg.Wait() + assert.Equal(t, testContent, data1) + assert.Equal(t, testContent, data2) + }) +} + +func TestAgitPullPush(t *testing.T) { + onGiteaRun(t, func(t *testing.T, u *url.URL) { + baseAPITestContext := NewAPITestContext(t, "user2", "repo1", auth_model.AccessTokenScopeWriteRepository, auth_model.AccessTokenScopeWriteUser) + + u.Path = baseAPITestContext.GitPath() + u.User = url.UserPassword("user2", userPassword) + + dstPath := t.TempDir() + doGitClone(dstPath, u)(t) + + gitRepo, err := git.OpenRepository(context.Background(), dstPath) + assert.NoError(t, err) + defer gitRepo.Close() + + doGitCreateBranch(dstPath, "test-agit-push") + + // commit 1 + _, err = generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-") + assert.NoError(t, err) + + // push to create an agit pull request + err = git.NewCommand(git.DefaultContext, "push", "origin", + "-o", "title=test-title", "-o", "description=test-description", + "HEAD:refs/for/master/test-agit-push", + ).Run(&git.RunOpts{Dir: dstPath}) + assert.NoError(t, err) + + // check pull request exist + pr := unittest.AssertExistsAndLoadBean(t, &issues_model.PullRequest{BaseRepoID: 1, Flow: issues_model.PullRequestFlowAGit, HeadBranch: "user2/test-agit-push"}) + assert.NoError(t, pr.LoadIssue(db.DefaultContext)) + assert.Equal(t, "test-title", pr.Issue.Title) + assert.Equal(t, "test-description", pr.Issue.Content) + + // commit 2 + _, err = generateCommitWithNewData(littleSize, dstPath, "user2@example.com", "User Two", "branch-data-file-2-") + assert.NoError(t, err) + + // push 2 + err = git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push").Run(&git.RunOpts{Dir: dstPath}) + assert.NoError(t, err) + + // reset to first commit + err = git.NewCommand(git.DefaultContext, "reset", "--hard", "HEAD~1").Run(&git.RunOpts{Dir: dstPath}) + assert.NoError(t, err) + + // test force push without confirm + _, stderr, err := git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push").RunStdString(&git.RunOpts{Dir: dstPath}) + assert.Error(t, err) + assert.Contains(t, stderr, "[remote rejected] HEAD -> refs/for/master/test-agit-push (request `force-push` push option)") + + // test force push with confirm + err = git.NewCommand(git.DefaultContext, "push", "origin", "HEAD:refs/for/master/test-agit-push", "-o", "force-push").Run(&git.RunOpts{Dir: dstPath}) + assert.NoError(t, err) + }) +} diff --git a/tests/test_utils.go b/tests/test_utils.go index e6ce3cce0e0..3503ca19758 100644 --- a/tests/test_utils.go +++ b/tests/test_utils.go @@ -9,7 +9,6 @@ import ( "database/sql" "fmt" "os" - "path" "path/filepath" "testing" @@ -53,7 +52,7 @@ func InitTest(requireGitea bool) { if setting.IsWindows { giteaBinary += ".exe" } - setting.AppPath = path.Join(giteaRoot, giteaBinary) + setting.AppPath = filepath.Join(giteaRoot, giteaBinary) if _, err := os.Stat(setting.AppPath); err != nil { exitf("Could not find gitea binary at %s", setting.AppPath) } @@ -70,7 +69,7 @@ func InitTest(requireGitea bool) { exitf(`sqlite3 requires: import _ "github.com/mattn/go-sqlite3" or -tags sqlite,sqlite_unlock_notify`) } } - if !path.IsAbs(giteaConf) { + if !filepath.IsAbs(giteaConf) { setting.CustomConf = filepath.Join(giteaRoot, giteaConf) } else { setting.CustomConf = giteaConf @@ -193,8 +192,12 @@ func PrepareAttachmentsStorage(t testing.TB) { } func PrepareGitRepoDirectory(t testing.TB) { + if !assert.NotEmpty(t, setting.RepoRootPath) { + return + } + assert.NoError(t, util.RemoveAll(setting.RepoRootPath)) - assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) + assert.NoError(t, unittest.CopyDir(filepath.Join(filepath.Dir(setting.AppPath), "tests/gitea-repositories-meta"), setting.RepoRootPath)) ownerDirs, err := os.ReadDir(setting.RepoRootPath) if err != nil { From 2763766f8563bd1abbc5806938103ae5e479a311 Mon Sep 17 00:00:00 2001 From: Albin Hedman Date: Tue, 12 Nov 2024 03:57:30 +0100 Subject: [PATCH 2/4] cargo registry - respect renamed dependencies (#32430) rust allows renaming dependencies such as when depending on multiple versions of the same package. This is not supported by gitea as discovered in #31500 . This PR tries to address that. --------- Co-authored-by: wxiaoguang --- modules/packages/cargo/parser.go | 11 ++++- modules/packages/cargo/parser_test.go | 58 +++++++++++++++++++-------- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/modules/packages/cargo/parser.go b/modules/packages/cargo/parser.go index 36cd44df847..d82e0e2f058 100644 --- a/modules/packages/cargo/parser.go +++ b/modules/packages/cargo/parser.go @@ -136,8 +136,16 @@ func parsePackage(r io.Reader) (*Package, error) { dependencies := make([]*Dependency, 0, len(meta.Deps)) for _, dep := range meta.Deps { + // https://doc.rust-lang.org/cargo/reference/registry-web-api.html#publish + // It is a string of the new package name if the dependency is renamed, otherwise empty + name := dep.ExplicitNameInToml + pkg := &dep.Name + if name == "" { + name = dep.Name + pkg = nil + } dependencies = append(dependencies, &Dependency{ - Name: dep.Name, + Name: name, Req: dep.VersionReq, Features: dep.Features, Optional: dep.Optional, @@ -145,6 +153,7 @@ func parsePackage(r io.Reader) (*Package, error) { Target: dep.Target, Kind: dep.Kind, Registry: dep.Registry, + Package: pkg, }) } diff --git a/modules/packages/cargo/parser_test.go b/modules/packages/cargo/parser_test.go index 2230a5b4999..0a120b8074c 100644 --- a/modules/packages/cargo/parser_test.go +++ b/modules/packages/cargo/parser_test.go @@ -13,16 +13,16 @@ import ( "github.com/stretchr/testify/assert" ) -const ( - description = "Package Description" - author = "KN4CK3R" - homepage = "https://gitea.io/" - license = "MIT" -) - func TestParsePackage(t *testing.T) { - createPackage := func(name, version string) io.Reader { - metadata := `{ + const ( + description = "Package Description" + author = "KN4CK3R" + homepage = "https://gitea.io/" + license = "MIT" + payload = "gitea test dummy payload" // a fake payload for test only + ) + makeDefaultPackageMeta := func(name, version string) string { + return `{ "name":"` + name + `", "vers":"` + version + `", "description":"` + description + `", @@ -36,18 +36,19 @@ func TestParsePackage(t *testing.T) { "homepage":"` + homepage + `", "license":"` + license + `" }` - + } + createPackage := func(metadata string) io.Reader { var buf bytes.Buffer binary.Write(&buf, binary.LittleEndian, uint32(len(metadata))) buf.WriteString(metadata) - binary.Write(&buf, binary.LittleEndian, uint32(4)) - buf.WriteString("test") + binary.Write(&buf, binary.LittleEndian, uint32(len(payload))) + buf.WriteString(payload) return &buf } t.Run("InvalidName", func(t *testing.T) { for _, name := range []string{"", "0test", "-test", "_test", strings.Repeat("a", 65)} { - data := createPackage(name, "1.0.0") + data := createPackage(makeDefaultPackageMeta(name, "1.0.0")) cp, err := ParsePackage(data) assert.Nil(t, cp) @@ -57,7 +58,7 @@ func TestParsePackage(t *testing.T) { t.Run("InvalidVersion", func(t *testing.T) { for _, version := range []string{"", "1.", "-1.0", "1.0.0/1"} { - data := createPackage("test", version) + data := createPackage(makeDefaultPackageMeta("test", version)) cp, err := ParsePackage(data) assert.Nil(t, cp) @@ -66,7 +67,7 @@ func TestParsePackage(t *testing.T) { }) t.Run("Valid", func(t *testing.T) { - data := createPackage("test", "1.0.0") + data := createPackage(makeDefaultPackageMeta("test", "1.0.0")) cp, err := ParsePackage(data) assert.NotNil(t, cp) @@ -78,9 +79,34 @@ func TestParsePackage(t *testing.T) { assert.Equal(t, []string{author}, cp.Metadata.Authors) assert.Len(t, cp.Metadata.Dependencies, 1) assert.Equal(t, "dep", cp.Metadata.Dependencies[0].Name) + assert.Nil(t, cp.Metadata.Dependencies[0].Package) assert.Equal(t, homepage, cp.Metadata.ProjectURL) assert.Equal(t, license, cp.Metadata.License) content, _ := io.ReadAll(cp.Content) - assert.Equal(t, "test", string(content)) + assert.Equal(t, payload, string(content)) + }) + + t.Run("Renamed", func(t *testing.T) { + data := createPackage(`{ + "name":"test-pkg", + "vers":"1.0", + "description":"test-desc", + "authors": ["test-author"], + "deps":[ + { + "name":"dep-renamed", + "explicit_name_in_toml":"dep-explicit", + "version_req":"1.0" + } + ], + "homepage":"https://gitea.io/", + "license":"MIT" +}`) + cp, err := ParsePackage(data) + assert.NoError(t, err) + assert.Equal(t, "test-pkg", cp.Name) + assert.Equal(t, "https://gitea.io/", cp.Metadata.ProjectURL) + assert.Equal(t, "dep-explicit", cp.Metadata.Dependencies[0].Name) + assert.Equal(t, "dep-renamed", *cp.Metadata.Dependencies[0].Package) }) } From 4c924bf43cdc944c6ce34cce54f216c455a346a2 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Tue, 12 Nov 2024 04:44:24 +0100 Subject: [PATCH 3/4] Limit org member view of restricted users (#32211) currently restricted users can only see the repos of teams in orgs they are part at. they also should only see the users that are also part at the same team. --- *Sponsored by Kithara Software GmbH* --- models/fixtures/org_user.yml | 6 +++ models/fixtures/user.yml | 2 +- models/organization/org.go | 33 +++++++++++++++- models/organization/org_test.go | 70 +++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+), 3 deletions(-) diff --git a/models/fixtures/org_user.yml b/models/fixtures/org_user.yml index cf21b84aa9f..73a3e9dba9b 100644 --- a/models/fixtures/org_user.yml +++ b/models/fixtures/org_user.yml @@ -129,3 +129,9 @@ uid: 2 org_id: 35 is_public: true + +- + id: 23 + uid: 20 + org_id: 17 + is_public: false diff --git a/models/fixtures/user.yml b/models/fixtures/user.yml index c0296deec55..1044e487f81 100644 --- a/models/fixtures/user.yml +++ b/models/fixtures/user.yml @@ -623,7 +623,7 @@ num_stars: 0 num_repos: 2 num_teams: 3 - num_members: 4 + num_members: 5 visibility: 0 repo_admin_change_team_access: false theme: "" diff --git a/models/organization/org.go b/models/organization/org.go index 28a46ec8f50..6231f1eeedf 100644 --- a/models/organization/org.go +++ b/models/organization/org.go @@ -22,6 +22,7 @@ import ( "code.gitea.io/gitea/modules/util" "xorm.io/builder" + "xorm.io/xorm" ) // ________ .__ __ .__ @@ -205,11 +206,28 @@ func (opts FindOrgMembersOpts) PublicOnly() bool { return opts.Doer == nil || !(opts.IsDoerMember || opts.Doer.IsAdmin) } +// applyTeamMatesOnlyFilter make sure restricted users only see public team members and there own team mates +func (opts FindOrgMembersOpts) applyTeamMatesOnlyFilter(sess *xorm.Session) { + if opts.Doer != nil && opts.IsDoerMember && opts.Doer.IsRestricted { + teamMates := builder.Select("DISTINCT team_user.uid"). + From("team_user"). + Where(builder.In("team_user.team_id", getUserTeamIDsQueryBuilder(opts.OrgID, opts.Doer.ID))). + And(builder.Eq{"team_user.org_id": opts.OrgID}) + + sess.And( + builder.In("org_user.uid", teamMates). + Or(builder.Eq{"org_user.is_public": true}), + ) + } +} + // CountOrgMembers counts the organization's members func CountOrgMembers(ctx context.Context, opts *FindOrgMembersOpts) (int64, error) { sess := db.GetEngine(ctx).Where("org_id=?", opts.OrgID) if opts.PublicOnly() { - sess.And("is_public = ?", true) + sess = sess.And("is_public = ?", true) + } else { + opts.applyTeamMatesOnlyFilter(sess) } return sess.Count(new(OrgUser)) @@ -533,7 +551,9 @@ func GetOrgsCanCreateRepoByUserID(ctx context.Context, userID int64) ([]*Organiz func GetOrgUsersByOrgID(ctx context.Context, opts *FindOrgMembersOpts) ([]*OrgUser, error) { sess := db.GetEngine(ctx).Where("org_id=?", opts.OrgID) if opts.PublicOnly() { - sess.And("is_public = ?", true) + sess = sess.And("is_public = ?", true) + } else { + opts.applyTeamMatesOnlyFilter(sess) } if opts.ListOptions.PageSize > 0 { @@ -664,6 +684,15 @@ func (org *Organization) getUserTeamIDs(ctx context.Context, userID int64) ([]in Find(&teamIDs) } +func getUserTeamIDsQueryBuilder(orgID, userID int64) *builder.Builder { + return builder.Select("team.id").From("team"). + InnerJoin("team_user", "team_user.team_id = team.id"). + Where(builder.Eq{ + "team_user.org_id": orgID, + "team_user.uid": userID, + }) +} + // TeamsWithAccessToRepo returns all teams that have given access level to the repository. func (org *Organization) TeamsWithAccessToRepo(ctx context.Context, repoID int64, mode perm.AccessMode) ([]*Team, error) { return GetTeamsWithAccessToRepo(ctx, org.ID, repoID, mode) diff --git a/models/organization/org_test.go b/models/organization/org_test.go index 5442c37ccc9..c614aaacf56 100644 --- a/models/organization/org_test.go +++ b/models/organization/org_test.go @@ -4,6 +4,7 @@ package organization_test import ( + "slices" "sort" "testing" @@ -181,6 +182,75 @@ func TestIsPublicMembership(t *testing.T) { test(unittest.NonexistentID, unittest.NonexistentID, false) } +func TestRestrictedUserOrgMembers(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + restrictedUser := unittest.AssertExistsAndLoadBean(t, &user_model.User{ + ID: 29, + IsRestricted: true, + }) + if !assert.True(t, restrictedUser.IsRestricted) { + return // ensure fixtures return restricted user + } + + testCases := []struct { + name string + opts *organization.FindOrgMembersOpts + expectedUIDs []int64 + }{ + { + name: "restricted user sees public members and teammates", + opts: &organization.FindOrgMembersOpts{ + OrgID: 17, // org17 where user29 is in team9 + Doer: restrictedUser, + IsDoerMember: true, + }, + expectedUIDs: []int64{2, 15, 20, 29}, // Public members (2) + teammates in team9 (15, 20, 29) + }, + { + name: "restricted user sees only public members when not member", + opts: &organization.FindOrgMembersOpts{ + OrgID: 3, // org3 where user29 is not a member + Doer: restrictedUser, + }, + expectedUIDs: []int64{2, 28}, // Only public members + }, + { + name: "non logged in only shows public members", + opts: &organization.FindOrgMembersOpts{ + OrgID: 3, + }, + expectedUIDs: []int64{2, 28}, // Only public members + }, + { + name: "non restricted user sees all members", + opts: &organization.FindOrgMembersOpts{ + OrgID: 17, + Doer: unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 15}), + IsDoerMember: true, + }, + expectedUIDs: []int64{2, 15, 18, 20, 29}, // All members + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + count, err := organization.CountOrgMembers(db.DefaultContext, tc.opts) + assert.NoError(t, err) + assert.EqualValues(t, len(tc.expectedUIDs), count) + + members, err := organization.GetOrgUsersByOrgID(db.DefaultContext, tc.opts) + assert.NoError(t, err) + memberUIDs := make([]int64, 0, len(members)) + for _, member := range members { + memberUIDs = append(memberUIDs, member.UID) + } + slices.Sort(memberUIDs) + assert.EqualValues(t, tc.expectedUIDs, memberUIDs) + }) + } +} + func TestFindOrgs(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) From 160ccb5ee2c9dce7e59fa2635c8539e0338d83ee Mon Sep 17 00:00:00 2001 From: wxiaoguang Date: Tue, 12 Nov 2024 13:41:22 +0800 Subject: [PATCH 4/4] Fix test fixtures for user2/lfs.git (#32477) --- models/fixtures/lfs_meta_object.yml | 18 +----------------- .../30/77e1c4c8964613df72c37d14275c1eda5228a9 | 2 -- .../6b/bc79965141058b0026f2064dfb6d2eae3c4540 | Bin 259 -> 0 bytes .../b0/89e97ee59224e8c5676673c096ee4b6a8b9342 | Bin 123 -> 0 bytes .../e9/c32647bab825977942598c0efa415de300304b | Bin 170 -> 0 bytes .../user2/lfs.git/refs/heads/master | 2 +- tests/integration/lfs_view_test.go | 17 ++++++++++++----- 7 files changed, 14 insertions(+), 25 deletions(-) delete mode 100644 tests/gitea-repositories-meta/user2/lfs.git/objects/30/77e1c4c8964613df72c37d14275c1eda5228a9 delete mode 100644 tests/gitea-repositories-meta/user2/lfs.git/objects/6b/bc79965141058b0026f2064dfb6d2eae3c4540 delete mode 100644 tests/gitea-repositories-meta/user2/lfs.git/objects/b0/89e97ee59224e8c5676673c096ee4b6a8b9342 delete mode 100644 tests/gitea-repositories-meta/user2/lfs.git/objects/e9/c32647bab825977942598c0efa415de300304b diff --git a/models/fixtures/lfs_meta_object.yml b/models/fixtures/lfs_meta_object.yml index 5430506d70a..ae5ae565425 100644 --- a/models/fixtures/lfs_meta_object.yml +++ b/models/fixtures/lfs_meta_object.yml @@ -1,11 +1,4 @@ # These are the LFS objects in user2/lfs.git -# user2/lfs is an INVALID repository -# -# commit e9c32647bab825977942598c0efa415de300304b (HEAD -> master) -# Author: Rowan Bohde -# Date: Thu Aug 1 14:38:23 2024 -0500 -# -# add invalid lfs file - id: 1 @@ -18,7 +11,7 @@ id: 2 oid: 2eccdb43825d2a49d99d542daa20075cff1d97d9d2349a8977efe9c03661737c - size: 107 # real size is 2048 + size: 2048 repository_id: 54 created_unix: 1671607299 @@ -37,12 +30,3 @@ size: 25 repository_id: 54 created_unix: 1671607299 - -# this file is missing -# - -# -# id: 5 -# oid: 9d178b5f15046343fd32f451df93acc2bdd9e6373be478b968e4cad6b6647351 -# size: 25 -# repository_id: 54 -# created_unix: 1671607299 diff --git a/tests/gitea-repositories-meta/user2/lfs.git/objects/30/77e1c4c8964613df72c37d14275c1eda5228a9 b/tests/gitea-repositories-meta/user2/lfs.git/objects/30/77e1c4c8964613df72c37d14275c1eda5228a9 deleted file mode 100644 index c2dc6e5a4f4..00000000000 --- a/tests/gitea-repositories-meta/user2/lfs.git/objects/30/77e1c4c8964613df72c37d14275c1eda5228a9 +++ /dev/null @@ -1,2 +0,0 @@ -xKÊÉOR0´0`pö÷ òt - ñôs×ËMQHËÌ)I-²ÍI+VHÉLK3rS‹ÒSÁ,Ý’ÔŠ.-½¬‚t"U&eæ¥23¯,1'“8ûØæAÅ \ No newline at end of file diff --git a/tests/gitea-repositories-meta/user2/lfs.git/objects/6b/bc79965141058b0026f2064dfb6d2eae3c4540 b/tests/gitea-repositories-meta/user2/lfs.git/objects/6b/bc79965141058b0026f2064dfb6d2eae3c4540 deleted file mode 100644 index 97455cbc46ed6801ec009d2a6b4cd2ce2347bcfa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 259 zcmV+e0sQ`W0V^p=O;s>5GiER}FfcPQQP4}zEJ-XWDauSLElDkAFera`L;rm60 zYem#!&dKpx_e*7yzQUx#SH0EjKjpuUfGQ1g zb#(D{1u1GOyR>L}<@y;qT)(Dg->$R{y5h6~swlarvY2PJph~g|Qq%Ra3ep)$ z6c2+oLUCzQN@fwmo~L{(J~6+WWE+*j%c{_|--yj4 J4*(iHX&$FKd=UTu diff --git a/tests/gitea-repositories-meta/user2/lfs.git/objects/b0/89e97ee59224e8c5676673c096ee4b6a8b9342 b/tests/gitea-repositories-meta/user2/lfs.git/objects/b0/89e97ee59224e8c5676673c096ee4b6a8b9342 deleted file mode 100644 index 33ab64e7303e418c3270708ab1134aa417e2f8a5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 123 zcmV->0EGW|0S(H*5yBu406^bVK?4Ti;0Wo4<3N~+k`c_q>dk9EOM54&jlZ4wGg^Pk zI_EJqrJilx_cE5t`lTiHml{V->eQk)mZL`Fa0{&cO0H=4um~0kgDKW&E&*anhL}S} d@IcvKG2ohQij0J4Jvc2!`(|>;)QDtA}L>CG-fnfu!AS1h!QiCB1%~E97$q2B!9P zZ3D>7A7dARSW2DdIR@)3hExzKDq-}jB{E4CSAFnke)BHdfXP)w$0bS?=fzUd*-A>m zaSci1l$10PoBSBJzQeWu0Z_Lv;< YW^OVwwC$Of#<1uev@K%(0i~r&qPFu