diff --git a/routers/api/packages/api.go b/routers/api/packages/api.go
index d17e4875b13..6dc2e813e9b 100644
--- a/routers/api/packages/api.go
+++ b/routers/api/packages/api.go
@@ -115,6 +115,7 @@ func CommonRoutes() *web.Router {
 
 	verifyAuth(r, []auth.Method{
 		&auth.OAuth2{},
+		&auth.AccessToken{},
 		&auth.Basic{},
 		&nuget.Auth{},
 		&conan.Auth{},
@@ -671,6 +672,7 @@ func ContainerRoutes() *web.Router {
 	r.Use(context.PackageContexter())
 
 	verifyAuth(r, []auth.Method{
+		&auth.AccessToken{},
 		&auth.Basic{},
 		&container.Auth{},
 	})
diff --git a/routers/api/v1/api.go b/routers/api/v1/api.go
index 23f466873ba..e623f1b08d9 100644
--- a/routers/api/v1/api.go
+++ b/routers/api/v1/api.go
@@ -739,6 +739,7 @@ func buildAuthGroup() *auth.Group {
 	group := auth.NewGroup(
 		&auth.OAuth2{},
 		&auth.HTTPSign{},
+		&auth.AccessToken{},
 		&auth.Basic{}, // FIXME: this should be removed once we don't allow basic auth in API
 	)
 	if setting.Service.EnableReverseProxyAuthAPI {
diff --git a/routers/web/web.go b/routers/web/web.go
index 137c6773065..87036f08f2b 100644
--- a/routers/web/web.go
+++ b/routers/web/web.go
@@ -99,7 +99,8 @@ func optionsCorsHandler() func(next http.Handler) http.Handler {
 func buildAuthGroup() *auth_service.Group {
 	group := auth_service.NewGroup()
 	group.Add(&auth_service.OAuth2{}) // FIXME: this should be removed and only applied in download and oauth related routers
-	group.Add(&auth_service.Basic{})  // FIXME: this should be removed and only applied in download and git/lfs routers
+	group.Add(&auth_service.AccessToken{})
+	group.Add(&auth_service.Basic{}) // FIXME: this should be removed and only applied in download and git/lfs routers
 
 	if setting.Service.EnableReverseProxyAuth {
 		group.Add(&auth_service.ReverseProxy{}) // reverseproxy should before Session, otherwise the header will be ignored if user has login
diff --git a/services/auth/access_token.go b/services/auth/access_token.go
new file mode 100644
index 00000000000..53195cfd720
--- /dev/null
+++ b/services/auth/access_token.go
@@ -0,0 +1,113 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package auth
+
+import (
+	"net/http"
+	"strings"
+
+	auth_model "code.gitea.io/gitea/models/auth"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/base"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/timeutil"
+	"code.gitea.io/gitea/modules/web/middleware"
+)
+
+// Ensure the struct implements the interface.
+var (
+	_ Method = &AccessToken{}
+)
+
+// BasicMethodName is the constant name of the basic authentication method
+const (
+	AccessTokenMethodName = "access_token"
+)
+
+// AccessToken implements the Auth interface and authenticates requests (API requests
+// only) by looking for access token
+type AccessToken struct{}
+
+// Name represents the name of auth method
+func (b *AccessToken) Name() string {
+	return AccessTokenMethodName
+}
+
+// Match returns true if the request matched AccessToken requirements
+// TODO: remove path check once AccessToken will not be a global middleware but only
+// for specific routes
+func (b *AccessToken) Match(req *http.Request) bool {
+	if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawOrAttachOrLFSPath(req) {
+		return false
+	}
+	baHead := req.Header.Get("Authorization")
+	if baHead == "" {
+		return false
+	}
+	auths := strings.SplitN(baHead, " ", 2)
+	if len(auths) != 2 || (strings.ToLower(auths[0]) != "basic") {
+		return false
+	}
+	return true
+}
+
+// Verify extracts and validates Basic data (username and password/token) from the
+// "Authorization" header of the request and returns the corresponding user object for that
+// name/token on successful validation.
+// Returns nil if header is empty or validation fails.
+func (b *AccessToken) Verify(req *http.Request, w http.ResponseWriter, store DataStore, sess SessionStore) (*user_model.User, error) {
+	// Basic authentication should only fire on API, Download or on Git or LFSPaths
+	if !middleware.IsAPIPath(req) && !isContainerPath(req) && !isAttachmentDownload(req) && !isGitRawOrAttachOrLFSPath(req) {
+		return nil, nil
+	}
+
+	baHead := req.Header.Get("Authorization")
+	if len(baHead) == 0 {
+		return nil, nil
+	}
+
+	auths := strings.SplitN(baHead, " ", 2)
+	if len(auths) != 2 || (strings.ToLower(auths[0]) != "basic") {
+		return nil, nil
+	}
+
+	uname, passwd, _ := base.BasicAuthDecode(auths[1])
+
+	// Check if username or password is a token
+	isUsernameToken := len(passwd) == 0 || passwd == "x-oauth-basic"
+	// Assume username is token
+	authToken := uname
+	if !isUsernameToken {
+		log.Trace("Basic Authorization: Attempting login for: %s", uname)
+		// Assume password is token
+		authToken = passwd
+	} else {
+		log.Trace("Basic Authorization: Attempting login with username as token")
+	}
+
+	// check personal access token
+	token, err := auth_model.GetAccessTokenBySHA(req.Context(), authToken)
+	if err == nil {
+		log.Trace("Basic Authorization: Valid AccessToken for user[%d]", token.UID)
+		u, err := user_model.GetUserByID(req.Context(), token.UID)
+		if err != nil {
+			log.Error("GetUserByID:  %v", err)
+			return nil, err
+		}
+
+		token.UpdatedUnix = timeutil.TimeStampNow()
+		if err = auth_model.UpdateAccessToken(req.Context(), token); err != nil {
+			log.Error("UpdateAccessToken:  %v", err)
+		}
+
+		store.GetData()["LoginMethod"] = AccessTokenMethodName
+		store.GetData()["IsApiToken"] = true
+		store.GetData()["ApiTokenScope"] = token.Scope
+		return u, nil
+	} else if !auth_model.IsErrAccessTokenNotExist(err) && !auth_model.IsErrAccessTokenEmpty(err) {
+		log.Error("GetAccessTokenBySha: %v", err)
+	}
+
+	return nil, nil
+}
diff --git a/services/auth/basic.go b/services/auth/basic.go
index 02526678ea5..22bce55173d 100644
--- a/services/auth/basic.go
+++ b/services/auth/basic.go
@@ -14,7 +14,6 @@ import (
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
-	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/modules/web/middleware"
 )
@@ -27,7 +26,6 @@ var (
 // BasicMethodName is the constant name of the basic authentication method
 const (
 	BasicMethodName       = "basic"
-	AccessTokenMethodName = "access_token"
 	OAuth2TokenMethodName = "oauth2_token"
 	ActionTokenMethodName = "action_token"
 )
@@ -96,29 +94,6 @@ func (b *Basic) Verify(req *http.Request, w http.ResponseWriter, store DataStore
 		return u, nil
 	}
 
-	// check personal access token
-	token, err := auth_model.GetAccessTokenBySHA(req.Context(), authToken)
-	if err == nil {
-		log.Trace("Basic Authorization: Valid AccessToken for user[%d]", uid)
-		u, err := user_model.GetUserByID(req.Context(), token.UID)
-		if err != nil {
-			log.Error("GetUserByID:  %v", err)
-			return nil, err
-		}
-
-		token.UpdatedUnix = timeutil.TimeStampNow()
-		if err = auth_model.UpdateAccessToken(req.Context(), token); err != nil {
-			log.Error("UpdateAccessToken:  %v", err)
-		}
-
-		store.GetData()["LoginMethod"] = AccessTokenMethodName
-		store.GetData()["IsApiToken"] = true
-		store.GetData()["ApiTokenScope"] = token.Scope
-		return u, nil
-	} else if !auth_model.IsErrAccessTokenNotExist(err) && !auth_model.IsErrAccessTokenEmpty(err) {
-		log.Error("GetAccessTokenBySha: %v", err)
-	}
-
 	// check task token
 	task, err := actions_model.GetRunningTaskByToken(req.Context(), authToken)
 	if err == nil && task != nil {
diff --git a/services/auth/oauth2.go b/services/auth/oauth2.go
index 089bbc44f2a..07d8bcb6023 100644
--- a/services/auth/oauth2.go
+++ b/services/auth/oauth2.go
@@ -131,8 +131,17 @@ func (o *OAuth2) userIDFromToken(ctx context.Context, tokenSHA string, store Dat
 	return t.UID
 }
 
+// Match returns true if the request matched OAuth2 requirements
+// TODO: remove path check once AccessToken will not be a global middleware but only
+// for specific routes
 func (o *OAuth2) Match(req *http.Request) bool {
-	return true
+	if !middleware.IsAPIPath(req) && !isAttachmentDownload(req) && !isAuthenticatedTokenRequest(req) &&
+		!isGitRawOrAttachPath(req) && !isArchivePath(req) {
+		return false
+	}
+
+	_, ok := parseToken(req)
+	return ok
 }
 
 // Verify extracts the user ID from the OAuth token in the query parameters