mirror of
https://github.com/go-gitea/gitea
synced 2024-09-10 08:27:04 +02:00
Add Allow-/Block-List for Migrate & Mirrors (#13610)
* add black list and white list support for migrating repositories * fix fmt * fix lint * fix vendor * fix modules.txt * clean diff * specify log message * use blocklist/allowlist * allways use lowercase to match url * Apply allow/block * Settings: use existing "migrations" section * convert domains lower case * dont store unused value * Block private addresses for migration by default * fix lint * use proposed-upstream func to detect private IP addr * a nit * add own error for blocked migration, add tests, imprufe api * fix test * fix-if-localhost-is-ipv4 * rename error & error message * rename setting options * Apply suggestions from code review Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> Co-authored-by: zeripath <art27@cantab.net> Co-authored-by: techknowlogick <techknowlogick@gitea.io>
This commit is contained in:
parent
0f14f69e60
commit
b2435af9be
@ -1187,6 +1187,14 @@ QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
|
||||
MAX_ATTEMPTS = 3
|
||||
; Backoff time per http/https request retry (seconds)
|
||||
RETRY_BACKOFF = 3
|
||||
; Allowed domains for migrating, default is blank. Blank means everything will be allowed.
|
||||
; Multiple domains could be separated by commas.
|
||||
ALLOWED_DOMAINS =
|
||||
; Blocklist for migrating, default is blank. Multiple domains could be separated by commas.
|
||||
; When ALLOWED_DOMAINS is not blank, this option will be ignored.
|
||||
BLOCKED_DOMAINS =
|
||||
; Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291 (false by default)
|
||||
ALLOW_LOCALNETWORKS = false
|
||||
|
||||
; default storage for attachments, lfs and avatars
|
||||
[storage]
|
||||
|
@ -846,6 +846,9 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
|
||||
|
||||
- `MAX_ATTEMPTS`: **3**: Max attempts per http/https request on migrations.
|
||||
- `RETRY_BACKOFF`: **3**: Backoff time per http/https request retry (seconds)
|
||||
- `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas.
|
||||
- `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored.
|
||||
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291
|
||||
|
||||
## Mirror (`mirror`)
|
||||
|
||||
|
@ -313,6 +313,9 @@ IS_INPUT_FILE = false
|
||||
|
||||
- `MAX_ATTEMPTS`: **3**: 在迁移过程中的 http/https 请求重试次数。
|
||||
- `RETRY_BACKOFF`: **3**: 等待下一次重试的时间,单位秒。
|
||||
- `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单,默认为空,表示允许从任意域名迁移仓库,多个域名用逗号分隔。
|
||||
- `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单,默认为空,多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空,此选项将会被忽略。
|
||||
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918
|
||||
|
||||
## LFS (`lfs`)
|
||||
|
||||
|
@ -309,6 +309,8 @@ func TestAPIRepoMigrate(t *testing.T) {
|
||||
{ctxUserID: 2, userID: 1, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad", expectedStatus: http.StatusForbidden},
|
||||
{ctxUserID: 2, userID: 3, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-org", expectedStatus: http.StatusCreated},
|
||||
{ctxUserID: 2, userID: 6, cloneURL: "https://github.com/go-gitea/test_repo.git", repoName: "git-bad-org", expectedStatus: http.StatusForbidden},
|
||||
{ctxUserID: 2, userID: 3, cloneURL: "https://localhost:3000/user/test_repo.git", repoName: "local-ip", expectedStatus: http.StatusUnprocessableEntity},
|
||||
{ctxUserID: 2, userID: 3, cloneURL: "https://10.0.0.1/user/test_repo.git", repoName: "private-ip", expectedStatus: http.StatusUnprocessableEntity},
|
||||
}
|
||||
|
||||
defer prepareTestEnv(t)()
|
||||
@ -325,8 +327,16 @@ func TestAPIRepoMigrate(t *testing.T) {
|
||||
if resp.Code == http.StatusUnprocessableEntity {
|
||||
respJSON := map[string]string{}
|
||||
DecodeJSON(t, resp, &respJSON)
|
||||
if assert.Equal(t, "Remote visit addressed rate limitation.", respJSON["message"]) {
|
||||
switch respJSON["message"] {
|
||||
case "Remote visit addressed rate limitation.":
|
||||
t.Log("test hit github rate limitation")
|
||||
case "migrate from '10.0.0.1' is not allowed: the host resolve to a private ip address '10.0.0.1'":
|
||||
assert.EqualValues(t, "private-ip", testCase.repoName)
|
||||
case "migrate from 'localhost:3000' is not allowed: the host resolve to a private ip address '::1'",
|
||||
"migrate from 'localhost:3000' is not allowed: the host resolve to a private ip address '127.0.0.1'":
|
||||
assert.EqualValues(t, "local-ip", testCase.repoName)
|
||||
default:
|
||||
t.Errorf("unexpected error '%v' on url '%s'", respJSON["message"], testCase.cloneURL)
|
||||
}
|
||||
} else {
|
||||
assert.EqualValues(t, testCase.expectedStatus, resp.Code)
|
||||
|
@ -1019,6 +1019,29 @@ func IsErrWontSign(err error) bool {
|
||||
return ok
|
||||
}
|
||||
|
||||
// ErrMigrationNotAllowed explains why a migration from an url is not allowed
|
||||
type ErrMigrationNotAllowed struct {
|
||||
Host string
|
||||
NotResolvedIP bool
|
||||
PrivateNet string
|
||||
}
|
||||
|
||||
func (e *ErrMigrationNotAllowed) Error() string {
|
||||
if e.NotResolvedIP {
|
||||
return fmt.Sprintf("migrate from '%s' is not allowed: unknown hostname", e.Host)
|
||||
}
|
||||
if len(e.PrivateNet) != 0 {
|
||||
return fmt.Sprintf("migrate from '%s' is not allowed: the host resolve to a private ip address '%s'", e.Host, e.PrivateNet)
|
||||
}
|
||||
return fmt.Sprintf("migrate from '%s is not allowed'", e.Host)
|
||||
}
|
||||
|
||||
// IsErrMigrationNotAllowed checks if an error is a ErrMigrationNotAllowed
|
||||
func IsErrMigrationNotAllowed(err error) bool {
|
||||
_, ok := err.(*ErrMigrationNotAllowed)
|
||||
return ok
|
||||
}
|
||||
|
||||
// __________ .__
|
||||
// \______ \____________ ____ ____ | |__
|
||||
// | | _/\_ __ \__ \ / \_/ ___\| | \
|
||||
|
46
modules/matchlist/matchlist.go
Normal file
46
modules/matchlist/matchlist.go
Normal file
@ -0,0 +1,46 @@
|
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package matchlist
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/gobwas/glob"
|
||||
)
|
||||
|
||||
// Matchlist represents a block or allow list
|
||||
type Matchlist struct {
|
||||
ruleGlobs []glob.Glob
|
||||
}
|
||||
|
||||
// NewMatchlist creates a new block or allow list
|
||||
func NewMatchlist(rules ...string) (*Matchlist, error) {
|
||||
for i := range rules {
|
||||
rules[i] = strings.ToLower(rules[i])
|
||||
}
|
||||
list := Matchlist{
|
||||
ruleGlobs: make([]glob.Glob, 0, len(rules)),
|
||||
}
|
||||
|
||||
for _, rule := range rules {
|
||||
rg, err := glob.Compile(rule)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
list.ruleGlobs = append(list.ruleGlobs, rg)
|
||||
}
|
||||
|
||||
return &list, nil
|
||||
}
|
||||
|
||||
// Match will matches
|
||||
func (b *Matchlist) Match(u string) bool {
|
||||
for _, r := range b.ruleGlobs {
|
||||
if r.Match(u) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
@ -8,9 +8,13 @@ package migrations
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"code.gitea.io/gitea/models"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/matchlist"
|
||||
"code.gitea.io/gitea/modules/migrations/base"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
)
|
||||
@ -20,6 +24,9 @@ type MigrateOptions = base.MigrateOptions
|
||||
|
||||
var (
|
||||
factories []base.DownloaderFactory
|
||||
|
||||
allowList *matchlist.Matchlist
|
||||
blockList *matchlist.Matchlist
|
||||
)
|
||||
|
||||
// RegisterDownloaderFactory registers a downloader factory
|
||||
@ -27,12 +34,49 @@ func RegisterDownloaderFactory(factory base.DownloaderFactory) {
|
||||
factories = append(factories, factory)
|
||||
}
|
||||
|
||||
func isMigrateURLAllowed(remoteURL string) error {
|
||||
u, err := url.Parse(strings.ToLower(remoteURL))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https") {
|
||||
if len(setting.Migrations.AllowedDomains) > 0 {
|
||||
if !allowList.Match(u.Host) {
|
||||
return &models.ErrMigrationNotAllowed{Host: u.Host}
|
||||
}
|
||||
} else {
|
||||
if blockList.Match(u.Host) {
|
||||
return &models.ErrMigrationNotAllowed{Host: u.Host}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !setting.Migrations.AllowLocalNetworks {
|
||||
addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0])
|
||||
if err != nil {
|
||||
return &models.ErrMigrationNotAllowed{Host: u.Host, NotResolvedIP: true}
|
||||
}
|
||||
for _, addr := range addrList {
|
||||
if isIPPrivate(addr) || !addr.IsGlobalUnicast() {
|
||||
return &models.ErrMigrationNotAllowed{Host: u.Host, PrivateNet: addr.String()}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// MigrateRepository migrate repository according MigrateOptions
|
||||
func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
|
||||
err := isMigrateURLAllowed(opts.CloneAddr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var (
|
||||
downloader base.Downloader
|
||||
uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
|
||||
err error
|
||||
)
|
||||
|
||||
for _, factory := range factories {
|
||||
@ -308,3 +352,32 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Init migrations service
|
||||
func Init() error {
|
||||
var err error
|
||||
allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init migration allowList domains failed: %v", err)
|
||||
}
|
||||
|
||||
blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("init migration blockList domains failed: %v", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isIPPrivate reports whether ip is a private address, according to
|
||||
// RFC 1918 (IPv4 addresses) and RFC 4193 (IPv6 addresses).
|
||||
// from https://github.com/golang/go/pull/42793
|
||||
// TODO remove if https://github.com/golang/go/issues/29146 got resolved
|
||||
func isIPPrivate(ip net.IP) bool {
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
return ip4[0] == 10 ||
|
||||
(ip4[0] == 172 && ip4[1]&0xf0 == 16) ||
|
||||
(ip4[0] == 192 && ip4[1] == 168)
|
||||
}
|
||||
return len(ip) == net.IPv6len && ip[0]&0xfe == 0xfc
|
||||
}
|
||||
|
34
modules/migrations/migrate_test.go
Normal file
34
modules/migrations/migrate_test.go
Normal file
@ -0,0 +1,34 @@
|
||||
// Copyright 2019 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package migrations
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestMigrateWhiteBlocklist(t *testing.T) {
|
||||
setting.Migrations.AllowedDomains = []string{"github.com"}
|
||||
assert.NoError(t, Init())
|
||||
|
||||
err := isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git")
|
||||
assert.Error(t, err)
|
||||
|
||||
err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git")
|
||||
assert.NoError(t, err)
|
||||
|
||||
setting.Migrations.AllowedDomains = []string{}
|
||||
setting.Migrations.BlockedDomains = []string{"github.com"}
|
||||
assert.NoError(t, Init())
|
||||
|
||||
err = isMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git")
|
||||
assert.NoError(t, err)
|
||||
|
||||
err = isMigrateURLAllowed("https://github.com/go-gitea/gitea.git")
|
||||
assert.Error(t, err)
|
||||
}
|
@ -4,11 +4,18 @@
|
||||
|
||||
package setting
|
||||
|
||||
import (
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
// Migrations settings
|
||||
Migrations = struct {
|
||||
MaxAttempts int
|
||||
RetryBackoff int
|
||||
MaxAttempts int
|
||||
RetryBackoff int
|
||||
AllowedDomains []string
|
||||
BlockedDomains []string
|
||||
AllowLocalNetworks bool
|
||||
}{
|
||||
MaxAttempts: 3,
|
||||
RetryBackoff: 3,
|
||||
@ -19,4 +26,15 @@ func newMigrationsService() {
|
||||
sec := Cfg.Section("migrations")
|
||||
Migrations.MaxAttempts = sec.Key("MAX_ATTEMPTS").MustInt(Migrations.MaxAttempts)
|
||||
Migrations.RetryBackoff = sec.Key("RETRY_BACKOFF").MustInt(Migrations.RetryBackoff)
|
||||
|
||||
Migrations.AllowedDomains = sec.Key("ALLOWED_DOMAINS").Strings(",")
|
||||
for i := range Migrations.AllowedDomains {
|
||||
Migrations.AllowedDomains[i] = strings.ToLower(Migrations.AllowedDomains[i])
|
||||
}
|
||||
Migrations.BlockedDomains = sec.Key("BLOCKED_DOMAINS").Strings(",")
|
||||
for i := range Migrations.BlockedDomains {
|
||||
Migrations.BlockedDomains[i] = strings.ToLower(Migrations.BlockedDomains[i])
|
||||
}
|
||||
|
||||
Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)
|
||||
}
|
||||
|
@ -212,6 +212,8 @@ func handleMigrateError(ctx *context.APIContext, repoOwner *models.User, remoteA
|
||||
ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The username '%s' contains invalid characters.", err.(models.ErrNameCharsNotAllowed).Name))
|
||||
case models.IsErrNamePatternNotAllowed(err):
|
||||
ctx.Error(http.StatusUnprocessableEntity, "", fmt.Sprintf("The pattern '%s' is not allowed in a username.", err.(models.ErrNamePatternNotAllowed).Pattern))
|
||||
case models.IsErrMigrationNotAllowed(err):
|
||||
ctx.Error(http.StatusUnprocessableEntity, "", err)
|
||||
default:
|
||||
err = util.URLSanitizedError(err, remoteAddr)
|
||||
if strings.Contains(err.Error(), "Authentication failed") ||
|
||||
|
@ -24,6 +24,7 @@ import (
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
"code.gitea.io/gitea/modules/markup"
|
||||
"code.gitea.io/gitea/modules/markup/external"
|
||||
repo_migrations "code.gitea.io/gitea/modules/migrations"
|
||||
"code.gitea.io/gitea/modules/notification"
|
||||
"code.gitea.io/gitea/modules/options"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
@ -201,6 +202,9 @@ func GlobalInit(ctx context.Context) {
|
||||
if err := task.Init(); err != nil {
|
||||
log.Fatal("Failed to initialize task scheduler: %v", err)
|
||||
}
|
||||
if err := repo_migrations.Init(); err != nil {
|
||||
log.Fatal("Failed to initialize repository migrations: %v", err)
|
||||
}
|
||||
eventsource.GetManager().Init()
|
||||
|
||||
if setting.EnableSQLite3 {
|
||||
|
Loading…
Reference in New Issue
Block a user