Update migrated repositories' issues/comments/prs poster id if user has a github external user saved (#7751)

* update migrated issues/comments when login as github

* add get userid when migrating or login with github oauth2

* fix lint

* add migrations for repository service type

* fix build

* remove unnecessary dependencies on migrations

* add cron task to update migrations poster ids and fix posterid when migrating

* fix lint

* fix lint

* improve code

* fix lint

* improve code

* replace releases publish id to actual author id

* fix import

* fix bug

* fix lint

* fix rawdata definition

* fix some bugs

* fix error message

custom/conf/app.ini.sample

@@ -690,6 +690,11 @@ SCHEDULE = @every 24h
 ;   or only create new users if UPDATE_EXISTING is set to false
 UPDATE_EXISTING = true
 
+; Update migrated repositories' issues and comments' posterid, it will always attempt synchronization when the instance starts.
+[cron.update_migration_post_id]
+; Interval as a duration between each synchronization. (default every 24h)
+SCHEDULE = @every 24h
+
 [git]
 ; The path of git executable. If empty, Gitea searches through the PATH environment.
 PATH =

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -419,6 +419,10 @@ NB: You must `REDIRECT_MACARON_LOG` and have `DISABLE_ROUTER_LOG` set to `false`
 - `RUN_AT_START`: **true**: Run repository statistics check at start time.
 - `SCHEDULE`: **@every 24h**: Cron syntax for scheduling repository statistics check.
 
+### Cron - Update Migration Poster ID (`cron.update_migration_post_id`)
+
+- `SCHEDULE`: **@every 24h** : Interval as a duration between each synchronization, it will always attempt synchronization when the instance starts.
+
 ## Git (`git`)
 
 - `PATH`: **""**: The path of git executable. If empty, Gitea searches through the PATH environment.

docs/content/doc/advanced/config-cheat-sheet.zh-cn.md

@@ -196,7 +196,11 @@ menu:
 ### Cron - Repository Statistics Check (`cron.check_repo_stats`)
 
 - `RUN_AT_START`: 是否启动时自动运行仓库统计。
-- `SCHEDULE`: 藏亏统计时的Cron 语法，比如：`@every 24h`.
+- `SCHEDULE`: 仓库统计时的Cron 语法，比如：`@every 24h`.
+
+### Cron - Update Migration Poster ID (`cron.update_migration_post_id`)
+
+- `SCHEDULE`: **@every 24h** : 每次同步的间隔时间。此任务总是在启动时自动进行。
 
 ## Git (`git`)
 

models/external_login_user.go

@@ -4,13 +4,34 @@
 
 package models
 
-import "github.com/markbates/goth"
+import (
+	"time"
+
+	"code.gitea.io/gitea/modules/structs"
+
+	"github.com/markbates/goth"
+	"xorm.io/builder"
+)
 
 // ExternalLoginUser makes the connecting between some existing user and additional external login sources
 type ExternalLoginUser struct {
-	ExternalID    string `xorm:"pk NOT NULL"`
+	ExternalID        string                 `xorm:"pk NOT NULL"`
-	UserID        int64  `xorm:"INDEX NOT NULL"`
+	UserID            int64                  `xorm:"INDEX NOT NULL"`
-	LoginSourceID int64  `xorm:"pk NOT NULL"`
+	LoginSourceID     int64                  `xorm:"pk NOT NULL"`
+	RawData           map[string]interface{} `xorm:"TEXT JSON"`
+	Provider          string                 `xorm:"index VARCHAR(25)"`
+	Email             string
+	Name              string
+	FirstName         string
+	LastName          string
+	NickName          string
+	Description       string
+	AvatarURL         string
+	Location          string
+	AccessToken       string
+	AccessTokenSecret string
+	RefreshToken      string
+	ExpiresAt         time.Time
 }
 
 // GetExternalLogin checks if a externalID in loginSourceID scope already exists
@@ -32,23 +53,15 @@ func ListAccountLinks(user *User) ([]*ExternalLoginUser, error) {
 	return externalAccounts, nil
 }
 
-// LinkAccountToUser link the gothUser to the user
+// LinkExternalToUser link the external user to the user
-func LinkAccountToUser(user *User, gothUser goth.User) error {
+func LinkExternalToUser(user *User, externalLoginUser *ExternalLoginUser) error {
-	loginSource, err := GetActiveOAuth2LoginSourceByName(gothUser.Provider)
+	has, err := x.Where("external_id=? AND login_source_id=?", externalLoginUser.ExternalID, externalLoginUser.LoginSourceID).
-	if err != nil {
+		NoAutoCondition().
-		return err
+		Exist(externalLoginUser)
-	}
-
-	externalLoginUser := &ExternalLoginUser{
-		ExternalID:    gothUser.UserID,
-		UserID:        user.ID,
-		LoginSourceID: loginSource.ID,
-	}
-	has, err := x.Get(externalLoginUser)
 	if err != nil {
 		return err
 	} else if has {
-		return ErrExternalLoginUserAlreadyExist{gothUser.UserID, user.ID, loginSource.ID}
+		return ErrExternalLoginUserAlreadyExist{externalLoginUser.ExternalID, user.ID, externalLoginUser.LoginSourceID}
 	}
 
 	_, err = x.Insert(externalLoginUser)
@@ -72,3 +85,97 @@ func removeAllAccountLinks(e Engine, user *User) error {
 	_, err := e.Delete(&ExternalLoginUser{UserID: user.ID})
 	return err
 }
+
+// GetUserIDByExternalUserID get user id according to provider and userID
+func GetUserIDByExternalUserID(provider string, userID string) (int64, error) {
+	var id int64
+	_, err := x.Table("external_login_user").
+		Select("user_id").
+		Where("provider=?", provider).
+		And("external_id=?", userID).
+		Get(&id)
+	if err != nil {
+		return 0, err
+	}
+	return id, nil
+}
+
+// UpdateExternalUser updates external user's information
+func UpdateExternalUser(user *User, gothUser goth.User) error {
+	loginSource, err := GetActiveOAuth2LoginSourceByName(gothUser.Provider)
+	if err != nil {
+		return err
+	}
+	externalLoginUser := &ExternalLoginUser{
+		ExternalID:        gothUser.UserID,
+		UserID:            user.ID,
+		LoginSourceID:     loginSource.ID,
+		RawData:           gothUser.RawData,
+		Provider:          gothUser.Provider,
+		Email:             gothUser.Email,
+		Name:              gothUser.Name,
+		FirstName:         gothUser.FirstName,
+		LastName:          gothUser.LastName,
+		NickName:          gothUser.NickName,
+		Description:       gothUser.Description,
+		AvatarURL:         gothUser.AvatarURL,
+		Location:          gothUser.Location,
+		AccessToken:       gothUser.AccessToken,
+		AccessTokenSecret: gothUser.AccessTokenSecret,
+		RefreshToken:      gothUser.RefreshToken,
+		ExpiresAt:         gothUser.ExpiresAt,
+	}
+
+	has, err := x.Where("external_id=? AND login_source_id=?", gothUser.UserID, loginSource.ID).
+		NoAutoCondition().
+		Exist(externalLoginUser)
+	if err != nil {
+		return err
+	} else if !has {
+		return ErrExternalLoginUserNotExist{user.ID, loginSource.ID}
+	}
+
+	_, err = x.Where("external_id=? AND login_source_id=?", gothUser.UserID, loginSource.ID).AllCols().Update(externalLoginUser)
+	return err
+}
+
+// FindExternalUserOptions represents an options to find external users
+type FindExternalUserOptions struct {
+	Provider string
+	Limit    int
+	Start    int
+}
+
+func (opts FindExternalUserOptions) toConds() builder.Cond {
+	var cond = builder.NewCond()
+	if len(opts.Provider) > 0 {
+		cond = cond.And(builder.Eq{"provider": opts.Provider})
+	}
+	return cond
+}
+
+// FindExternalUsersByProvider represents external users via provider
+func FindExternalUsersByProvider(opts FindExternalUserOptions) ([]ExternalLoginUser, error) {
+	var users []ExternalLoginUser
+	err := x.Where(opts.toConds()).
+		Limit(opts.Limit, opts.Start).
+		Asc("id").
+		Find(&users)
+	if err != nil {
+		return nil, err
+	}
+	return users, nil
+}
+
+// UpdateMigrationsByType updates all migrated repositories' posterid from gitServiceType to replace originalAuthorID to posterID
+func UpdateMigrationsByType(tp structs.GitServiceType, externalUserID, userID int64) error {
+	if err := UpdateIssuesMigrationsByType(tp, externalUserID, userID); err != nil {
+		return err
+	}
+
+	if err := UpdateCommentsMigrationsByType(tp, externalUserID, userID); err != nil {
+		return err
+	}
+
+	return UpdateReleasesMigrationsByType(tp, externalUserID, userID)
+}

models/issue.go

@@ -14,6 +14,7 @@ import (
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/structs"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/util"
@@ -32,7 +33,7 @@ type Issue struct {
 	PosterID         int64       `xorm:"INDEX"`
 	Poster           *User       `xorm:"-"`
 	OriginalAuthor   string
-	OriginalAuthorID int64
+	OriginalAuthorID int64      `xorm:"index"`
 	Title            string     `xorm:"name"`
 	Content          string     `xorm:"TEXT"`
 	RenderedContent  string     `xorm:"-"`
@@ -1947,3 +1948,16 @@ func (issue *Issue) ResolveMentionsByVisibility(ctx DBContext, doer *User, menti
 
 	return
 }
+
+// UpdateIssuesMigrationsByType updates all migrated repositories' issues from gitServiceType to replace originalAuthorID to posterID
+func UpdateIssuesMigrationsByType(gitServiceType structs.GitServiceType, originalAuthorID, posterID int64) error {
+	_, err := x.Table("issue").
+		Where("repo_id IN (SELECT id FROM repository WHERE original_service_type = ?)", gitServiceType).
+		And("original_author_id = ?", originalAuthorID).
+		Update(map[string]interface{}{
+			"poster_id":          posterID,
+			"original_author":    "",
+			"original_author_id": 0,
+		})
+	return err
+}

models/issue_comment.go

@@ -14,6 +14,7 @@ import (
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup/markdown"
 	"code.gitea.io/gitea/modules/references"
+	"code.gitea.io/gitea/modules/structs"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 
@@ -1022,3 +1023,23 @@ func fetchCodeCommentsByReview(e Engine, issue *Issue, currentUser *User, review
 func FetchCodeComments(issue *Issue, currentUser *User) (CodeComments, error) {
 	return fetchCodeComments(x, issue, currentUser)
 }
+
+// UpdateCommentsMigrationsByType updates comments' migrations information via given git service type and original id and poster id
+func UpdateCommentsMigrationsByType(tp structs.GitServiceType, originalAuthorID, posterID int64) error {
+	_, err := x.Table("comment").
+		Where(builder.In("issue_id",
+			builder.Select("issue.id").
+				From("issue").
+				InnerJoin("repository", "issue.repo_id = repository.id").
+				Where(builder.Eq{
+					"repository.original_service_type": tp,
+				}),
+		)).
+		And("comment.original_author_id = ?", originalAuthorID).
+		Update(map[string]interface{}{
+			"poster_id":          posterID,
+			"original_author":    "",
+			"original_author_id": 0,
+		})
+	return err
+}

models/migrations/migrations.go

@@ -254,6 +254,8 @@ var migrations = []Migration{
 	NewMigration("add original author name and id on migrated release", addOriginalAuthorOnMigratedReleases),
 	// v99 -> v100
 	NewMigration("add task table and status column for repository table", addTaskTable),
+	// v100 -> v101
+	NewMigration("update migration repositories' service type", updateMigrationServiceTypes),
 }
 
 // Migrate database to current version

models/migrations/v100.go

@@ -0,0 +1,83 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/go-xorm/xorm"
+)
+
+func updateMigrationServiceTypes(x *xorm.Engine) error {
+	type Repository struct {
+		ID                  int64
+		OriginalServiceType int    `xorm:"index default(0)"`
+		OriginalURL         string `xorm:"VARCHAR(2048)"`
+	}
+
+	if err := x.Sync2(new(Repository)); err != nil {
+		return err
+	}
+
+	var last int
+	const batchSize = 50
+	for {
+		var results = make([]Repository, 0, batchSize)
+		err := x.Where("original_url <> '' AND original_url IS NOT NULL").
+			And("original_service_type = 0 OR original_service_type IS NULL").
+			OrderBy("id").
+			Limit(batchSize, last).
+			Find(&results)
+		if err != nil {
+			return err
+		}
+		if len(results) == 0 {
+			break
+		}
+		last += len(results)
+
+		const PlainGitService = 1 // 1 plain git service
+		const GithubService = 2   // 2 github.com
+
+		for _, res := range results {
+			u, err := url.Parse(res.OriginalURL)
+			if err != nil {
+				return err
+			}
+			var serviceType = PlainGitService
+			if strings.EqualFold(u.Host, "github.com") {
+				serviceType = GithubService
+			}
+			_, err = x.Exec("UPDATE repository SET original_service_type = ? WHERE id = ?", serviceType, res.ID)
+			if err != nil {
+				return err
+			}
+		}
+	}
+
+	type ExternalLoginUser struct {
+		ExternalID        string                 `xorm:"pk NOT NULL"`
+		UserID            int64                  `xorm:"INDEX NOT NULL"`
+		LoginSourceID     int64                  `xorm:"pk NOT NULL"`
+		RawData           map[string]interface{} `xorm:"TEXT JSON"`
+		Provider          string                 `xorm:"index VARCHAR(25)"`
+		Email             string
+		Name              string
+		FirstName         string
+		LastName          string
+		NickName          string
+		Description       string
+		AvatarURL         string
+		Location          string
+		AccessToken       string
+		AccessTokenSecret string
+		RefreshToken      string
+		ExpiresAt         time.Time
+	}
+
+	return x.Sync2(new(ExternalLoginUser))
+}

models/release.go

@@ -12,6 +12,7 @@ import (
 
 	"code.gitea.io/gitea/modules/git"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/structs"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/timeutil"
 
@@ -366,3 +367,16 @@ func SyncReleasesWithTags(repo *Repository, gitRepo *git.Repository) error {
 	}
 	return nil
 }
+
+// UpdateReleasesMigrationsByType updates all migrated repositories' releases from gitServiceType to replace originalAuthorID to posterID
+func UpdateReleasesMigrationsByType(gitServiceType structs.GitServiceType, originalAuthorID, posterID int64) error {
+	_, err := x.Table("release").
+		Where("repo_id IN (SELECT id FROM repository WHERE original_service_type = ?)", gitServiceType).
+		And("original_author_id = ?", originalAuthorID).
+		Update(map[string]interface{}{
+			"publisher_id":       posterID,
+			"original_author":    "",
+			"original_author_id": 0,
+		})
+	return err
+}

models/repo.go

@@ -32,6 +32,7 @@ import (
 	"code.gitea.io/gitea/modules/options"
 	"code.gitea.io/gitea/modules/process"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/structs"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/sync"
 	"code.gitea.io/gitea/modules/timeutil"
@@ -137,16 +138,17 @@ const (
 
 // Repository represents a git repository.
 type Repository struct {
-	ID            int64  `xorm:"pk autoincr"`
+	ID                  int64                  `xorm:"pk autoincr"`
-	OwnerID       int64  `xorm:"UNIQUE(s) index"`
+	OwnerID             int64                  `xorm:"UNIQUE(s) index"`
-	OwnerName     string `xorm:"-"`
+	OwnerName           string                 `xorm:"-"`
-	Owner         *User  `xorm:"-"`
+	Owner               *User                  `xorm:"-"`
-	LowerName     string `xorm:"UNIQUE(s) INDEX NOT NULL"`
+	LowerName           string                 `xorm:"UNIQUE(s) INDEX NOT NULL"`
-	Name          string `xorm:"INDEX NOT NULL"`
+	Name                string                 `xorm:"INDEX NOT NULL"`
-	Description   string `xorm:"TEXT"`
+	Description         string                 `xorm:"TEXT"`
-	Website       string `xorm:"VARCHAR(2048)"`
+	Website             string                 `xorm:"VARCHAR(2048)"`
-	OriginalURL   string `xorm:"VARCHAR(2048)"`
+	OriginalServiceType structs.GitServiceType `xorm:"index"`
-	DefaultBranch string
+	OriginalURL         string                 `xorm:"VARCHAR(2048)"`
+	DefaultBranch       string
 
 	NumWatches          int
 	NumStars            int

modules/cron/cron.go

@@ -10,6 +10,7 @@ import (
 
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/migrations"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/sync"
 	mirror_service "code.gitea.io/gitea/services/mirror"
@@ -18,12 +19,13 @@ import (
 )
 
 const (
-	mirrorUpdate           = "mirror_update"
+	mirrorUpdate            = "mirror_update"
-	gitFsck                = "git_fsck"
+	gitFsck                 = "git_fsck"
-	checkRepos             = "check_repos"
+	checkRepos              = "check_repos"
-	archiveCleanup         = "archive_cleanup"
+	archiveCleanup          = "archive_cleanup"
-	syncExternalUsers      = "sync_external_users"
+	syncExternalUsers       = "sync_external_users"
-	deletedBranchesCleanup = "deleted_branches_cleanup"
+	deletedBranchesCleanup  = "deleted_branches_cleanup"
+	updateMigrationPosterID = "update_migration_post_id"
 )
 
 var c = cron.New()
@@ -117,6 +119,15 @@ func NewContext() {
 			go WithUnique(deletedBranchesCleanup, models.RemoveOldDeletedBranches)()
 		}
 	}
+
+	entry, err = c.AddFunc("Update migrated repositories' issues and comments' posterid", setting.Cron.UpdateMigrationPosterID.Schedule, WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID))
+	if err != nil {
+		log.Fatal("Cron[Update migrated repositories]: %v", err)
+	}
+	entry.Prev = time.Now()
+	entry.ExecTimes++
+	go WithUnique(updateMigrationPosterID, migrations.UpdateMigrationPosterID)()
+
 	c.Start()
 }
 

modules/migrations/base/downloader.go

@@ -5,6 +5,8 @@
 
 package base
 
+import "code.gitea.io/gitea/modules/structs"
+
 // Downloader downloads the site repo informations
 type Downloader interface {
 	GetRepoInfo() (*Repository, error)
@@ -21,4 +23,5 @@ type Downloader interface {
 type DownloaderFactory interface {
 	Match(opts MigrateOptions) (bool, error)
 	New(opts MigrateOptions) (Downloader, error)
+	GitServiceType() structs.GitServiceType
 }

modules/migrations/gitea.go

@@ -34,15 +34,17 @@ var (
 
 // GiteaLocalUploader implements an Uploader to gitea sites
 type GiteaLocalUploader struct {
-	doer        *models.User
+	doer           *models.User
-	repoOwner   string
+	repoOwner      string
-	repoName    string
+	repoName       string
-	repo        *models.Repository
+	repo           *models.Repository
-	labels      sync.Map
+	labels         sync.Map
-	milestones  sync.Map
+	milestones     sync.Map
-	issues      sync.Map
+	issues         sync.Map
-	gitRepo     *git.Repository
+	gitRepo        *git.Repository
-	prHeadCache map[string]struct{}
+	prHeadCache    map[string]struct{}
+	userMap        map[int64]int64 // external user id mapping to user id
+	gitServiceType structs.GitServiceType
 }
 
 // NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
@@ -52,6 +54,7 @@ func NewGiteaLocalUploader(doer *models.User, repoOwner, repoName string) *Gitea
 		repoOwner:   repoOwner,
 		repoName:    repoName,
 		prHeadCache: make(map[string]struct{}),
+		userMap:     make(map[int64]int64),
 	}
 }
 
@@ -109,13 +112,15 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
 	}
 
 	r, err = models.MigrateRepositoryGitData(g.doer, owner, r, structs.MigrateRepoOption{
-		RepoName:    g.repoName,
+		RepoName:       g.repoName,
-		Description: repo.Description,
+		Description:    repo.Description,
-		Mirror:      repo.IsMirror,
+		OriginalURL:    repo.OriginalURL,
-		CloneAddr:   remoteAddr,
+		GitServiceType: opts.GitServiceType,
-		Private:     repo.IsPrivate,
+		Mirror:         repo.IsMirror,
-		Wiki:        opts.Wiki,
+		CloneAddr:      remoteAddr,
-		Releases:    opts.Releases, // if didn't get releases, then sync them from tags
+		Private:        repo.IsPrivate,
+		Wiki:           opts.Wiki,
+		Releases:       opts.Releases, // if didn't get releases, then sync them from tags
 	})
 
 	g.repo = r
@@ -193,20 +198,38 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
 	var rels = make([]*models.Release, 0, len(releases))
 	for _, release := range releases {
 		var rel = models.Release{
-			RepoID:           g.repo.ID,
+			RepoID:       g.repo.ID,
-			PublisherID:      g.doer.ID,
+			TagName:      release.TagName,
-			TagName:          release.TagName,
+			LowerTagName: strings.ToLower(release.TagName),
-			LowerTagName:     strings.ToLower(release.TagName),
+			Target:       release.TargetCommitish,
-			Target:           release.TargetCommitish,
+			Title:        release.Name,
-			Title:            release.Name,
+			Sha1:         release.TargetCommitish,
-			Sha1:             release.TargetCommitish,
+			Note:         release.Body,
-			Note:             release.Body,
+			IsDraft:      release.Draft,
-			IsDraft:          release.Draft,
+			IsPrerelease: release.Prerelease,
-			IsPrerelease:     release.Prerelease,
+			IsTag:        false,
-			IsTag:            false,
+			CreatedUnix:  timeutil.TimeStamp(release.Created.Unix()),
-			CreatedUnix:      timeutil.TimeStamp(release.Created.Unix()),
+		}
-			OriginalAuthor:   release.PublisherName,
+
-			OriginalAuthorID: release.PublisherID,
+		userid, ok := g.userMap[release.PublisherID]
+		tp := g.gitServiceType.Name()
+		if !ok && tp != "" {
+			var err error
+			userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", release.PublisherID))
+			if err != nil {
+				log.Error("GetUserIDByExternalUserID: %v", err)
+			}
+			if userid > 0 {
+				g.userMap[release.PublisherID] = userid
+			}
+		}
+
+		if userid > 0 {
+			rel.PublisherID = userid
+		} else {
+			rel.PublisherID = g.doer.ID
+			rel.OriginalAuthor = release.PublisherName
+			rel.OriginalAuthorID = release.PublisherID
 		}
 
 		// calc NumCommits
@@ -284,20 +307,39 @@ func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
 		}
 
 		var is = models.Issue{
-			RepoID:           g.repo.ID,
+			RepoID:      g.repo.ID,
-			Repo:             g.repo,
+			Repo:        g.repo,
-			Index:            issue.Number,
+			Index:       issue.Number,
-			PosterID:         g.doer.ID,
+			Title:       issue.Title,
-			OriginalAuthor:   issue.PosterName,
+			Content:     issue.Content,
-			OriginalAuthorID: issue.PosterID,
+			IsClosed:    issue.State == "closed",
-			Title:            issue.Title,
+			IsLocked:    issue.IsLocked,
-			Content:          issue.Content,
+			MilestoneID: milestoneID,
-			IsClosed:         issue.State == "closed",
+			Labels:      labels,
-			IsLocked:         issue.IsLocked,
+			CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()),
-			MilestoneID:      milestoneID,
-			Labels:           labels,
-			CreatedUnix:      timeutil.TimeStamp(issue.Created.Unix()),
 		}
+
+		userid, ok := g.userMap[issue.PosterID]
+		tp := g.gitServiceType.Name()
+		if !ok && tp != "" {
+			var err error
+			userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", issue.PosterID))
+			if err != nil {
+				log.Error("GetUserIDByExternalUserID: %v", err)
+			}
+			if userid > 0 {
+				g.userMap[issue.PosterID] = userid
+			}
+		}
+
+		if userid > 0 {
+			is.PosterID = userid
+		} else {
+			is.PosterID = g.doer.ID
+			is.OriginalAuthor = issue.PosterName
+			is.OriginalAuthorID = issue.PosterID
+		}
+
 		if issue.Closed != nil {
 			is.ClosedUnix = timeutil.TimeStamp(issue.Closed.Unix())
 		}
@@ -331,15 +373,35 @@ func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
 			issueID = issueIDStr.(int64)
 		}
 
-		cms = append(cms, &models.Comment{
+		userid, ok := g.userMap[comment.PosterID]
-			IssueID:          issueID,
+		tp := g.gitServiceType.Name()
-			Type:             models.CommentTypeComment,
+		if !ok && tp != "" {
-			PosterID:         g.doer.ID,
+			var err error
-			OriginalAuthor:   comment.PosterName,
+			userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", comment.PosterID))
-			OriginalAuthorID: comment.PosterID,
+			if err != nil {
-			Content:          comment.Content,
+				log.Error("GetUserIDByExternalUserID: %v", err)
-			CreatedUnix:      timeutil.TimeStamp(comment.Created.Unix()),
+			}
-		})
+			if userid > 0 {
+				g.userMap[comment.PosterID] = userid
+			}
+		}
+
+		cm := models.Comment{
+			IssueID:     issueID,
+			Type:        models.CommentTypeComment,
+			Content:     comment.Content,
+			CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()),
+		}
+
+		if userid > 0 {
+			cm.PosterID = userid
+		} else {
+			cm.PosterID = g.doer.ID
+			cm.OriginalAuthor = comment.PosterName
+			cm.OriginalAuthorID = comment.PosterID
+		}
+
+		cms = append(cms, &cm)
 
 		// TODO: Reactions
 	}
@@ -355,6 +417,28 @@ func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error
 		if err != nil {
 			return err
 		}
+
+		userid, ok := g.userMap[pr.PosterID]
+		tp := g.gitServiceType.Name()
+		if !ok && tp != "" {
+			var err error
+			userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
+			if err != nil {
+				log.Error("GetUserIDByExternalUserID: %v", err)
+			}
+			if userid > 0 {
+				g.userMap[pr.PosterID] = userid
+			}
+		}
+
+		if userid > 0 {
+			gpr.Issue.PosterID = userid
+		} else {
+			gpr.Issue.PosterID = g.doer.ID
+			gpr.Issue.OriginalAuthor = pr.PosterName
+			gpr.Issue.OriginalAuthorID = pr.PosterID
+		}
+
 		gprs = append(gprs, gpr)
 	}
 	if err := models.InsertPullRequests(gprs...); err != nil {
@@ -460,6 +544,40 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 		head = pr.Head.Ref
 	}
 
+	var issue = models.Issue{
+		RepoID:      g.repo.ID,
+		Repo:        g.repo,
+		Title:       pr.Title,
+		Index:       pr.Number,
+		Content:     pr.Content,
+		MilestoneID: milestoneID,
+		IsPull:      true,
+		IsClosed:    pr.State == "closed",
+		IsLocked:    pr.IsLocked,
+		Labels:      labels,
+		CreatedUnix: timeutil.TimeStamp(pr.Created.Unix()),
+	}
+
+	userid, ok := g.userMap[pr.PosterID]
+	if !ok {
+		var err error
+		userid, err = models.GetUserIDByExternalUserID("github", fmt.Sprintf("%v", pr.PosterID))
+		if err != nil {
+			log.Error("GetUserIDByExternalUserID: %v", err)
+		}
+		if userid > 0 {
+			g.userMap[pr.PosterID] = userid
+		}
+	}
+
+	if userid > 0 {
+		issue.PosterID = userid
+	} else {
+		issue.PosterID = g.doer.ID
+		issue.OriginalAuthor = pr.PosterName
+		issue.OriginalAuthorID = pr.PosterID
+	}
+
 	var pullRequest = models.PullRequest{
 		HeadRepoID:   g.repo.ID,
 		HeadBranch:   head,
@@ -470,22 +588,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
 		Index:        pr.Number,
 		HasMerged:    pr.Merged,
 
-		Issue: &models.Issue{
+		Issue: &issue,
-			RepoID:           g.repo.ID,
-			Repo:             g.repo,
-			Title:            pr.Title,
-			Index:            pr.Number,
-			PosterID:         g.doer.ID,
-			OriginalAuthor:   pr.PosterName,
-			OriginalAuthorID: pr.PosterID,
-			Content:          pr.Content,
-			MilestoneID:      milestoneID,
-			IsPull:           true,
-			IsClosed:         pr.State == "closed",
-			IsLocked:         pr.IsLocked,
-			Labels:           labels,
-			CreatedUnix:      timeutil.TimeStamp(pr.Created.Unix()),
-		},
 	}
 
 	if pullRequest.Issue.IsClosed && pr.Closed != nil {

modules/migrations/github.go

@@ -14,6 +14,7 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/structs"
 
 	"github.com/google/go-github/v24/github"
 	"golang.org/x/oauth2"
@@ -39,7 +40,7 @@ func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error
 		return false, err
 	}
 
-	return u.Host == "github.com" && opts.AuthUsername != "", nil
+	return strings.EqualFold(u.Host, "github.com") && opts.AuthUsername != "", nil
 }
 
 // New returns a Downloader related to this factory according MigrateOptions
@@ -58,6 +59,11 @@ func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Download
 	return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil
 }
 
+// GitServiceType returns the type of git service
+func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
+	return structs.GithubService
+}
+
 // GithubDownloaderV3 implements a Downloader interface to get repository informations
 // from github via APIv3
 type GithubDownloaderV3 struct {

modules/migrations/migrate.go

@@ -11,6 +11,7 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/migrations/base"
+	"code.gitea.io/gitea/modules/structs"
 )
 
 // MigrateOptions is equal to base.MigrateOptions
@@ -30,6 +31,7 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
 	var (
 		downloader base.Downloader
 		uploader   = NewGiteaLocalUploader(doer, ownerName, opts.RepoName)
+		theFactory base.DownloaderFactory
 	)
 
 	for _, factory := range factories {
@@ -40,6 +42,7 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
 			if err != nil {
 				return nil, err
 			}
+			theFactory = factory
 			break
 		}
 	}
@@ -52,10 +55,14 @@ func MigrateRepository(doer *models.User, ownerName string, opts base.MigrateOpt
 		opts.Comments = false
 		opts.Issues = false
 		opts.PullRequests = false
+		opts.GitServiceType = structs.PlainGitService
 		downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr)
 		log.Trace("Will migrate from git: %s", opts.CloneAddr)
+	} else if opts.GitServiceType == structs.NotMigrated {
+		opts.GitServiceType = theFactory.GitServiceType()
 	}
 
+	uploader.gitServiceType = opts.GitServiceType
 	if err := migrateRepository(downloader, uploader, opts); err != nil {
 		if err1 := uploader.Rollback(); err1 != nil {
 			log.Error("rollback failed: %v", err1)

modules/migrations/update.go

@@ -0,0 +1,59 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"strconv"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/structs"
+)
+
+// UpdateMigrationPosterID updates all migrated repositories' issues and comments posterID
+func UpdateMigrationPosterID() {
+	for _, gitService := range structs.SupportedFullGitService {
+		if err := updateMigrationPosterIDByGitService(gitService); err != nil {
+			log.Error("updateMigrationPosterIDByGitService failed: %v", err)
+		}
+	}
+}
+
+func updateMigrationPosterIDByGitService(tp structs.GitServiceType) error {
+	provider := tp.Name()
+	if len(provider) == 0 {
+		return nil
+	}
+
+	const batchSize = 100
+	var start int
+	for {
+		users, err := models.FindExternalUsersByProvider(models.FindExternalUserOptions{
+			Provider: provider,
+			Start:    start,
+			Limit:    batchSize,
+		})
+		if err != nil {
+			return err
+		}
+
+		for _, user := range users {
+			externalUserID, err := strconv.ParseInt(user.ExternalID, 10, 64)
+			if err != nil {
+				log.Warn("Parse externalUser %#v 's userID failed: %v", user, err)
+				continue
+			}
+			if err := models.UpdateMigrationsByType(tp, externalUserID, user.UserID); err != nil {
+				log.Error("UpdateMigrationsByType type %s external user id %v to local user id %v failed: %v", tp.Name(), user.ExternalID, user.UserID, err)
+			}
+		}
+
+		if len(users) < batchSize {
+			break
+		}
+		start += len(users)
+	}
+	return nil
+}

modules/setting/cron.go

@@ -49,6 +49,9 @@ var (
 			Schedule   string
 			OlderThan  time.Duration
 		} `ini:"cron.deleted_branches_cleanup"`
+		UpdateMigrationPosterID struct {
+			Schedule string
+		} `ini:"cron.update_migration_poster_id"`
 	}{
 		UpdateMirror: struct {
 			Enabled    bool
@@ -114,6 +117,11 @@ var (
 			Schedule:   "@every 24h",
 			OlderThan:  24 * time.Hour,
 		},
+		UpdateMigrationPosterID: struct {
+			Schedule string
+		}{
+			Schedule: "@every 24h",
+		},
 	}
 )
 

modules/structs/repo.go

@@ -153,6 +153,43 @@ type EditRepoOption struct {
 	Archived *bool `json:"archived,omitempty"`
 }
 
+// GitServiceType represents a git service
+type GitServiceType int
+
+// enumerate all GitServiceType
+const (
+	NotMigrated     GitServiceType = iota // 0 not migrated from external sites
+	PlainGitService                       // 1 plain git service
+	GithubService                         // 2 github.com
+	GiteaService                          // 3 gitea service
+	GitlabService                         // 4 gitlab service
+	GogsService                           // 5 gogs service
+)
+
+// Name represents the service type's name
+// WARNNING: the name have to be equal to that on goth's library
+func (gt GitServiceType) Name() string {
+	switch gt {
+	case GithubService:
+		return "github"
+	case GiteaService:
+		return "gitea"
+	case GitlabService:
+		return "gitlab"
+	case GogsService:
+		return "gogs"
+	}
+	return ""
+}
+
+var (
+	// SupportedFullGitService represents all git services supported to migrate issues/labels/prs and etc.
+	// TODO: add to this list after new git service added
+	SupportedFullGitService = []GitServiceType{
+		GithubService,
+	}
+)
+
 // MigrateRepoOption options for migrating a repository from an external service
 type MigrateRepoOption struct {
 	// required: true
@@ -166,6 +203,8 @@ type MigrateRepoOption struct {
 	Mirror          bool   `json:"mirror"`
 	Private         bool   `json:"private"`
 	Description     string `json:"description"`
+	OriginalURL     string
+	GitServiceType  GitServiceType
 	Wiki            bool
 	Issues          bool
 	Milestones      bool

routers/api/v1/repo/repo.go

@@ -8,6 +8,7 @@ package repo
 import (
 	"fmt"
 	"net/http"
+	"net/url"
 	"strings"
 
 	"code.gitea.io/gitea/models"
@@ -17,6 +18,7 @@ import (
 	"code.gitea.io/gitea/modules/migrations"
 	"code.gitea.io/gitea/modules/notification"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/structs"
 	api "code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/modules/validation"
@@ -397,21 +399,28 @@ func Migrate(ctx *context.APIContext, form auth.MigrateRepoForm) {
 		return
 	}
 
+	var gitServiceType = structs.PlainGitService
+	u, err := url.Parse(remoteAddr)
+	if err == nil && strings.EqualFold(u.Host, "github.com") {
+		gitServiceType = structs.GithubService
+	}
+
 	var opts = migrations.MigrateOptions{
-		CloneAddr:    remoteAddr,
+		CloneAddr:      remoteAddr,
-		RepoName:     form.RepoName,
+		RepoName:       form.RepoName,
-		Description:  form.Description,
+		Description:    form.Description,
-		Private:      form.Private || setting.Repository.ForcePrivate,
+		Private:        form.Private || setting.Repository.ForcePrivate,
-		Mirror:       form.Mirror,
+		Mirror:         form.Mirror,
-		AuthUsername: form.AuthUsername,
+		AuthUsername:   form.AuthUsername,
-		AuthPassword: form.AuthPassword,
+		AuthPassword:   form.AuthPassword,
-		Wiki:         form.Wiki,
+		Wiki:           form.Wiki,
-		Issues:       form.Issues,
+		Issues:         form.Issues,
-		Milestones:   form.Milestones,
+		Milestones:     form.Milestones,
-		Labels:       form.Labels,
+		Labels:         form.Labels,
-		Comments:     true,
+		Comments:       true,
-		PullRequests: form.PullRequests,
+		PullRequests:   form.PullRequests,
-		Releases:     form.Releases,
+		Releases:       form.Releases,
+		GitServiceType: gitServiceType,
 	}
 	if opts.Mirror {
 		opts.Issues = false

routers/user/auth.go

@@ -21,6 +21,7 @@ import (
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/timeutil"
 	"code.gitea.io/gitea/modules/util"
+	"code.gitea.io/gitea/services/externalaccount"
 	"code.gitea.io/gitea/services/mailer"
 
 	"gitea.com/macaron/captcha"
@@ -277,7 +278,7 @@ func TwoFactorPost(ctx *context.Context, form auth.TwoFactorAuthForm) {
 				return
 			}
 
-			err = models.LinkAccountToUser(u, gothUser.(goth.User))
+			err = externalaccount.LinkAccountToUser(u, gothUser.(goth.User))
 			if err != nil {
 				ctx.ServerError("UserSignIn", err)
 				return
@@ -452,7 +453,7 @@ func U2FSign(ctx *context.Context, signResp u2f.SignResponse) {
 					return
 				}
 
-				err = models.LinkAccountToUser(user, gothUser.(goth.User))
+				err = externalaccount.LinkAccountToUser(user, gothUser.(goth.User))
 				if err != nil {
 					ctx.ServerError("UserSignIn", err)
 					return
@@ -601,36 +602,42 @@ func handleOAuth2SignIn(u *models.User, gothUser goth.User, ctx *context.Context
 	// Instead, redirect them to the 2FA authentication page.
 	_, err = models.GetTwoFactorByUID(u.ID)
 	if err != nil {
-		if models.IsErrTwoFactorNotEnrolled(err) {
+		if !models.IsErrTwoFactorNotEnrolled(err) {
-			err = ctx.Session.Set("uid", u.ID)
-			if err != nil {
-				log.Error(fmt.Sprintf("Error setting session: %v", err))
-			}
-			err = ctx.Session.Set("uname", u.Name)
-			if err != nil {
-				log.Error(fmt.Sprintf("Error setting session: %v", err))
-			}
-
-			// Clear whatever CSRF has right now, force to generate a new one
-			ctx.SetCookie(setting.CSRFCookieName, "", -1, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true)
-
-			// Register last login
-			u.SetLastLogin()
-			if err := models.UpdateUserCols(u, "last_login_unix"); err != nil {
-				ctx.ServerError("UpdateUserCols", err)
-				return
-			}
-
-			if redirectTo := ctx.GetCookie("redirect_to"); len(redirectTo) > 0 {
-				ctx.SetCookie("redirect_to", "", -1, setting.AppSubURL, "", setting.SessionConfig.Secure, true)
-				ctx.RedirectToFirst(redirectTo)
-				return
-			}
-
-			ctx.Redirect(setting.AppSubURL + "/")
-		} else {
 			ctx.ServerError("UserSignIn", err)
+			return
 		}
+
+		err = ctx.Session.Set("uid", u.ID)
+		if err != nil {
+			log.Error(fmt.Sprintf("Error setting session: %v", err))
+		}
+		err = ctx.Session.Set("uname", u.Name)
+		if err != nil {
+			log.Error(fmt.Sprintf("Error setting session: %v", err))
+		}
+
+		// Clear whatever CSRF has right now, force to generate a new one
+		ctx.SetCookie(setting.CSRFCookieName, "", -1, setting.AppSubURL, setting.SessionConfig.Domain, setting.SessionConfig.Secure, true)
+
+		// Register last login
+		u.SetLastLogin()
+		if err := models.UpdateUserCols(u, "last_login_unix"); err != nil {
+			ctx.ServerError("UpdateUserCols", err)
+			return
+		}
+
+		// update external user information
+		if err := models.UpdateExternalUser(u, gothUser); err != nil {
+			log.Error("UpdateExternalUser failed: %v", err)
+		}
+
+		if redirectTo := ctx.GetCookie("redirect_to"); len(redirectTo) > 0 {
+			ctx.SetCookie("redirect_to", "", -1, setting.AppSubURL, "", setting.SessionConfig.Secure, true)
+			ctx.RedirectToFirst(redirectTo)
+			return
+		}
+
+		ctx.Redirect(setting.AppSubURL + "/")
 		return
 	}
 
@@ -675,7 +682,7 @@ func oAuth2UserLoginCallback(loginSource *models.LoginSource, request *http.Requ
 	}
 
 	if hasUser {
-		return user, goth.User{}, nil
+		return user, gothUser, nil
 	}
 
 	// search in external linked users
@@ -689,7 +696,7 @@ func oAuth2UserLoginCallback(loginSource *models.LoginSource, request *http.Requ
 	}
 	if hasUser {
 		user, err = models.GetUserByID(externalLoginUser.UserID)
-		return user, goth.User{}, err
+		return user, gothUser, err
 	}
 
 	// no user found to login
@@ -789,16 +796,18 @@ func LinkAccountPostSignIn(ctx *context.Context, signInForm auth.SignInForm) {
 	// Instead, redirect them to the 2FA authentication page.
 	_, err = models.GetTwoFactorByUID(u.ID)
 	if err != nil {
-		if models.IsErrTwoFactorNotEnrolled(err) {
+		if !models.IsErrTwoFactorNotEnrolled(err) {
-			err = models.LinkAccountToUser(u, gothUser.(goth.User))
-			if err != nil {
-				ctx.ServerError("UserLinkAccount", err)
-			} else {
-				handleSignIn(ctx, u, signInForm.Remember)
-			}
-		} else {
 			ctx.ServerError("UserLinkAccount", err)
+			return
 		}
+
+		err = externalaccount.LinkAccountToUser(u, gothUser.(goth.User))
+		if err != nil {
+			ctx.ServerError("UserLinkAccount", err)
+			return
+		}
+
+		handleSignIn(ctx, u, signInForm.Remember)
 		return
 	}
 
@@ -947,6 +956,11 @@ func LinkAccountPostRegister(ctx *context.Context, cpt *captcha.Captcha, form au
 		}
 	}
 
+	// update external user information
+	if err := models.UpdateExternalUser(u, gothUser.(goth.User)); err != nil {
+		log.Error("UpdateExternalUser failed: %v", err)
+	}
+
 	// Send confirmation email
 	if setting.Service.RegisterEmailConfirm && u.ID > 1 {
 		mailer.SendActivateAccountMail(ctx.Locale, u)

services/externalaccount/user.go

@@ -0,0 +1,66 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package externalaccount
+
+import (
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/structs"
+
+	"github.com/markbates/goth"
+)
+
+// LinkAccountToUser link the gothUser to the user
+func LinkAccountToUser(user *models.User, gothUser goth.User) error {
+	loginSource, err := models.GetActiveOAuth2LoginSourceByName(gothUser.Provider)
+	if err != nil {
+		return err
+	}
+
+	externalLoginUser := &models.ExternalLoginUser{
+		ExternalID:        gothUser.UserID,
+		UserID:            user.ID,
+		LoginSourceID:     loginSource.ID,
+		RawData:           gothUser.RawData,
+		Provider:          gothUser.Provider,
+		Email:             gothUser.Email,
+		Name:              gothUser.Name,
+		FirstName:         gothUser.FirstName,
+		LastName:          gothUser.LastName,
+		NickName:          gothUser.NickName,
+		Description:       gothUser.Description,
+		AvatarURL:         gothUser.AvatarURL,
+		Location:          gothUser.Location,
+		AccessToken:       gothUser.AccessToken,
+		AccessTokenSecret: gothUser.AccessTokenSecret,
+		RefreshToken:      gothUser.RefreshToken,
+		ExpiresAt:         gothUser.ExpiresAt,
+	}
+
+	if err := models.LinkExternalToUser(user, externalLoginUser); err != nil {
+		return err
+	}
+
+	externalID, err := strconv.ParseInt(externalLoginUser.ExternalID, 10, 64)
+	if err != nil {
+		return err
+	}
+
+	var tp structs.GitServiceType
+	for _, s := range structs.SupportedFullGitService {
+		if strings.EqualFold(s.Name(), gothUser.Provider) {
+			tp = s
+			break
+		}
+	}
+
+	if tp.Name() != "" {
+		return models.UpdateMigrationsByType(tp, externalID, user.ID)
+	}
+
+	return nil
+}