From d130cd147f24eb222a539a5b262d166f94aad5fa Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sun, 25 Oct 2020 13:11:03 +0800 Subject: [PATCH] Fix bug isEnd detection on getIssues/getPullRequests (#13299) --- modules/migrations/github.go | 52 +++++++++++++++++++++--------------- modules/migrations/gitlab.go | 30 ++++++++++++++------- 2 files changed, 50 insertions(+), 32 deletions(-) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 088e54744d9..7aa1e57274c 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -65,23 +65,25 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType { // GithubDownloaderV3 implements a Downloader interface to get repository informations // from github via APIv3 type GithubDownloaderV3 struct { - ctx context.Context - client *github.Client - repoOwner string - repoName string - userName string - password string - rate *github.Rate + ctx context.Context + client *github.Client + repoOwner string + repoName string + userName string + password string + rate *github.Rate + maxPerPage int } // NewGithubDownloaderV3 creates a github Downloader via github v3 API func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 { var downloader = GithubDownloaderV3{ - userName: userName, - password: password, - ctx: ctx, - repoOwner: repoOwner, - repoName: repoName, + userName: userName, + password: password, + ctx: ctx, + repoOwner: repoOwner, + repoName: repoName, + maxPerPage: 100, } client := &http.Client{ @@ -177,7 +179,7 @@ func (g *GithubDownloaderV3) GetTopics() ([]string, error) { // GetMilestones returns milestones func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { - var perPage = 100 + var perPage = g.maxPerPage var milestones = make([]*base.Milestone, 0, perPage) for i := 1; ; i++ { g.sleep() @@ -233,7 +235,7 @@ func convertGithubLabel(label *github.Label) *base.Label { // GetLabels returns labels func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { - var perPage = 100 + var perPage = g.maxPerPage var labels = make([]*base.Label, 0, perPage) for i := 1; ; i++ { g.sleep() @@ -304,7 +306,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) // GetReleases returns releases func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { - var perPage = 100 + var perPage = g.maxPerPage var releases = make([]*base.Release, 0, perPage) for i := 1; ; i++ { g.sleep() @@ -342,6 +344,9 @@ func (g *GithubDownloaderV3) GetAsset(_ string, _, id int64) (io.ReadCloser, err // GetIssues returns issues according start and limit func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { + if perPage > g.maxPerPage { + perPage = g.maxPerPage + } opt := &github.IssueListByRepoOptions{ Sort: "created", Direction: "asc", @@ -429,7 +434,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, // GetComments returns comments according issueNumber func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, error) { var ( - allComments = make([]*base.Comment, 0, 100) + allComments = make([]*base.Comment, 0, g.maxPerPage) created = "created" asc = "asc" ) @@ -437,7 +442,7 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er Sort: &created, Direction: &asc, ListOptions: github.ListOptions{ - PerPage: 100, + PerPage: g.maxPerPage, }, } for { @@ -459,7 +464,7 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er g.sleep() res, resp, err := g.client.Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ Page: i, - PerPage: 100, + PerPage: g.maxPerPage, }) if err != nil { return nil, err @@ -497,6 +502,9 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er // GetPullRequests returns pull requests according page and perPage func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { + if perPage > g.maxPerPage { + perPage = g.maxPerPage + } opt := &github.PullRequestListOptions{ Sort: "created", Direction: "asc", @@ -650,7 +658,7 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques g.sleep() res, resp, err := g.client.Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{ Page: i, - PerPage: 100, + PerPage: g.maxPerPage, }) if err != nil { return nil, err @@ -687,9 +695,9 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques // GetReviews returns pull requests review func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { - var allReviews = make([]*base.Review, 0, 100) + var allReviews = make([]*base.Review, 0, g.maxPerPage) opt := &github.ListOptions{ - PerPage: 100, + PerPage: g.maxPerPage, } for { g.sleep() @@ -703,7 +711,7 @@ func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review r.IssueIndex = pullRequestNumber // retrieve all review comments opt2 := &github.ListOptions{ - PerPage: 100, + PerPage: g.maxPerPage, } for { g.sleep() diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go index 23cd90c7478..d4b725b5bea 100644 --- a/modules/migrations/gitlab.go +++ b/modules/migrations/gitlab.go @@ -68,6 +68,7 @@ type GitlabDownloader struct { repoName string issueCount int64 fetchPRcomments bool + maxPerPage int } // NewGitlabDownloader creates a gitlab Downloader via gitlab API @@ -99,10 +100,11 @@ func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, passw } return &GitlabDownloader{ - ctx: ctx, - client: gitlabClient, - repoID: gr.ID, - repoName: gr.Name, + ctx: ctx, + client: gitlabClient, + repoID: gr.ID, + repoName: gr.Name, + maxPerPage: 100, }, nil } @@ -159,7 +161,7 @@ func (g *GitlabDownloader) GetTopics() ([]string, error) { // GetMilestones returns milestones func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) { - var perPage = 100 + var perPage = g.maxPerPage var state = "all" var milestones = make([]*base.Milestone, 0, perPage) for i := 1; ; i++ { @@ -230,7 +232,7 @@ func (g *GitlabDownloader) normalizeColor(val string) string { // GetLabels returns labels func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) { - var perPage = 100 + var perPage = g.maxPerPage var labels = make([]*base.Label, 0, perPage) for i := 1; ; i++ { ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{ @@ -281,7 +283,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea // GetReleases returns releases func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) { - var perPage = 100 + var perPage = g.maxPerPage var releases = make([]*base.Release, 0, perPage) for i := 1; ; i++ { ls, _, err := g.client.Releases.ListReleases(g.repoID, &gitlab.ListReleasesOptions{ @@ -330,6 +332,10 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er state := "all" sort := "asc" + if perPage > g.maxPerPage { + perPage = g.maxPerPage + } + opt := &gitlab.ListProjectIssuesOptions{ State: &state, Sort: &sort, @@ -401,7 +407,7 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er // GetComments returns comments according issueNumber // TODO: figure out how to transfer comment reactions func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) { - var allComments = make([]*base.Comment, 0, 100) + var allComments = make([]*base.Comment, 0, g.maxPerPage) var page = 1 var realIssueNumber int64 @@ -415,14 +421,14 @@ func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, erro realIssueNumber = issueNumber comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListIssueDiscussionsOptions{ Page: page, - PerPage: 100, + PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) } else { // If this is a PR, we need to figure out the Gitlab/original PR ID to be passed below realIssueNumber = issueNumber - g.issueCount comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListMergeRequestDiscussionsOptions{ Page: page, - PerPage: 100, + PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) } @@ -465,6 +471,10 @@ func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, erro // GetPullRequests returns pull requests according page and perPage func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { + if perPage > g.maxPerPage { + perPage = g.maxPerPage + } + opt := &gitlab.ListProjectMergeRequestsOptions{ ListOptions: gitlab.ListOptions{ PerPage: perPage,