From 06cc290657a916b70e15e518ed9e4dfb83e9cd41 Mon Sep 17 00:00:00 2001 From: harryzcy Date: Fri, 15 Jul 2022 21:05:45 -0400 Subject: [PATCH] Get new comments/reviews --- modules/migration/downloader.go | 3 +++ modules/migration/null_downloader.go | 19 +++++++++++++++++-- services/migrations/github.go | 27 +++++++++++++++++++++++++-- services/migrations/migrate.go | 10 +++++----- 4 files changed, 50 insertions(+), 9 deletions(-) diff --git a/modules/migration/downloader.go b/modules/migration/downloader.go index 02f01987ce..a22c737f7a 100644 --- a/modules/migration/downloader.go +++ b/modules/migration/downloader.go @@ -30,7 +30,10 @@ type Downloader interface { // For syncing issues and pull requests GetNewIssues(page, perPage int, updatedAfter time.Time) ([]*Issue, bool, error) + GetNewComments(commentable Commentable, updatedAfter time.Time) ([]*Comment, bool, error) + GetAllNewComments(page, perPage int, updatedAfter time.Time) ([]*Comment, bool, error) GetNewPullRequests(page, perPage int, updatedAfter time.Time) ([]*PullRequest, bool, error) + GetNewReviews(reviewable Reviewable, updatedAfter time.Time) ([]*Review, error) } // DownloaderFactory defines an interface to match a downloader implementation and create a downloader diff --git a/modules/migration/null_downloader.go b/modules/migration/null_downloader.go index be2d28923d..3e90ff0fb2 100644 --- a/modules/migration/null_downloader.go +++ b/modules/migration/null_downloader.go @@ -91,10 +91,25 @@ func (n NullDownloader) SupportGetRepoComments() bool { // GetNewIssues returns new issues updated after the given time according start and limit func (n NullDownloader) GetNewIssues(page, perPage int, updatedAfter time.Time) ([]*Issue, bool, error) { - return nil, false, ErrNotSupported{Entity: "Issues"} + return nil, false, ErrNotSupported{Entity: "NewIssues"} +} + +// GetNewComments returns comments of an issue or PR after the given time +func (n NullDownloader) GetNewComments(commentable Commentable, updatedAfter time.Time) ([]*Comment, bool, error) { + return nil, false, ErrNotSupported{Entity: "NewComments"} +} + +// GetAllNewComments returns paginated comments after the given time +func (n NullDownloader) GetAllNewComments(page, perPage int, updatedAfter time.Time) ([]*Comment, bool, error) { + return nil, false, ErrNotSupported{Entity: "AllNewComments"} } // GetNewPullRequests returns pull requests after the given time according page and perPage func (n NullDownloader) GetNewPullRequests(page, perPage int, updatedAfter time.Time) ([]*PullRequest, bool, error) { - return nil, false, ErrNotSupported{Entity: "PullRequests"} + return nil, false, ErrNotSupported{Entity: "NewPullRequests"} +} + +// GetNewReviews returns new pull requests review after the given time +func (n NullDownloader) GetNewReviews(reviewable Reviewable, updatedAfter time.Time) ([]*Review, error) { + return nil, ErrNotSupported{Entity: "NewReviews"} } diff --git a/services/migrations/github.go b/services/migrations/github.go index cb218ed33e..477483db00 100644 --- a/services/migrations/github.go +++ b/services/migrations/github.go @@ -393,11 +393,11 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool { // GetComments returns comments according issueNumber func (g *GithubDownloaderV3) GetComments(commentable base.Commentable) ([]*base.Comment, bool, error) { - comments, err := g.getComments(commentable) + comments, err := g.getCommentsSince(commentable, nil) return comments, false, err } -func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.Comment, error) { +func (g *GithubDownloaderV3) getCommentsSince(commentable base.Commentable, since *time.Time) ([]*base.Comment, error) { var ( allComments = make([]*base.Comment, 0, g.maxPerPage) created = "created" @@ -406,6 +406,7 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. opt := &github.IssueListCommentsOptions{ Sort: &created, Direction: &asc, + Since: since, ListOptions: github.ListOptions{ PerPage: g.maxPerPage, }, @@ -466,6 +467,10 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base. // GetAllComments returns repository comments according page and perPageSize func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) { + return g.getAllCommentsSince(page, perPage, nil) +} + +func (g *GithubDownloaderV3) getAllCommentsSince(page, perPage int, since *time.Time) ([]*base.Comment, bool, error) { var ( allComments = make([]*base.Comment, 0, perPage) created = "created" @@ -477,6 +482,7 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, opt := &github.IssueListCommentsOptions{ Sort: &created, Direction: &asc, + Since: since, ListOptions: github.ListOptions{ Page: page, PerPage: perPage, @@ -772,6 +778,17 @@ func (g *GithubDownloaderV3) getIssuesSince(page, perPage int, since time.Time) return allIssues, len(issues) < perPage, nil } +// GetNewComments returns comments of an issue or PR after the given time +func (g GithubDownloaderV3) GetNewComments(commentable base.Commentable, updatedAfter time.Time) ([]*base.Comment, bool, error) { + comments, err := g.getCommentsSince(commentable, &updatedAfter) + return comments, false, err +} + +// GetAllNewComments returns paginated comments after the given time +func (g GithubDownloaderV3) GetAllNewComments(page, perPage int, updatedAfter time.Time) ([]*base.Comment, bool, error) { + return g.getAllCommentsSince(page, perPage, &updatedAfter) +} + // GetNewPullRequests returns pull requests after the given time according page and perPage func (g *GithubDownloaderV3) GetNewPullRequests(page, perPage int, updatedAfter time.Time) ([]*base.PullRequest, bool, error) { // Every pull request is an issue, and only Issues API provides parameter `since`, @@ -818,6 +835,12 @@ func (g *GithubDownloaderV3) GetNewPullRequests(page, perPage int, updatedAfter return allPRs, len(issues) < perPage, nil } +// GetNewReviews returns new pull requests review after the given time +func (g GithubDownloaderV3) GetNewReviews(reviewable base.Reviewable, updatedAfter time.Time) ([]*base.Review, error) { + // Github does not support since parameter for reviews, so we need to get all reviews + return g.GetReviews(reviewable) +} + func (g *GithubDownloaderV3) convertGithubPullRequest(pr *github.PullRequest, perPage int) (*base.PullRequest, error) { labels := make([]*base.Label, 0, len(pr.Labels)) for _, l := range pr.Labels { diff --git a/services/migrations/migrate.go b/services/migrations/migrate.go index 37f7a63c99..ca56344b7a 100644 --- a/services/migrations/migrate.go +++ b/services/migrations/migrate.go @@ -613,7 +613,7 @@ func syncRepository(downloader base.Downloader, uploader base.Uploader, opts bas allComments := make([]*base.Comment, 0, commentBatchSize) for _, issue := range issues { log.Trace("syncing issue %d's comments", issue.Number) - comments, _, err := downloader.GetComments(issue) + comments, _, err := downloader.GetNewComments(issue, lastSynced) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -670,7 +670,7 @@ func syncRepository(downloader base.Downloader, uploader base.Uploader, opts bas allComments := make([]*base.Comment, 0, commentBatchSize) for _, pr := range prs { log.Trace("syncing pull request %d's comments", pr.Number) - comments, _, err := downloader.GetComments(pr) + comments, _, err := downloader.GetNewComments(pr, lastSynced) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -694,10 +694,10 @@ func syncRepository(downloader base.Downloader, uploader base.Uploader, opts bas } } - // migrate reviews + // sync reviews allReviews := make([]*base.Review, 0, reviewBatchSize) for _, pr := range prs { - reviews, err := downloader.GetReviews(pr) + reviews, err := downloader.GetNewReviews(pr, lastSynced) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -731,7 +731,7 @@ func syncRepository(downloader base.Downloader, uploader base.Uploader, opts bas if opts.Comments && supportAllComments { log.Trace("syncing comments") for i := 1; ; i++ { - comments, isEnd, err := downloader.GetAllComments(i, commentBatchSize) + comments, isEnd, err := downloader.GetAllNewComments(i, commentBatchSize, lastSynced) if err != nil { return err }