forgejo/modules/migrations/github.go
mrsdizzie 1f1ecda541 Display original author and URL information when showing migrated issues/comments (#7352)
* Store original author info for migrated issues and comments

Keep original author name for displaying in Gitea interface and also
store original author user ID for potential future use in linking
accounts from old location.

* Add original_url for repo

Store the original URL for a migrated repo

Clean up migrations/tests

* fix migration

* fix golangci-lint

* make 'make revive' happy also

* Modify templates to use OriginalAuthor if set

Use the original author name in templates if it is set rather than the
user who migrated/currently owns the issues

* formatting fixes

* make generate-swagger

* Use default avatar for imported comments

* Remove no longer used IgnoreIssueAuthor option

* Add OriginalAuthorID to swagger also
2019-07-08 10:14:12 +08:00

487 lines
12 KiB
Go

// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"net/http"
"net/url"
"strings"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"github.com/google/go-github/v24/github"
"golang.org/x/oauth2"
)
var (
_ base.Downloader = &GithubDownloaderV3{}
_ base.DownloaderFactory = &GithubDownloaderV3Factory{}
)
func init() {
RegisterDownloaderFactory(&GithubDownloaderV3Factory{})
}
// GithubDownloaderV3Factory defines a github downloader v3 factory
type GithubDownloaderV3Factory struct {
}
// Match returns ture if the migration remote URL matched this downloader factory
func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error) {
u, err := url.Parse(opts.RemoteURL)
if err != nil {
return false, err
}
return u.Host == "github.com" && opts.AuthUsername != "", nil
}
// New returns a Downloader related to this factory according MigrateOptions
func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.RemoteURL)
if err != nil {
return nil, err
}
fields := strings.Split(u.Path, "/")
oldOwner := fields[1]
oldName := strings.TrimSuffix(fields[2], ".git")
log.Trace("Create github downloader: %s/%s", oldOwner, oldName)
return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil
}
// GithubDownloaderV3 implements a Downloader interface to get repository informations
// from github via APIv3
type GithubDownloaderV3 struct {
ctx context.Context
client *github.Client
repoOwner string
repoName string
userName string
password string
}
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *GithubDownloaderV3 {
var downloader = GithubDownloaderV3{
userName: userName,
password: password,
ctx: context.Background(),
repoOwner: repoOwner,
repoName: repoName,
}
var client *http.Client
if userName != "" {
if password == "" {
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: userName},
)
client = oauth2.NewClient(downloader.ctx, ts)
} else {
client = &http.Client{
Transport: &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
return nil, nil
},
},
}
}
}
downloader.client = github.NewClient(client)
return &downloader
}
// GetRepoInfo returns a repository information
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
gr, _, err := g.client.Repositories.Get(g.ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
// convert github repo to stand Repo
return &base.Repository{
Owner: g.repoOwner,
Name: gr.GetName(),
IsPrivate: *gr.Private,
Description: gr.GetDescription(),
OriginalURL: gr.GetHTMLURL(),
CloneURL: gr.GetCloneURL(),
}, nil
}
// GetMilestones returns milestones
func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
var perPage = 100
var milestones = make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ {
ms, _, err := g.client.Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
&github.MilestoneListOptions{
State: "all",
ListOptions: github.ListOptions{
Page: i,
PerPage: perPage,
}})
if err != nil {
return nil, err
}
for _, m := range ms {
var desc string
if m.Description != nil {
desc = *m.Description
}
var state = "open"
if m.State != nil {
state = *m.State
}
milestones = append(milestones, &base.Milestone{
Title: *m.Title,
Description: desc,
Deadline: m.DueOn,
State: state,
Created: *m.CreatedAt,
Updated: m.UpdatedAt,
Closed: m.ClosedAt,
})
}
if len(ms) < perPage {
break
}
}
return milestones, nil
}
func convertGithubLabel(label *github.Label) *base.Label {
var desc string
if label.Description != nil {
desc = *label.Description
}
return &base.Label{
Name: *label.Name,
Color: *label.Color,
Description: desc,
}
}
// GetLabels returns labels
func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
var perPage = 100
var labels = make([]*base.Label, 0, perPage)
for i := 1; ; i++ {
ls, _, err := g.client.Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
&github.ListOptions{
Page: i,
PerPage: perPage,
})
if err != nil {
return nil, err
}
for _, label := range ls {
labels = append(labels, convertGithubLabel(label))
}
if len(ls) < perPage {
break
}
}
return labels, nil
}
func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) *base.Release {
var (
name string
desc string
)
if rel.Body != nil {
desc = *rel.Body
}
if rel.Name != nil {
name = *rel.Name
}
r := &base.Release{
TagName: *rel.TagName,
TargetCommitish: *rel.TargetCommitish,
Name: name,
Body: desc,
Draft: *rel.Draft,
Prerelease: *rel.Prerelease,
Created: rel.CreatedAt.Time,
Published: rel.PublishedAt.Time,
}
for _, asset := range rel.Assets {
u, _ := url.Parse(*asset.BrowserDownloadURL)
u.User = url.UserPassword(g.userName, g.password)
r.Assets = append(r.Assets, base.ReleaseAsset{
URL: u.String(),
Name: *asset.Name,
ContentType: asset.ContentType,
Size: asset.Size,
DownloadCount: asset.DownloadCount,
Created: asset.CreatedAt.Time,
Updated: asset.UpdatedAt.Time,
})
}
return r
}
// GetReleases returns releases
func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
var perPage = 100
var releases = make([]*base.Release, 0, perPage)
for i := 1; ; i++ {
ls, _, err := g.client.Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
&github.ListOptions{
Page: i,
PerPage: perPage,
})
if err != nil {
return nil, err
}
for _, release := range ls {
releases = append(releases, g.convertGithubRelease(release))
}
if len(ls) < perPage {
break
}
}
return releases, nil
}
func convertGithubReactions(reactions *github.Reactions) *base.Reactions {
return &base.Reactions{
TotalCount: *reactions.TotalCount,
PlusOne: *reactions.PlusOne,
MinusOne: *reactions.MinusOne,
Laugh: *reactions.Laugh,
Confused: *reactions.Confused,
Heart: *reactions.Heart,
Hooray: *reactions.Hooray,
}
}
// GetIssues returns issues according start and limit
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
opt := &github.IssueListByRepoOptions{
Sort: "created",
Direction: "asc",
State: "all",
ListOptions: github.ListOptions{
PerPage: perPage,
Page: page,
},
}
var allIssues = make([]*base.Issue, 0, perPage)
issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
for _, issue := range issues {
if issue.IsPullRequest() {
continue
}
var body string
if issue.Body != nil {
body = *issue.Body
}
var milestone string
if issue.Milestone != nil {
milestone = *issue.Milestone.Title
}
var labels = make([]*base.Label, 0, len(issue.Labels))
for _, l := range issue.Labels {
labels = append(labels, convertGithubLabel(&l))
}
var reactions *base.Reactions
if issue.Reactions != nil {
reactions = convertGithubReactions(issue.Reactions)
}
var email string
if issue.User.Email != nil {
email = *issue.User.Email
}
allIssues = append(allIssues, &base.Issue{
Title: *issue.Title,
Number: int64(*issue.Number),
PosterID: *issue.User.ID,
PosterName: *issue.User.Login,
PosterEmail: email,
Content: body,
Milestone: milestone,
State: *issue.State,
Created: *issue.CreatedAt,
Labels: labels,
Reactions: reactions,
Closed: issue.ClosedAt,
IsLocked: *issue.Locked,
})
}
return allIssues, len(issues) < perPage, nil
}
// GetComments returns comments according issueNumber
func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, error) {
var allComments = make([]*base.Comment, 0, 100)
opt := &github.IssueListCommentsOptions{
Sort: "created",
Direction: "asc",
ListOptions: github.ListOptions{
PerPage: 100,
},
}
for {
comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
for _, comment := range comments {
var email string
if comment.User.Email != nil {
email = *comment.User.Email
}
var reactions *base.Reactions
if comment.Reactions != nil {
reactions = convertGithubReactions(comment.Reactions)
}
allComments = append(allComments, &base.Comment{
IssueIndex: issueNumber,
PosterID: *comment.User.ID,
PosterName: *comment.User.Login,
PosterEmail: email,
Content: *comment.Body,
Created: *comment.CreatedAt,
Reactions: reactions,
})
}
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
}
return allComments, nil
}
// GetPullRequests returns pull requests according page and perPage
func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, error) {
opt := &github.PullRequestListOptions{
Sort: "created",
Direction: "asc",
State: "all",
ListOptions: github.ListOptions{
PerPage: perPage,
Page: page,
},
}
var allPRs = make([]*base.PullRequest, 0, perPage)
prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
for _, pr := range prs {
var body string
if pr.Body != nil {
body = *pr.Body
}
var milestone string
if pr.Milestone != nil {
milestone = *pr.Milestone.Title
}
var labels = make([]*base.Label, 0, len(pr.Labels))
for _, l := range pr.Labels {
labels = append(labels, convertGithubLabel(l))
}
// FIXME: This API missing reactions, we may need another extra request to get reactions
var email string
if pr.User.Email != nil {
email = *pr.User.Email
}
var merged bool
// pr.Merged is not valid, so use MergedAt to test if it's merged
if pr.MergedAt != nil {
merged = true
}
var (
headRepoName string
cloneURL string
headRef string
headSHA string
)
if pr.Head.Repo != nil {
if pr.Head.Repo.Name != nil {
headRepoName = *pr.Head.Repo.Name
}
if pr.Head.Repo.CloneURL != nil {
cloneURL = *pr.Head.Repo.CloneURL
}
}
if pr.Head.Ref != nil {
headRef = *pr.Head.Ref
}
if pr.Head.SHA != nil {
headSHA = *pr.Head.SHA
}
var mergeCommitSHA string
if pr.MergeCommitSHA != nil {
mergeCommitSHA = *pr.MergeCommitSHA
}
var headUserName string
if pr.Head.User != nil && pr.Head.User.Login != nil {
headUserName = *pr.Head.User.Login
}
allPRs = append(allPRs, &base.PullRequest{
Title: *pr.Title,
Number: int64(*pr.Number),
PosterName: *pr.User.Login,
PosterID: *pr.User.ID,
PosterEmail: email,
Content: body,
Milestone: milestone,
State: *pr.State,
Created: *pr.CreatedAt,
Closed: pr.ClosedAt,
Labels: labels,
Merged: merged,
MergeCommitSHA: mergeCommitSHA,
MergedTime: pr.MergedAt,
IsLocked: pr.ActiveLockReason != nil,
Head: base.PullRequestBranch{
Ref: headRef,
SHA: headSHA,
RepoName: headRepoName,
OwnerName: headUserName,
CloneURL: cloneURL,
},
Base: base.PullRequestBranch{
Ref: *pr.Base.Ref,
SHA: *pr.Base.SHA,
RepoName: *pr.Base.Repo.Name,
OwnerName: *pr.Base.User.Login,
},
PatchURL: *pr.PatchURL,
})
}
return allPRs, nil
}