oss/forgejo
oss/forgejo
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2025-10-05 19:30:58 +00:00
Code Wiki Activity

[v12.0/forgejo] fix: migration failing when importing either issues or PRs but not the other (#9121)

Browse source 
**Backport:** https://codeberg.org/forgejo/forgejo/pulls/8892

Related to https://codeberg.org/Codeberg/Community/issues/1944

## Checklist

The [contributor guide](https://forgejo.org/docs/next/contributor/) contains information that will be helpful to first time contributors. There also are a few [conditions for merging Pull Requests in Forgejo repositories](https://codeberg.org/forgejo/governance/src/branch/main/PullRequestsAgreement.md). You are also welcome to join the [Forgejo development chatroom](https://matrix.to/#/#forgejo-development:matrix.org).

### Tests

- I added test coverage for Go changes...
  - [x] in their respective `*_test.go` for unit tests.
  - [ ] in the `tests/integration` directory if it involves interactions with a live Forgejo server.
- I added test coverage for JavaScript changes...
  - [ ] in `web_src/js/*.test.js` if it can be unit tested.
  - [ ] in `tests/e2e/*.test.e2e.js` if it requires interactions with a live Forgejo server (see also the [developer guide for JavaScript testing](https://codeberg.org/forgejo/forgejo/src/branch/forgejo/tests/e2e/README.md#end-to-end-tests)).

### Documentation

- [ ] I created a pull request [to the documentation](https://codeberg.org/forgejo/docs) to explain to Forgejo users how to use this change.
  - A place for fix architecture is still in discussion with @fnetX
- [x] I did not document these changes and I do not expect someone else to do it.

### Release notes

- [ ] I do not want this change to show in the release notes.
- [x] I want the title to show in the release notes with a link to this pull request.
- [ ] I want the content of the `release-notes/<pull request number>.md` to be be used for the release notes instead of the title.

## What was done

* Allowed the githubdownloaderv3 to know whether issues and, or PRs are requested to migrate
* Used this information to decide to filter for "/pulls/" or "/issues"
  * Or not to filter at all if issues == true && prs == true
* Added isolated test for the downloader and for the uploader
* Created a new test_repo in github.com/forgejo and set it up properly together with @Gusted
* Updated github_downloader_test with the new URLs and test data from the repo
* Recorded the API calls for local testing
* Added a minimal gitbucket test (which uses the github downloader under the hood)

Co-authored-by: patdyn <patdyn@noreply.codeberg.org>
Reviewed-on: https://codeberg.org/forgejo/forgejo/pulls/9121
Reviewed-by: Earl Warren <earl-warren@noreply.codeberg.org>
Co-authored-by: forgejo-backport-action <forgejo-backport-action@noreply.codeberg.org>
Co-committed-by: forgejo-backport-action <forgejo-backport-action@noreply.codeberg.org>
This commit is contained in:
forgejo-backport-action 2025-09-02 10:31:10 +02:00 • committed by Earl Warren
parent c897e40d4f
commit 19d920931d
70 changed files with 966 additions and 980 deletions
Download patch file Download diff file Expand all files Collapse all files

70
services/migrations/github.go
View file

@ -57,7 +57,7 @@ func (f *GithubDownloaderV3Factory) New(ctx context.Context, opts base.MigrateOp
log.Trace("Create github downloader BaseURL: %s %s/%s", baseURL, oldOwner, oldName)
return NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil
return NewGithubDownloaderV3(ctx, baseURL, opts.PullRequests, opts.Issues, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil
}
// GitServiceType returns the type of git service
@ -69,30 +69,34 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
// from github via APIv3
type GithubDownloaderV3 struct {
base.NullDownloader
ctx context.Context
clients []*github.Client
baseURL string
repoOwner string
repoName string
userName string
password string
rates []*github.Rate
curClientIdx int
maxPerPage int
SkipReactions bool
SkipReviews bool
ctx context.Context
clients []*github.Client
baseURL string
repoOwner string
repoName string
userName string
password string
getPullRequests bool
getIssues bool
rates []*github.Rate
curClientIdx int
maxPerPage int
SkipReactions bool
SkipReviews bool
}
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
func NewGithubDownloaderV3(ctx context.Context, baseURL string, getPullRequests, getIssues bool, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
downloader := GithubDownloaderV3{
userName: userName,
baseURL: baseURL,
password: password,
ctx: ctx,
repoOwner: repoOwner,
repoName: repoName,
maxPerPage: 100,
userName: userName,
baseURL: baseURL,
password: password,
ctx: ctx,
repoOwner: repoOwner,
repoName: repoName,
maxPerPage: 100,
getPullRequests: getPullRequests,
getIssues: getIssues,
}
if token != "" {
@ -582,6 +586,24 @@ func (g *GithubDownloaderV3) getComments(commentable base.Commentable) ([]*base.
return allComments, nil
}
func (g *GithubDownloaderV3) filterByHTMLURL(comments []*github.IssueComment, filterBy string) []*github.IssueComment {
var result []*github.IssueComment
for _, val := range comments {
if !strings.Contains(*val.HTMLURL, filterBy) {
result = append(result, val)
}
}
return result
}
func (g *GithubDownloaderV3) filterPRComments(comments []*github.IssueComment) []*github.IssueComment {
return g.filterByHTMLURL(comments, "/pull/")
}
func (g *GithubDownloaderV3) filterIssueComments(comments []*github.IssueComment) []*github.IssueComment {
return g.filterByHTMLURL(comments, "/issues/")
}
// GetAllComments returns repository comments according page and perPageSize
func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) {
var (
@ -608,6 +630,12 @@ func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment,
}
isEnd := resp.NextPage == 0
if g.getIssues && !g.getPullRequests {
comments = g.filterPRComments(comments)
} else if !g.getIssues && g.getPullRequests {
comments = g.filterIssueComments(comments)
}
log.Trace("Request get comments %d/%d, but in fact get %d, next page is %d", perPage, page, len(comments), resp.NextPage)
g.setRate(&resp.Rate)
for _, comment := range comments {