first-commit

modules/migration/comment.go

@@ -0,0 +1,34 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import "time"
+
+// Commentable can be commented upon
+type Commentable interface {
+	Reviewable
+	GetContext() DownloaderContext
+}
+
+// Comment is a standard comment information
+type Comment struct {
+	IssueIndex  int64 `yaml:"issue_index"`
+	Index       int64
+	CommentType string `yaml:"comment_type"` // see `commentStrings` in models/issues/comment.go
+	PosterID    int64  `yaml:"poster_id"`
+	PosterName  string `yaml:"poster_name"`
+	PosterEmail string `yaml:"poster_email"`
+	Created     time.Time
+	Updated     time.Time
+	Content     string
+	Reactions   []*Reaction
+	Meta        map[string]any `yaml:"meta,omitempty"` // see models/issues/comment.go for fields in Comment struct
+}
+
+// GetExternalName ExternalUserMigrated interface
+func (c *Comment) GetExternalName() string { return c.PosterName }
+
+// ExternalID ExternalUserMigrated interface
+func (c *Comment) GetExternalID() int64 { return c.PosterID }

modules/migration/downloader.go

@@ -0,0 +1,36 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import (
+	"context"
+
+	"code.gitea.io/gitea/modules/structs"
+)
+
+// Downloader downloads the site repo information
+type Downloader interface {
+	GetRepoInfo(ctx context.Context) (*Repository, error)
+	GetTopics(ctx context.Context) ([]string, error)
+	GetMilestones(ctx context.Context) ([]*Milestone, error)
+	GetReleases(ctx context.Context) ([]*Release, error)
+	GetLabels(ctx context.Context) ([]*Label, error)
+	GetIssues(ctx context.Context, page, perPage int) ([]*Issue, bool, error)
+	GetComments(ctx context.Context, commentable Commentable) ([]*Comment, bool, error)
+	GetAllComments(ctx context.Context, page, perPage int) ([]*Comment, bool, error)
+	SupportGetRepoComments() bool
+	GetPullRequests(ctx context.Context, page, perPage int) ([]*PullRequest, bool, error)
+	GetReviews(ctx context.Context, reviewable Reviewable) ([]*Review, error)
+	FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error)
+}
+
+// DownloaderFactory defines an interface to match a downloader implementation and create a downloader
+type DownloaderFactory interface {
+	New(ctx context.Context, opts MigrateOptions) (Downloader, error)
+	GitServiceType() structs.GitServiceType
+}
+
+// DownloaderContext has opaque information only relevant to a given downloader
+type DownloaderContext any

modules/migration/error.go

@@ -0,0 +1,25 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import "fmt"
+
+// ErrNotSupported represents status if a downloader do not supported something.
+type ErrNotSupported struct {
+	Entity string
+}
+
+// IsErrNotSupported checks if an error is an ErrNotSupported
+func IsErrNotSupported(err error) bool {
+	_, ok := err.(ErrNotSupported)
+	return ok
+}
+
+// Error return error message
+func (err ErrNotSupported) Error() string {
+	if len(err.Entity) != 0 {
+		return fmt.Sprintf("'%s' not supported", err.Entity)
+	}
+	return "not supported"
+}

modules/migration/file_format.go

@@ -0,0 +1,110 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"time"
+
+	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/log"
+
+	"github.com/santhosh-tekuri/jsonschema/v5"
+	"gopkg.in/yaml.v3"
+)
+
+// Load project data from file, with optional validation
+func Load(filename string, data any, validation bool) error {
+	isJSON := strings.HasSuffix(filename, ".json")
+
+	bs, err := os.ReadFile(filename)
+	if err != nil {
+		return err
+	}
+
+	if validation {
+		err := validate(bs, data, isJSON)
+		if err != nil {
+			return err
+		}
+	}
+	return unmarshal(bs, data, isJSON)
+}
+
+func unmarshal(bs []byte, data any, isJSON bool) error {
+	if isJSON {
+		return json.Unmarshal(bs, data)
+	}
+	return yaml.Unmarshal(bs, data)
+}
+
+func getSchema(filename string) (*jsonschema.Schema, error) {
+	c := jsonschema.NewCompiler()
+	c.LoadURL = openSchema
+	return c.Compile(filename)
+}
+
+func validate(bs []byte, datatype any, isJSON bool) error {
+	var v any
+	err := unmarshal(bs, &v, isJSON)
+	if err != nil {
+		return err
+	}
+	if !isJSON {
+		v, err = toStringKeys(v)
+		if err != nil {
+			return err
+		}
+	}
+
+	var schemaFilename string
+	switch datatype := datatype.(type) {
+	case *[]*Issue:
+		schemaFilename = "issue.json"
+	case *[]*Milestone:
+		schemaFilename = "milestone.json"
+	default:
+		return fmt.Errorf("file_format:validate: %T has not a validation implemented", datatype)
+	}
+
+	sch, err := getSchema(schemaFilename)
+	if err != nil {
+		return err
+	}
+	err = sch.Validate(v)
+	if err != nil {
+		log.Error("migration validation with %s failed:\n%#v", schemaFilename, err)
+	}
+	return err
+}
+
+func toStringKeys(val any) (any, error) {
+	var err error
+	switch val := val.(type) {
+	case map[string]any:
+		m := make(map[string]any)
+		for k, v := range val {
+			m[k], err = toStringKeys(v)
+			if err != nil {
+				return nil, err
+			}
+		}
+		return m, nil
+	case []any:
+		l := make([]any, len(val))
+		for i, v := range val {
+			l[i], err = toStringKeys(v)
+			if err != nil {
+				return nil, err
+			}
+		}
+		return l, nil
+	case time.Time:
+		return val.Format(time.RFC3339), nil
+	default:
+		return val, nil
+	}
+}

modules/migration/file_format_test.go

@@ -0,0 +1,38 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/santhosh-tekuri/jsonschema/v5"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestMigrationJSON_IssueOK(t *testing.T) {
+	issues := make([]*Issue, 0, 10)
+	err := Load("file_format_testdata/issue_a.json", &issues, true)
+	assert.NoError(t, err)
+	err = Load("file_format_testdata/issue_a.yml", &issues, true)
+	assert.NoError(t, err)
+}
+
+func TestMigrationJSON_IssueFail(t *testing.T) {
+	issues := make([]*Issue, 0, 10)
+	err := Load("file_format_testdata/issue_b.json", &issues, true)
+	if _, ok := err.(*jsonschema.ValidationError); ok {
+		errors := strings.Split(err.(*jsonschema.ValidationError).GoString(), "\n")
+		assert.Contains(t, errors[1], "missing properties")
+		assert.Contains(t, errors[1], "poster_id")
+	} else {
+		t.Fatalf("got: type %T with value %s, want: *jsonschema.ValidationError", err, err)
+	}
+}
+
+func TestMigrationJSON_MilestoneOK(t *testing.T) {
+	milestones := make([]*Milestone, 0, 10)
+	err := Load("file_format_testdata/milestones.json", &milestones, true)
+	assert.NoError(t, err)
+}

modules/migration/file_format_testdata/issue_a.json

@@ -0,0 +1,14 @@
+[
+  {
+      "number": 1,
+      "poster_id": 1,
+      "poster_name": "name_a",
+      "title": "title_a",
+      "content": "content_a",
+      "state": "closed",
+      "is_locked": false,
+      "created": "1985-04-12T23:20:50.52Z",
+      "updated": "1986-04-12T23:20:50.52Z",
+      "closed": "1987-04-12T23:20:50.52Z"
+  }
+]

modules/migration/file_format_testdata/issue_a.yml

@@ -0,0 +1,10 @@
+- number: 1
+  poster_id: 1
+  poster_name: name_a
+  title: title_a
+  content: content_a
+  state: closed
+  is_locked: false
+  created: 2021-05-27T15:24:13+02:00
+  updated: 2021-11-11T10:52:45+01:00
+  closed: 2021-11-11T10:52:45+01:00

modules/migration/file_format_testdata/issue_b.json

@@ -0,0 +1,5 @@
+[
+  {
+      "number": 1
+  }
+]

modules/migration/file_format_testdata/milestones.json

@@ -0,0 +1,20 @@
+[
+  {
+      "title": "title_a",
+      "description": "description_a",
+      "deadline": "1988-04-12T23:20:50.52Z",
+      "created": "1985-04-12T23:20:50.52Z",
+      "updated": "1986-04-12T23:20:50.52Z",
+      "closed": "1987-04-12T23:20:50.52Z",
+      "state": "closed"
+  },
+  {
+      "title": "title_b",
+      "description": "description_b",
+      "deadline": "1998-04-12T23:20:50.52Z",
+      "created": "1995-04-12T23:20:50.52Z",
+      "updated": "1996-04-12T23:20:50.52Z",
+      "closed": null,
+      "state": "open"
+  }
+]

modules/migration/issue.go

@@ -0,0 +1,48 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import "time"
+
+// Issue is a standard issue information
+type Issue struct {
+	Number       int64             `json:"number"`
+	PosterID     int64             `yaml:"poster_id" json:"poster_id"`
+	PosterName   string            `yaml:"poster_name" json:"poster_name"`
+	PosterEmail  string            `yaml:"poster_email" json:"poster_email"`
+	Title        string            `json:"title"`
+	Content      string            `json:"content"`
+	Ref          string            `json:"ref"`
+	Milestone    string            `json:"milestone"`
+	State        string            `json:"state"` // closed, open
+	IsLocked     bool              `yaml:"is_locked" json:"is_locked"`
+	Created      time.Time         `json:"created"`
+	Updated      time.Time         `json:"updated"`
+	Closed       *time.Time        `json:"closed"`
+	Labels       []*Label          `json:"labels"`
+	Reactions    []*Reaction       `json:"reactions"`
+	Assignees    []string          `json:"assignees"`
+	ForeignIndex int64             `json:"foreign_id"`
+	Context      DownloaderContext `yaml:"-"`
+}
+
+// GetExternalName ExternalUserMigrated interface
+func (issue *Issue) GetExternalName() string { return issue.PosterName }
+
+// GetExternalID ExternalUserMigrated interface
+func (issue *Issue) GetExternalID() int64 { return issue.PosterID }
+
+func (issue *Issue) GetLocalIndex() int64 { return issue.Number }
+
+func (issue *Issue) GetForeignIndex() int64 {
+	// see the comment of Reviewable.GetForeignIndex
+	// if there is no ForeignIndex, then use LocalIndex
+	if issue.ForeignIndex == 0 {
+		return issue.Number
+	}
+	return issue.ForeignIndex
+}
+
+func (issue *Issue) GetContext() DownloaderContext { return issue.Context }

modules/migration/label.go

@@ -0,0 +1,13 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+// Label defines a standard label information
+type Label struct {
+	Name        string `json:"name"`
+	Color       string `json:"color"`
+	Description string `json:"description"`
+	Exclusive   bool   `json:"exclusive"`
+}

modules/migration/messenger.go

@@ -0,0 +1,10 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+// Messenger is a formatting function similar to i18n.TrString
+type Messenger func(key string, args ...any)
+
+// NilMessenger represents an empty formatting function
+func NilMessenger(string, ...any) {}

modules/migration/milestone.go

@@ -0,0 +1,18 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import "time"
+
+// Milestone defines a standard milestone
+type Milestone struct {
+	Title       string     `json:"title"`
+	Description string     `json:"description"`
+	Deadline    *time.Time `json:"deadline"`
+	Created     time.Time  `json:"created"`
+	Updated     *time.Time `json:"updated"`
+	Closed      *time.Time `json:"closed"`
+	State       string     `json:"state"` // open, closed
+}

modules/migration/null_downloader.go

@@ -0,0 +1,85 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import (
+	"context"
+	"net/url"
+)
+
+// NullDownloader implements a blank downloader
+type NullDownloader struct{}
+
+var _ Downloader = &NullDownloader{}
+
+// GetRepoInfo returns a repository information
+func (n NullDownloader) GetRepoInfo(_ context.Context) (*Repository, error) {
+	return nil, ErrNotSupported{Entity: "RepoInfo"}
+}
+
+// GetTopics return repository topics
+func (n NullDownloader) GetTopics(_ context.Context) ([]string, error) {
+	return nil, ErrNotSupported{Entity: "Topics"}
+}
+
+// GetMilestones returns milestones
+func (n NullDownloader) GetMilestones(_ context.Context) ([]*Milestone, error) {
+	return nil, ErrNotSupported{Entity: "Milestones"}
+}
+
+// GetReleases returns releases
+func (n NullDownloader) GetReleases(_ context.Context) ([]*Release, error) {
+	return nil, ErrNotSupported{Entity: "Releases"}
+}
+
+// GetLabels returns labels
+func (n NullDownloader) GetLabels(_ context.Context) ([]*Label, error) {
+	return nil, ErrNotSupported{Entity: "Labels"}
+}
+
+// GetIssues returns issues according start and limit
+func (n NullDownloader) GetIssues(_ context.Context, page, perPage int) ([]*Issue, bool, error) {
+	return nil, false, ErrNotSupported{Entity: "Issues"}
+}
+
+// GetComments returns comments of an issue or PR
+func (n NullDownloader) GetComments(_ context.Context, commentable Commentable) ([]*Comment, bool, error) {
+	return nil, false, ErrNotSupported{Entity: "Comments"}
+}
+
+// GetAllComments returns paginated comments
+func (n NullDownloader) GetAllComments(_ context.Context, page, perPage int) ([]*Comment, bool, error) {
+	return nil, false, ErrNotSupported{Entity: "AllComments"}
+}
+
+// GetPullRequests returns pull requests according page and perPage
+func (n NullDownloader) GetPullRequests(_ context.Context, page, perPage int) ([]*PullRequest, bool, error) {
+	return nil, false, ErrNotSupported{Entity: "PullRequests"}
+}
+
+// GetReviews returns pull requests review
+func (n NullDownloader) GetReviews(_ context.Context, reviewable Reviewable) ([]*Review, error) {
+	return nil, ErrNotSupported{Entity: "Reviews"}
+}
+
+// FormatCloneURL add authentication into remote URLs
+func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
+	if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
+		u, err := url.Parse(remoteAddr)
+		if err != nil {
+			return "", err
+		}
+		u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
+		if len(opts.AuthToken) > 0 {
+			u.User = url.UserPassword("oauth2", opts.AuthToken)
+		}
+		return u.String(), nil
+	}
+	return remoteAddr, nil
+}
+
+// SupportGetRepoComments return true if it supports get repo comments
+func (n NullDownloader) SupportGetRepoComments() bool {
+	return false
+}

modules/migration/options.go

@@ -0,0 +1,44 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import "code.gitea.io/gitea/modules/structs"
+
+// MigrateOptions defines the way a repository gets migrated
+// this is for internal usage by migrations module and func who interact with it
+type MigrateOptions struct {
+	// required: true
+	CloneAddr             string `json:"clone_addr" binding:"Required"`
+	CloneAddrEncrypted    string `json:"clone_addr_encrypted,omitempty"`
+	AuthUsername          string `json:"auth_username"`
+	AuthPassword          string `json:"-"`
+	AuthPasswordEncrypted string `json:"auth_password_encrypted,omitempty"`
+	AuthToken             string `json:"-"`
+	AuthTokenEncrypted    string `json:"auth_token_encrypted,omitempty"`
+	// required: true
+	UID int `json:"uid" binding:"Required"`
+	// required: true
+	RepoName        string `json:"repo_name" binding:"Required"`
+	Mirror          bool   `json:"mirror"`
+	LFS             bool   `json:"lfs"`
+	LFSEndpoint     string `json:"lfs_endpoint"`
+	Private         bool   `json:"private"`
+	Description     string `json:"description"`
+	OriginalURL     string
+	GitServiceType  structs.GitServiceType
+	Wiki            bool
+	Issues          bool
+	Milestones      bool
+	Labels          bool
+	Releases        bool
+	Comments        bool
+	PullRequests    bool
+	ReleaseAssets   bool
+	MigrateToRepoID int64
+	MirrorInterval  string `json:"mirror_interval"`
+
+	AWSAccessKeyID     string
+	AWSSecretAccessKey string
+}

modules/migration/pullrequest.go

@@ -0,0 +1,75 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import (
+	"fmt"
+	"time"
+
+	"code.gitea.io/gitea/modules/git"
+)
+
+// PullRequest defines a standard pull request information
+type PullRequest struct {
+	Number         int64
+	Title          string
+	PosterName     string `yaml:"poster_name"`
+	PosterID       int64  `yaml:"poster_id"`
+	PosterEmail    string `yaml:"poster_email"`
+	Content        string
+	Milestone      string
+	State          string
+	Created        time.Time
+	Updated        time.Time
+	Closed         *time.Time
+	Labels         []*Label
+	PatchURL       string `yaml:"patch_url"` // SECURITY: This must be safe to download directly from
+	Merged         bool
+	MergedTime     *time.Time `yaml:"merged_time"`
+	MergeCommitSHA string     `yaml:"merge_commit_sha"`
+	Head           PullRequestBranch
+	Base           PullRequestBranch
+	Assignees      []string
+	IsLocked       bool `yaml:"is_locked"`
+	Reactions      []*Reaction
+	ForeignIndex   int64
+	Context        DownloaderContext `yaml:"-"`
+	EnsuredSafe    bool              `yaml:"ensured_safe"`
+	IsDraft        bool              `yaml:"is_draft"`
+}
+
+func (p *PullRequest) GetLocalIndex() int64          { return p.Number }
+func (p *PullRequest) GetForeignIndex() int64        { return p.ForeignIndex }
+func (p *PullRequest) GetContext() DownloaderContext { return p.Context }
+
+// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository
+func (p *PullRequest) IsForkPullRequest() bool {
+	return p.Head.RepoFullName() != p.Base.RepoFullName()
+}
+
+// GetGitRefName returns pull request relative path to head
+func (p PullRequest) GetGitRefName() string {
+	return fmt.Sprintf("%s%d/head", git.PullPrefix, p.Number)
+}
+
+// PullRequestBranch represents a pull request branch
+type PullRequestBranch struct {
+	CloneURL  string `yaml:"clone_url"` // SECURITY: This must be safe to download from
+	Ref       string // SECURITY: this must be a git.IsValidRefPattern
+	SHA       string // SECURITY: this must be a git.IsValidSHAPattern
+	RepoName  string `yaml:"repo_name"`
+	OwnerName string `yaml:"owner_name"`
+}
+
+// RepoFullName returns pull request repo full name
+func (p PullRequestBranch) RepoFullName() string {
+	return fmt.Sprintf("%s/%s", p.OwnerName, p.RepoName)
+}
+
+// GetExternalName ExternalUserMigrated interface
+func (p *PullRequest) GetExternalName() string { return p.PosterName }
+
+// ExternalID ExternalUserMigrated interface
+func (p *PullRequest) GetExternalID() int64 { return p.PosterID }

modules/migration/reaction.go

@@ -0,0 +1,17 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+// Reaction represents a reaction to an issue/pr/comment.
+type Reaction struct {
+	UserID   int64  `yaml:"user_id" json:"user_id"`
+	UserName string `yaml:"user_name" json:"user_name"`
+	Content  string `json:"content"`
+}
+
+// GetExternalName ExternalUserMigrated interface
+func (r *Reaction) GetExternalName() string { return r.UserName }
+
+// GetExternalID ExternalUserMigrated interface
+func (r *Reaction) GetExternalID() int64 { return r.UserID }

modules/migration/release.go

@@ -0,0 +1,46 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import (
+	"io"
+	"time"
+)
+
+// ReleaseAsset represents a release asset
+type ReleaseAsset struct {
+	ID            int64
+	Name          string
+	ContentType   *string `yaml:"content_type"`
+	Size          *int
+	DownloadCount *int `yaml:"download_count"`
+	Created       time.Time
+	Updated       time.Time
+
+	DownloadURL *string `yaml:"download_url"` // SECURITY: It is the responsibility of downloader to make sure this is safe
+	// if DownloadURL is nil, the function should be invoked
+	DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` // SECURITY: It is the responsibility of downloader to make sure this is safe
+}
+
+// Release represents a release
+type Release struct {
+	TagName         string `yaml:"tag_name"`         // SECURITY: This must pass git.IsValidRefPattern
+	TargetCommitish string `yaml:"target_commitish"` // SECURITY: This must pass git.IsValidRefPattern
+	Name            string
+	Body            string
+	Draft           bool
+	Prerelease      bool
+	PublisherID     int64  `yaml:"publisher_id"`
+	PublisherName   string `yaml:"publisher_name"`
+	PublisherEmail  string `yaml:"publisher_email"`
+	Assets          []*ReleaseAsset
+	Created         time.Time
+	Published       time.Time
+}
+
+// GetExternalName ExternalUserMigrated interface
+func (r *Release) GetExternalName() string { return r.PublisherName }
+
+// GetExternalID ExternalUserMigrated interface
+func (r *Release) GetExternalID() int64 { return r.PublisherID }

modules/migration/repo.go

@@ -0,0 +1,17 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+// Repository defines a standard repository information
+type Repository struct {
+	Name          string
+	Owner         string
+	IsPrivate     bool `yaml:"is_private"`
+	IsMirror      bool `yaml:"is_mirror"`
+	Description   string
+	CloneURL      string `yaml:"clone_url"` // SECURITY: This must be checked to ensure that is safe to be used
+	OriginalURL   string `yaml:"original_url"`
+	DefaultBranch string
+}

modules/migration/retry_downloader.go

@@ -0,0 +1,185 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import (
+	"context"
+	"time"
+)
+
+var _ Downloader = &RetryDownloader{}
+
+// RetryDownloader retry the downloads
+type RetryDownloader struct {
+	Downloader
+	RetryTimes int // the total execute times
+	RetryDelay int // time to delay seconds
+}
+
+// NewRetryDownloader creates a retry downloader
+func NewRetryDownloader(downloader Downloader, retryTimes, retryDelay int) *RetryDownloader {
+	return &RetryDownloader{
+		Downloader: downloader,
+		RetryTimes: retryTimes,
+		RetryDelay: retryDelay,
+	}
+}
+
+func (d *RetryDownloader) retry(ctx context.Context, work func(context.Context) error) error {
+	var (
+		times = d.RetryTimes
+		err   error
+	)
+	for ; times > 0; times-- {
+		if err = work(ctx); err == nil {
+			return nil
+		}
+		if IsErrNotSupported(err) {
+			return err
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-time.After(time.Second * time.Duration(d.RetryDelay)):
+		}
+	}
+	return err
+}
+
+// GetRepoInfo returns a repository information with retry
+func (d *RetryDownloader) GetRepoInfo(ctx context.Context) (*Repository, error) {
+	var (
+		repo *Repository
+		err  error
+	)
+
+	err = d.retry(ctx, func(ctx context.Context) error {
+		repo, err = d.Downloader.GetRepoInfo(ctx)
+		return err
+	})
+
+	return repo, err
+}
+
+// GetTopics returns a repository's topics with retry
+func (d *RetryDownloader) GetTopics(ctx context.Context) ([]string, error) {
+	var (
+		topics []string
+		err    error
+	)
+
+	err = d.retry(ctx, func(ctx context.Context) error {
+		topics, err = d.Downloader.GetTopics(ctx)
+		return err
+	})
+
+	return topics, err
+}
+
+// GetMilestones returns a repository's milestones with retry
+func (d *RetryDownloader) GetMilestones(ctx context.Context) ([]*Milestone, error) {
+	var (
+		milestones []*Milestone
+		err        error
+	)
+
+	err = d.retry(ctx, func(ctx context.Context) error {
+		milestones, err = d.Downloader.GetMilestones(ctx)
+		return err
+	})
+
+	return milestones, err
+}
+
+// GetReleases returns a repository's releases with retry
+func (d *RetryDownloader) GetReleases(ctx context.Context) ([]*Release, error) {
+	var (
+		releases []*Release
+		err      error
+	)
+
+	err = d.retry(ctx, func(ctx context.Context) error {
+		releases, err = d.Downloader.GetReleases(ctx)
+		return err
+	})
+
+	return releases, err
+}
+
+// GetLabels returns a repository's labels with retry
+func (d *RetryDownloader) GetLabels(ctx context.Context) ([]*Label, error) {
+	var (
+		labels []*Label
+		err    error
+	)
+
+	err = d.retry(ctx, func(ctx context.Context) error {
+		labels, err = d.Downloader.GetLabels(ctx)
+		return err
+	})
+
+	return labels, err
+}
+
+// GetIssues returns a repository's issues with retry
+func (d *RetryDownloader) GetIssues(ctx context.Context, page, perPage int) ([]*Issue, bool, error) {
+	var (
+		issues []*Issue
+		isEnd  bool
+		err    error
+	)
+
+	err = d.retry(ctx, func(ctx context.Context) error {
+		issues, isEnd, err = d.Downloader.GetIssues(ctx, page, perPage)
+		return err
+	})
+
+	return issues, isEnd, err
+}
+
+// GetComments returns a repository's comments with retry
+func (d *RetryDownloader) GetComments(ctx context.Context, commentable Commentable) ([]*Comment, bool, error) {
+	var (
+		comments []*Comment
+		isEnd    bool
+		err      error
+	)
+
+	err = d.retry(ctx, func(context.Context) error {
+		comments, isEnd, err = d.Downloader.GetComments(ctx, commentable)
+		return err
+	})
+
+	return comments, isEnd, err
+}
+
+// GetPullRequests returns a repository's pull requests with retry
+func (d *RetryDownloader) GetPullRequests(ctx context.Context, page, perPage int) ([]*PullRequest, bool, error) {
+	var (
+		prs   []*PullRequest
+		err   error
+		isEnd bool
+	)
+
+	err = d.retry(ctx, func(ctx context.Context) error {
+		prs, isEnd, err = d.Downloader.GetPullRequests(ctx, page, perPage)
+		return err
+	})
+
+	return prs, isEnd, err
+}
+
+// GetReviews returns pull requests reviews
+func (d *RetryDownloader) GetReviews(ctx context.Context, reviewable Reviewable) ([]*Review, error) {
+	var (
+		reviews []*Review
+		err     error
+	)
+	err = d.retry(ctx, func(ctx context.Context) error {
+		reviews, err = d.Downloader.GetReviews(ctx, reviewable)
+		return err
+	})
+
+	return reviews, err
+}

modules/migration/review.go

@@ -0,0 +1,67 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import "time"
+
+// Reviewable can be reviewed
+type Reviewable interface {
+	GetLocalIndex() int64
+
+	// GetForeignIndex presents the foreign index, which could be misused:
+	// For example, if there are 2 Gitea sites: site-A exports a dataset, then site-B imports it:
+	// * if site-A exports files by using its LocalIndex
+	// * from site-A's view, LocalIndex is site-A's IssueIndex while ForeignIndex is site-B's IssueIndex
+	// * but from site-B's view, LocalIndex is site-B's IssueIndex while ForeignIndex is site-A's IssueIndex
+	//
+	// So the exporting/importing must be paired, but the meaning of them looks confusing then:
+	// * either site-A and site-B both use LocalIndex during dumping/restoring
+	// * or site-A and site-B both use ForeignIndex
+	GetForeignIndex() int64
+}
+
+// enumerate all review states
+const (
+	ReviewStatePending          = "PENDING"
+	ReviewStateApproved         = "APPROVED"
+	ReviewStateChangesRequested = "CHANGES_REQUESTED"
+	ReviewStateCommented        = "COMMENTED"
+	ReviewStateRequestReview    = "REQUEST_REVIEW"
+)
+
+// Review is a standard review information
+type Review struct {
+	ID           int64
+	IssueIndex   int64  `yaml:"issue_index"`
+	ReviewerID   int64  `yaml:"reviewer_id"`
+	ReviewerName string `yaml:"reviewer_name"`
+	Official     bool
+	CommitID     string `yaml:"commit_id"`
+	Content      string
+	CreatedAt    time.Time `yaml:"created_at"`
+	State        string    // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT
+	Comments     []*ReviewComment
+}
+
+// GetExternalName ExternalUserMigrated interface
+func (r *Review) GetExternalName() string { return r.ReviewerName }
+
+// GetExternalID ExternalUserMigrated interface
+func (r *Review) GetExternalID() int64 { return r.ReviewerID }
+
+// ReviewComment represents a review comment
+type ReviewComment struct {
+	ID        int64
+	InReplyTo int64 `yaml:"in_reply_to"`
+	Content   string
+	TreePath  string `yaml:"tree_path"`
+	DiffHunk  string `yaml:"diff_hunk"`
+	Position  int
+	Line      int
+	CommitID  string `yaml:"commit_id"`
+	PosterID  int64  `yaml:"poster_id"`
+	Reactions []*Reaction
+	CreatedAt time.Time `yaml:"created_at"`
+	UpdatedAt time.Time `yaml:"updated_at"`
+}

modules/migration/schemas/issue.json

@@ -0,0 +1,114 @@
+{
+    "title": "Issue",
+    "description": "Issues associated to a repository within a forge (Gitea, GitLab, etc.).",
+
+    "type": "array",
+    "items": {
+	"type": "object",
+	"additionalProperties": false,
+	"properties": {
+	    "number": {
+		"description": "Unique identifier, relative to the repository.",
+		"type": "number"
+	    },
+	    "poster_id": {
+		"description": "Unique identifier of the user who authored the issue.",
+		"type": "number"
+	    },
+	    "poster_name": {
+		"description": "Name of the user who authored the issue.",
+		"type": "string"
+	    },
+	    "poster_email": {
+		"description": "Email of the user who authored the issue.",
+		"type": "string"
+	    },
+	    "title": {
+		"description": "Short description displayed as the title.",
+		"type": "string"
+	    },
+	    "content": {
+		"description": "Long, multiline, description.",
+		"type": "string"
+	    },
+	    "ref": {
+		"description": "Target branch in the repository.",
+		"type": "string"
+	    },
+	    "milestone": {
+		"description": "Name of the milestone.",
+		"type": "string"
+	    },
+	    "state": {
+		"description": "A 'closed' issue will not see any activity in the future, otherwise it is 'open'.",
+		"enum": [
+		    "closed",
+		    "open"
+		]
+	    },
+	    "is_locked": {
+		"description": "A locked issue can only be modified by privileged users.",
+		"type": "boolean"
+	    },
+	    "created": {
+		"description": "Creation time.",
+		"type": "string",
+		"format": "date-time"
+	    },
+	    "updated": {
+		"description": "Last update time.",
+		"type": "string",
+		"format": "date-time"
+	    },
+	    "closed": {
+		"description": "The last time 'state' changed to 'closed'.",
+		"anyOf": [
+		    {
+			"type": "string",
+			"format": "date-time"
+		    },
+		    {
+			"type": "null"
+		    }
+		]
+	    },
+	    "labels": {
+		"description": "List of labels.",
+		"type": "array",
+		"items": {
+		    "$ref": "label.json"
+		}
+	    },
+	    "reactions": {
+		"description": "List of reactions.",
+		"type": "array",
+		"items": {
+		    "$ref": "reaction.json"
+		}
+	    },
+	    "assignees": {
+		"description": "List of assignees.",
+		"type": "array",
+		"items": {
+		    "description": "Name of a user assigned to the issue.",
+		    "type": "string"
+		}
+	    }
+	},
+	"required": [
+	    "number",
+	    "poster_id",
+	    "poster_name",
+	    "title",
+	    "content",
+	    "state",
+	    "is_locked",
+	    "created",
+	    "updated"
+	]
+    },
+
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$id": "http://example.com/issue.json",
+    "$$target": "issue.json"
+}

modules/migration/schemas/label.json

@@ -0,0 +1,28 @@
+{
+    "title": "Label",
+    "description": "Label associated to an issue.",
+
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {
+	"name": {
+	    "description": "Name of the label, unique within the repository.",
+	    "type": "string"
+	},
+	"color": {
+	    "description": "Color code of the label.",
+	    "type": "string"
+	},
+	"description": {
+	    "description": "Long, multiline, description.",
+	    "type": "string"
+	}
+    },
+    "required": [
+	"name"
+    ],
+
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$id": "label.json",
+    "$$target": "label.json"
+}

modules/migration/schemas/milestone.json

@@ -0,0 +1,67 @@
+{
+    "title": "Milestone",
+    "description": "Milestone associated to a repository within a forge.",
+
+    "type": "array",
+    "items": {
+	"type": "object",
+	"additionalProperties": false,
+	"properties": {
+	    "title": {
+		"description": "Short description.",
+		"type": "string"
+	    },
+	    "description": {
+		"description": "Long, multiline, description.",
+		"type": "string"
+	    },
+	    "deadline": {
+		"description": "Deadline after which the milestone is overdue.",
+		"type": "string",
+		"format": "date-time"
+	    },
+	    "created": {
+		"description": "Creation time.",
+		"type": "string",
+		"format": "date-time"
+	    },
+	    "updated": {
+		"description": "Last update time.",
+		"type": "string",
+		"format": "date-time"
+	    },
+	    "closed": {
+		"description": "The last time 'state' changed to 'closed'.",
+		"anyOf": [
+		    {
+			"type": "string",
+			"format": "date-time"
+		    },
+		    {
+			"type": "null"
+		    }
+		]
+	    },
+	    "state": {
+		"description": "A 'closed' issue will not see any activity in the future, otherwise it is 'open'.",
+		"enum": [
+		    "closed",
+		    "open"
+		]
+	    }
+	},
+	"required": [
+	    "title",
+	    "description",
+	    "deadline",
+	    "created",
+	    "updated",
+	    "closed",
+	    "state"
+	]
+    },
+
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$id": "http://example.com/milestone.json",
+    "$$target": "milestone.json"
+}

modules/migration/schemas/reaction.json

@@ -0,0 +1,29 @@
+{
+    "title": "Reaction",
+    "description": "Reaction associated to an issue or a comment.",
+
+    "type": "object",
+    "additionalProperties": false,
+    "properties": {
+	"user_id": {
+	    "description": "Unique identifier of the user who authored the reaction.",
+	    "type": "number"
+	},
+	"user_name": {
+	    "description": "Name of the user who authored the reaction.",
+	    "type": "string"
+	},
+	"content": {
+	    "description": "Representation of the reaction",
+	    "type": "string"
+	}
+    },
+    "required": [
+	"user_id",
+	"content"
+    ],
+
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "$id": "http://example.com/reaction.json",
+    "$$target": "reaction.json"
+}

modules/migration/schemas_bindata.go

@@ -0,0 +1,30 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build bindata
+
+//go:generate go run ../../build/generate-bindata.go ../../modules/migration/schemas bindata.dat
+
+package migration
+
+import (
+	"io"
+	"io/fs"
+	"path"
+	"sync"
+
+	_ "embed"
+
+	"code.gitea.io/gitea/modules/assetfs"
+)
+
+//go:embed bindata.dat
+var bindata []byte
+
+var BuiltinAssets = sync.OnceValue(func() fs.FS {
+	return assetfs.NewEmbeddedFS(bindata)
+})
+
+func openSchema(filename string) (io.ReadCloser, error) {
+	return BuiltinAssets().Open(path.Base(filename))
+}

modules/migration/schemas_dynamic.go

@@ -0,0 +1,38 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build !bindata
+
+package migration
+
+import (
+	"io"
+	"net/url"
+	"os"
+	"path"
+	"path/filepath"
+)
+
+func openSchema(s string) (io.ReadCloser, error) {
+	u, err := url.Parse(s)
+	if err != nil {
+		return nil, err
+	}
+	basename := path.Base(u.Path)
+	filename := basename
+	//
+	// Schema reference each other within the schemas directory but
+	// the tests run in the parent directory.
+	//
+	if _, err := os.Stat(filename); os.IsNotExist(err) {
+		filename = filepath.Join("schemas", basename)
+		//
+		// Integration tests run from the git root directory, not the
+		// directory in which the test source is located.
+		//
+		if _, err := os.Stat(filename); os.IsNotExist(err) {
+			filename = filepath.Join("modules/migration/schemas", basename)
+		}
+	}
+	return os.Open(filename)
+}

modules/migration/uploader.go

@@ -0,0 +1,25 @@
+// Copyright 2019 The Gitea Authors. All rights reserved.
+// Copyright 2018 Jonas Franz. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package migration
+
+import "context"
+
+// Uploader uploads all the information of one repository
+type Uploader interface {
+	MaxBatchInsertSize(tp string) int
+	CreateRepo(ctx context.Context, repo *Repository, opts MigrateOptions) error
+	CreateTopics(ctx context.Context, topic ...string) error
+	CreateMilestones(ctx context.Context, milestones ...*Milestone) error
+	CreateReleases(ctx context.Context, releases ...*Release) error
+	SyncTags(ctx context.Context) error
+	CreateLabels(ctx context.Context, labels ...*Label) error
+	CreateIssues(ctx context.Context, issues ...*Issue) error
+	CreateComments(ctx context.Context, comments ...*Comment) error
+	CreatePullRequests(ctx context.Context, prs ...*PullRequest) error
+	CreateReviews(ctx context.Context, reviews ...*Review) error
+	Rollback() error
+	Finish(ctx context.Context) error
+	Close()
+}