first-commit

models/unittest/consistency.go

@@ -0,0 +1,189 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package unittest
+
+import (
+	"reflect"
+	"strconv"
+	"strings"
+
+	"code.gitea.io/gitea/models/db"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"xorm.io/builder"
+)
+
+const (
+	// these const values are copied from `models` package to prevent from cycle-import
+	modelsUserTypeOrganization = 1
+	modelsRepoWatchModeDont    = 2
+	modelsCommentTypeComment   = 0
+)
+
+var consistencyCheckMap = make(map[string]func(t assert.TestingT, bean any))
+
+// CheckConsistencyFor test that all matching database entries are consistent
+func CheckConsistencyFor(t require.TestingT, beansToCheck ...any) {
+	for _, bean := range beansToCheck {
+		sliceType := reflect.SliceOf(reflect.TypeOf(bean))
+		sliceValue := reflect.MakeSlice(sliceType, 0, 10)
+
+		ptrToSliceValue := reflect.New(sliceType)
+		ptrToSliceValue.Elem().Set(sliceValue)
+
+		assert.NoError(t, db.GetEngine(db.DefaultContext).Table(bean).Find(ptrToSliceValue.Interface()))
+		sliceValue = ptrToSliceValue.Elem()
+
+		for i := 0; i < sliceValue.Len(); i++ {
+			entity := sliceValue.Index(i).Interface()
+			checkForConsistency(t, entity)
+		}
+	}
+}
+
+func checkForConsistency(t require.TestingT, bean any) {
+	tb, err := db.TableInfo(bean)
+	assert.NoError(t, err)
+	f := consistencyCheckMap[tb.Name]
+	require.NotNil(t, f, "unknown bean type: %#v", bean)
+	f(t, bean)
+}
+
+func init() {
+	parseBool := func(v string) bool {
+		b, _ := strconv.ParseBool(v)
+		return b
+	}
+	parseInt := func(v string) int {
+		i, _ := strconv.Atoi(v)
+		return i
+	}
+
+	checkForUserConsistency := func(t assert.TestingT, bean any) {
+		user := reflectionWrap(bean)
+		AssertCountByCond(t, "repository", builder.Eq{"owner_id": user.int("ID")}, user.int("NumRepos"))
+		AssertCountByCond(t, "star", builder.Eq{"uid": user.int("ID")}, user.int("NumStars"))
+		AssertCountByCond(t, "org_user", builder.Eq{"org_id": user.int("ID")}, user.int("NumMembers"))
+		AssertCountByCond(t, "team", builder.Eq{"org_id": user.int("ID")}, user.int("NumTeams"))
+		AssertCountByCond(t, "follow", builder.Eq{"user_id": user.int("ID")}, user.int("NumFollowing"))
+		AssertCountByCond(t, "follow", builder.Eq{"follow_id": user.int("ID")}, user.int("NumFollowers"))
+		if user.int("Type") != modelsUserTypeOrganization {
+			assert.Equal(t, 0, user.int("NumMembers"), "Unexpected number of members for user id: %d", user.int("ID"))
+			assert.Equal(t, 0, user.int("NumTeams"), "Unexpected number of teams for user id: %d", user.int("ID"))
+		}
+	}
+
+	checkForRepoConsistency := func(t assert.TestingT, bean any) {
+		repo := reflectionWrap(bean)
+		assert.Equal(t, repo.str("LowerName"), strings.ToLower(repo.str("Name")), "repo: %+v", repo)
+		AssertCountByCond(t, "star", builder.Eq{"repo_id": repo.int("ID")}, repo.int("NumStars"))
+		AssertCountByCond(t, "milestone", builder.Eq{"repo_id": repo.int("ID")}, repo.int("NumMilestones"))
+		AssertCountByCond(t, "repository", builder.Eq{"fork_id": repo.int("ID")}, repo.int("NumForks"))
+		if repo.bool("IsFork") {
+			AssertExistsAndLoadMap(t, "repository", builder.Eq{"id": repo.int("ForkID")})
+		}
+
+		actual := GetCountByCond(t, "watch", builder.Eq{"repo_id": repo.int("ID")}.
+			And(builder.Neq{"mode": modelsRepoWatchModeDont}))
+		assert.EqualValues(t, repo.int("NumWatches"), actual,
+			"Unexpected number of watches for repo id: %d", repo.int("ID"))
+
+		actual = GetCountByCond(t, "issue", builder.Eq{"is_pull": false, "repo_id": repo.int("ID")})
+		assert.EqualValues(t, repo.int("NumIssues"), actual,
+			"Unexpected number of issues for repo id: %d", repo.int("ID"))
+
+		actual = GetCountByCond(t, "issue", builder.Eq{"is_pull": false, "is_closed": true, "repo_id": repo.int("ID")})
+		assert.EqualValues(t, repo.int("NumClosedIssues"), actual,
+			"Unexpected number of closed issues for repo id: %d", repo.int("ID"))
+
+		actual = GetCountByCond(t, "issue", builder.Eq{"is_pull": true, "repo_id": repo.int("ID")})
+		assert.EqualValues(t, repo.int("NumPulls"), actual,
+			"Unexpected number of pulls for repo id: %d", repo.int("ID"))
+
+		actual = GetCountByCond(t, "issue", builder.Eq{"is_pull": true, "is_closed": true, "repo_id": repo.int("ID")})
+		assert.EqualValues(t, repo.int("NumClosedPulls"), actual,
+			"Unexpected number of closed pulls for repo id: %d", repo.int("ID"))
+
+		actual = GetCountByCond(t, "milestone", builder.Eq{"is_closed": true, "repo_id": repo.int("ID")})
+		assert.EqualValues(t, repo.int("NumClosedMilestones"), actual,
+			"Unexpected number of closed milestones for repo id: %d", repo.int("ID"))
+	}
+
+	checkForIssueConsistency := func(t assert.TestingT, bean any) {
+		issue := reflectionWrap(bean)
+		typeComment := modelsCommentTypeComment
+		actual := GetCountByCond(t, "comment", builder.Eq{"`type`": typeComment, "issue_id": issue.int("ID")})
+		assert.EqualValues(t, issue.int("NumComments"), actual, "Unexpected number of comments for issue id: %d", issue.int("ID"))
+		if issue.bool("IsPull") {
+			prRow := AssertExistsAndLoadMap(t, "pull_request", builder.Eq{"issue_id": issue.int("ID")})
+			assert.Equal(t, parseInt(prRow["index"]), issue.int("Index"), "Unexpected index for issue id: %d", issue.int("ID"))
+		}
+	}
+
+	checkForPullRequestConsistency := func(t assert.TestingT, bean any) {
+		pr := reflectionWrap(bean)
+		issueRow := AssertExistsAndLoadMap(t, "issue", builder.Eq{"id": pr.int("IssueID")})
+		assert.True(t, parseBool(issueRow["is_pull"]))
+		assert.Equal(t, parseInt(issueRow["index"]), pr.int("Index"), "Unexpected index for pull request id: %d", pr.int("ID"))
+	}
+
+	checkForMilestoneConsistency := func(t assert.TestingT, bean any) {
+		milestone := reflectionWrap(bean)
+		AssertCountByCond(t, "issue", builder.Eq{"milestone_id": milestone.int("ID")}, milestone.int("NumIssues"))
+
+		actual := GetCountByCond(t, "issue", builder.Eq{"is_closed": true, "milestone_id": milestone.int("ID")})
+		assert.EqualValues(t, milestone.int("NumClosedIssues"), actual, "Unexpected number of closed issues for milestone id: %d", milestone.int("ID"))
+
+		completeness := 0
+		if milestone.int("NumIssues") > 0 {
+			completeness = milestone.int("NumClosedIssues") * 100 / milestone.int("NumIssues")
+		}
+		assert.Equal(t, completeness, milestone.int("Completeness"))
+	}
+
+	checkForLabelConsistency := func(t assert.TestingT, bean any) {
+		label := reflectionWrap(bean)
+		issueLabels, err := db.GetEngine(db.DefaultContext).Table("issue_label").
+			Where(builder.Eq{"label_id": label.int("ID")}).
+			Query()
+		assert.NoError(t, err)
+
+		assert.Len(t, issueLabels, label.int("NumIssues"), "Unexpected number of issue for label id: %d", label.int("ID"))
+
+		issueIDs := make([]int, len(issueLabels))
+		for i, issueLabel := range issueLabels {
+			issueIDs[i], _ = strconv.Atoi(string(issueLabel["issue_id"]))
+		}
+
+		expected := int64(0)
+		if len(issueIDs) > 0 {
+			expected = GetCountByCond(t, "issue", builder.In("id", issueIDs).And(builder.Eq{"is_closed": true}))
+		}
+		assert.EqualValues(t, expected, label.int("NumClosedIssues"), "Unexpected number of closed issues for label id: %d", label.int("ID"))
+	}
+
+	checkForTeamConsistency := func(t assert.TestingT, bean any) {
+		team := reflectionWrap(bean)
+		AssertCountByCond(t, "team_user", builder.Eq{"team_id": team.int("ID")}, team.int("NumMembers"))
+		AssertCountByCond(t, "team_repo", builder.Eq{"team_id": team.int("ID")}, team.int("NumRepos"))
+	}
+
+	checkForActionConsistency := func(t assert.TestingT, bean any) {
+		action := reflectionWrap(bean)
+		if action.int("RepoID") != 1700 { // dangling intentional
+			repoRow := AssertExistsAndLoadMap(t, "repository", builder.Eq{"id": action.int("RepoID")})
+			assert.Equal(t, parseBool(repoRow["is_private"]), action.bool("IsPrivate"), "Unexpected is_private field for action id: %d", action.int("ID"))
+		}
+	}
+
+	consistencyCheckMap["user"] = checkForUserConsistency
+	consistencyCheckMap["repository"] = checkForRepoConsistency
+	consistencyCheckMap["issue"] = checkForIssueConsistency
+	consistencyCheckMap["pull_request"] = checkForPullRequestConsistency
+	consistencyCheckMap["milestone"] = checkForMilestoneConsistency
+	consistencyCheckMap["label"] = checkForLabelConsistency
+	consistencyCheckMap["team"] = checkForTeamConsistency
+	consistencyCheckMap["action"] = checkForActionConsistency
+}

models/unittest/fixtures.go

@@ -0,0 +1,84 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package unittest
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/auth/password/hash"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/util"
+
+	"xorm.io/xorm"
+	"xorm.io/xorm/schemas"
+)
+
+type FixturesLoader interface {
+	Load() error
+}
+
+var fixturesLoader FixturesLoader
+
+// GetXORMEngine gets the XORM engine
+func GetXORMEngine() (x *xorm.Engine) {
+	return db.GetEngine(db.DefaultContext).(*xorm.Engine)
+}
+
+func loadFixtureResetSeqPgsql(e *xorm.Engine) error {
+	results, err := e.QueryString(`SELECT 'SELECT SETVAL(' ||
+		quote_literal(quote_ident(PGT.schemaname) || '.' || quote_ident(S.relname)) ||
+		', COALESCE(MAX(' ||quote_ident(C.attname)|| '), 1) ) FROM ' ||
+		quote_ident(PGT.schemaname)|| '.'||quote_ident(T.relname)|| ';'
+	 FROM pg_class AS S,
+	      pg_depend AS D,
+	      pg_class AS T,
+	      pg_attribute AS C,
+	      pg_tables AS PGT
+	 WHERE S.relkind = 'S'
+	     AND S.oid = D.objid
+	     AND D.refobjid = T.oid
+	     AND D.refobjid = C.attrelid
+	     AND D.refobjsubid = C.attnum
+	     AND T.relname = PGT.tablename
+	 ORDER BY S.relname;`)
+	if err != nil {
+		return fmt.Errorf("failed to generate sequence update: %w", err)
+	}
+	for _, r := range results {
+		for _, value := range r {
+			_, err = e.Exec(value)
+			if err != nil {
+				return fmt.Errorf("failed to update sequence: %s, error: %w", value, err)
+			}
+		}
+	}
+	return nil
+}
+
+// InitFixtures initialize test fixtures for a test database
+func InitFixtures(opts FixturesOptions, engine ...*xorm.Engine) (err error) {
+	xormEngine := util.IfZero(util.OptionalArg(engine), GetXORMEngine())
+	fixturesLoader, err = NewFixturesLoader(xormEngine, opts)
+	// fixturesLoader = NewFixturesLoaderVendor(xormEngine, opts)
+
+	// register the dummy hash algorithm function used in the test fixtures
+	_ = hash.Register("dummy", hash.NewDummyHasher)
+	setting.PasswordHashAlgo, _ = hash.SetDefaultPasswordHashAlgorithm("dummy")
+	return err
+}
+
+// LoadFixtures load fixtures for a test database
+func LoadFixtures() error {
+	if err := fixturesLoader.Load(); err != nil {
+		return err
+	}
+	// Now if we're running postgres we need to tell it to update the sequences
+	if GetXORMEngine().Dialect().URI().DBType == schemas.POSTGRES {
+		if err := loadFixtureResetSeqPgsql(GetXORMEngine()); err != nil {
+			return err
+		}
+	}
+	return nil
+}

models/unittest/fixtures_loader.go

@@ -0,0 +1,225 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package unittest
+
+import (
+	"database/sql"
+	"encoding/hex"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"code.gitea.io/gitea/models/db"
+
+	"gopkg.in/yaml.v3"
+	"xorm.io/xorm"
+	"xorm.io/xorm/schemas"
+)
+
+type FixtureItem struct {
+	fileFullPath string
+	tableName    string
+
+	tableNameQuoted string
+	sqlInserts      []string
+	sqlInsertArgs   [][]any
+
+	mssqlHasIdentityColumn bool
+}
+
+type fixturesLoaderInternal struct {
+	xormEngine       *xorm.Engine
+	xormTableNames   map[string]bool
+	db               *sql.DB
+	dbType           schemas.DBType
+	fixtures         map[string]*FixtureItem
+	quoteObject      func(string) string
+	paramPlaceholder func(idx int) string
+}
+
+func (f *fixturesLoaderInternal) mssqlTableHasIdentityColumn(db *sql.DB, tableName string) (bool, error) {
+	row := db.QueryRow(`SELECT COUNT(*) FROM sys.identity_columns WHERE OBJECT_ID = OBJECT_ID(?)`, tableName)
+	var count int
+	if err := row.Scan(&count); err != nil {
+		return false, err
+	}
+	return count > 0, nil
+}
+
+func (f *fixturesLoaderInternal) preprocessFixtureRow(row []map[string]any) (err error) {
+	for _, m := range row {
+		for k, v := range m {
+			if s, ok := v.(string); ok {
+				if strings.HasPrefix(s, "0x") {
+					if m[k], err = hex.DecodeString(s[2:]); err != nil {
+						return err
+					}
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (f *fixturesLoaderInternal) prepareFixtureItem(fixture *FixtureItem) (err error) {
+	fixture.tableNameQuoted = f.quoteObject(fixture.tableName)
+
+	if f.dbType == schemas.MSSQL {
+		fixture.mssqlHasIdentityColumn, err = f.mssqlTableHasIdentityColumn(f.db, fixture.tableName)
+		if err != nil {
+			return err
+		}
+	}
+
+	data, err := os.ReadFile(fixture.fileFullPath)
+	if err != nil {
+		return fmt.Errorf("failed to read file %q: %w", fixture.fileFullPath, err)
+	}
+
+	var rows []map[string]any
+	if err = yaml.Unmarshal(data, &rows); err != nil {
+		return fmt.Errorf("failed to unmarshal yaml data from %q: %w", fixture.fileFullPath, err)
+	}
+	if err = f.preprocessFixtureRow(rows); err != nil {
+		return fmt.Errorf("failed to preprocess fixture rows from %q: %w", fixture.fileFullPath, err)
+	}
+
+	var sqlBuf []byte
+	var sqlArguments []any
+	for _, row := range rows {
+		sqlBuf = append(sqlBuf, fmt.Sprintf("INSERT INTO %s (", fixture.tableNameQuoted)...)
+		for k, v := range row {
+			sqlBuf = append(sqlBuf, f.quoteObject(k)...)
+			sqlBuf = append(sqlBuf, ","...)
+			sqlArguments = append(sqlArguments, v)
+		}
+		sqlBuf = sqlBuf[:len(sqlBuf)-1]
+		sqlBuf = append(sqlBuf, ") VALUES ("...)
+		paramIdx := 1
+		for range row {
+			sqlBuf = append(sqlBuf, f.paramPlaceholder(paramIdx)...)
+			sqlBuf = append(sqlBuf, ',')
+			paramIdx++
+		}
+		sqlBuf[len(sqlBuf)-1] = ')'
+		fixture.sqlInserts = append(fixture.sqlInserts, string(sqlBuf))
+		fixture.sqlInsertArgs = append(fixture.sqlInsertArgs, slices.Clone(sqlArguments))
+		sqlBuf = sqlBuf[:0]
+		sqlArguments = sqlArguments[:0]
+	}
+	return nil
+}
+
+func (f *fixturesLoaderInternal) loadFixtures(tx *sql.Tx, fixture *FixtureItem) (err error) {
+	if fixture.tableNameQuoted == "" {
+		if err = f.prepareFixtureItem(fixture); err != nil {
+			return err
+		}
+	}
+
+	_, err = tx.Exec("DELETE FROM " + fixture.tableNameQuoted) // sqlite3 doesn't support truncate
+	if err != nil {
+		return err
+	}
+
+	if fixture.mssqlHasIdentityColumn {
+		_, err = tx.Exec(fmt.Sprintf("SET IDENTITY_INSERT %s ON", fixture.tableNameQuoted))
+		if err != nil {
+			return err
+		}
+		defer func() { _, err = tx.Exec(fmt.Sprintf("SET IDENTITY_INSERT %s OFF", fixture.tableNameQuoted)) }()
+	}
+	for i := range fixture.sqlInserts {
+		_, err = tx.Exec(fixture.sqlInserts[i], fixture.sqlInsertArgs[i]...)
+	}
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (f *fixturesLoaderInternal) Load() error {
+	tx, err := f.db.Begin()
+	if err != nil {
+		return err
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	for _, fixture := range f.fixtures {
+		if !f.xormTableNames[fixture.tableName] {
+			continue
+		}
+		if err := f.loadFixtures(tx, fixture); err != nil {
+			return fmt.Errorf("failed to load fixtures from %s: %w", fixture.fileFullPath, err)
+		}
+	}
+	if err = tx.Commit(); err != nil {
+		return err
+	}
+	for xormTableName := range f.xormTableNames {
+		if f.fixtures[xormTableName] == nil {
+			_, _ = f.xormEngine.Exec("DELETE FROM `" + xormTableName + "`")
+		}
+	}
+	return nil
+}
+
+func FixturesFileFullPaths(dir string, files []string) (map[string]*FixtureItem, error) {
+	if files != nil && len(files) == 0 {
+		return nil, nil // load nothing
+	}
+	files = slices.Clone(files)
+	if len(files) == 0 {
+		entries, err := os.ReadDir(dir)
+		if err != nil {
+			return nil, err
+		}
+		for _, e := range entries {
+			files = append(files, e.Name())
+		}
+	}
+	fixtureItems := map[string]*FixtureItem{}
+	for _, file := range files {
+		fileFillPath := file
+		if !filepath.IsAbs(fileFillPath) {
+			fileFillPath = filepath.Join(dir, file)
+		}
+		tableName, _, _ := strings.Cut(filepath.Base(file), ".")
+		fixtureItems[tableName] = &FixtureItem{fileFullPath: fileFillPath, tableName: tableName}
+	}
+	return fixtureItems, nil
+}
+
+func NewFixturesLoader(x *xorm.Engine, opts FixturesOptions) (FixturesLoader, error) {
+	fixtureItems, err := FixturesFileFullPaths(opts.Dir, opts.Files)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get fixtures files: %w", err)
+	}
+
+	f := &fixturesLoaderInternal{xormEngine: x, db: x.DB().DB, dbType: x.Dialect().URI().DBType, fixtures: fixtureItems}
+	switch f.dbType {
+	case schemas.SQLITE:
+		f.quoteObject = func(s string) string { return fmt.Sprintf(`"%s"`, s) }
+		f.paramPlaceholder = func(idx int) string { return "?" }
+	case schemas.POSTGRES:
+		f.quoteObject = func(s string) string { return fmt.Sprintf(`"%s"`, s) }
+		f.paramPlaceholder = func(idx int) string { return fmt.Sprintf(`$%d`, idx) }
+	case schemas.MYSQL:
+		f.quoteObject = func(s string) string { return fmt.Sprintf("`%s`", s) }
+		f.paramPlaceholder = func(idx int) string { return "?" }
+	case schemas.MSSQL:
+		f.quoteObject = func(s string) string { return fmt.Sprintf("[%s]", s) }
+		f.paramPlaceholder = func(idx int) string { return "?" }
+	}
+
+	xormBeans, _ := db.NamesToBean()
+	f.xormTableNames = map[string]bool{}
+	for _, bean := range xormBeans {
+		f.xormTableNames[db.TableName(bean)] = true
+	}
+
+	return f, nil
+}

models/unittest/fixtures_test.go

@@ -0,0 +1,114 @@
+// Copyright 2024 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package unittest_test
+
+import (
+	"path/filepath"
+	"testing"
+
+	"code.gitea.io/gitea/models/unittest"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/test"
+
+	"github.com/stretchr/testify/require"
+	"xorm.io/xorm"
+)
+
+var NewFixturesLoaderVendor = func(e *xorm.Engine, opts unittest.FixturesOptions) (unittest.FixturesLoader, error) {
+	return nil, nil
+}
+
+/*
+// the old code is kept here in case we are still interested in benchmarking the two implementations
+func init() {
+	NewFixturesLoaderVendor = func(e *xorm.Engine, opts unittest.FixturesOptions) (unittest.FixturesLoader, error) {
+		return NewFixturesLoaderVendorGoTestfixtures(e, opts)
+	}
+}
+
+func NewFixturesLoaderVendorGoTestfixtures(e *xorm.Engine, opts unittest.FixturesOptions) (*testfixtures.Loader, error) {
+	files, err := unittest.FixturesFileFullPaths(opts.Dir, opts.Files)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get fixtures files: %w", err)
+	}
+	var dialect string
+	switch e.Dialect().URI().DBType {
+	case schemas.POSTGRES:
+		dialect = "postgres"
+	case schemas.MYSQL:
+		dialect = "mysql"
+	case schemas.MSSQL:
+		dialect = "mssql"
+	case schemas.SQLITE:
+		dialect = "sqlite3"
+	default:
+		return nil, fmt.Errorf("unsupported RDBMS for integration tests: %q", e.Dialect().URI().DBType)
+	}
+	loaderOptions := []func(loader *testfixtures.Loader) error{
+		testfixtures.Database(e.DB().DB),
+		testfixtures.Dialect(dialect),
+		testfixtures.DangerousSkipTestDatabaseCheck(),
+		testfixtures.Files(files...),
+	}
+	if e.Dialect().URI().DBType == schemas.POSTGRES {
+		loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences())
+	}
+	return testfixtures.New(loaderOptions...)
+}
+*/
+
+func prepareTestFixturesLoaders(t testing.TB) unittest.FixturesOptions {
+	_ = user_model.User{}
+	opts := unittest.FixturesOptions{Dir: filepath.Join(test.SetupGiteaRoot(), "models", "fixtures"), Files: []string{
+		"user.yml",
+	}}
+	require.NoError(t, unittest.CreateTestEngine(opts))
+	return opts
+}
+
+func TestFixturesLoader(t *testing.T) {
+	opts := prepareTestFixturesLoaders(t)
+	loaderInternal, err := unittest.NewFixturesLoader(unittest.GetXORMEngine(), opts)
+	require.NoError(t, err)
+	loaderVendor, err := NewFixturesLoaderVendor(unittest.GetXORMEngine(), opts)
+	require.NoError(t, err)
+	t.Run("Internal", func(t *testing.T) {
+		require.NoError(t, loaderInternal.Load())
+		require.NoError(t, loaderInternal.Load())
+	})
+	t.Run("Vendor", func(t *testing.T) {
+		if loaderVendor == nil {
+			t.Skip()
+		}
+		require.NoError(t, loaderVendor.Load())
+		require.NoError(t, loaderVendor.Load())
+	})
+}
+
+func BenchmarkFixturesLoader(b *testing.B) {
+	opts := prepareTestFixturesLoaders(b)
+	require.NoError(b, unittest.CreateTestEngine(opts))
+	loaderInternal, err := unittest.NewFixturesLoader(unittest.GetXORMEngine(), opts)
+	require.NoError(b, err)
+	loaderVendor, err := NewFixturesLoaderVendor(unittest.GetXORMEngine(), opts)
+	require.NoError(b, err)
+
+	// BenchmarkFixturesLoader/Vendor
+	// BenchmarkFixturesLoader/Vendor-12         	    1696	    719416 ns/op
+	// BenchmarkFixturesLoader/Internal
+	// BenchmarkFixturesLoader/Internal-12       	    1746	    670457 ns/op
+	b.Run("Internal", func(b *testing.B) {
+		for b.Loop() {
+			require.NoError(b, loaderInternal.Load())
+		}
+	})
+	b.Run("Vendor", func(b *testing.B) {
+		if loaderVendor == nil {
+			b.Skip()
+		}
+		for b.Loop() {
+			require.NoError(b, loaderVendor.Load())
+		}
+	})
+}

models/unittest/fscopy.go

@@ -0,0 +1,92 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package unittest
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+
+	"code.gitea.io/gitea/modules/util"
+)
+
+// SyncFile synchronizes the two files. This is skipped if both files
+// exist and the size, modtime, and mode match.
+func SyncFile(srcPath, destPath string) error {
+	dest, err := os.Stat(destPath)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return util.CopyFile(srcPath, destPath)
+		}
+		return err
+	}
+
+	src, err := os.Stat(srcPath)
+	if err != nil {
+		return err
+	}
+
+	if src.Size() == dest.Size() &&
+		src.ModTime().Equal(dest.ModTime()) &&
+		src.Mode() == dest.Mode() {
+		return nil
+	}
+
+	return util.CopyFile(srcPath, destPath)
+}
+
+// SyncDirs synchronizes files recursively from source to target directory.
+// It returns error when error occurs in underlying functions.
+func SyncDirs(srcPath, destPath string) error {
+	err := os.MkdirAll(destPath, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	// the keep file is used to keep the directory in a git repository, it doesn't need to be synced
+	// and go-git doesn't work with the ".keep" file (it would report errors like "ref is empty")
+	const keepFile = ".keep"
+
+	// find and delete all untracked files
+	destFiles, err := util.ListDirRecursively(destPath, &util.ListDirOptions{IncludeDir: true})
+	if err != nil {
+		return err
+	}
+	for _, destFile := range destFiles {
+		destFilePath := filepath.Join(destPath, destFile)
+		shouldRemove := filepath.Base(destFilePath) == keepFile
+		if _, err = os.Stat(filepath.Join(srcPath, destFile)); err != nil {
+			if os.IsNotExist(err) {
+				shouldRemove = true
+			} else {
+				return err
+			}
+		}
+		// if src file does not exist, remove dest file
+		if shouldRemove {
+			if err = os.RemoveAll(destFilePath); err != nil {
+				return err
+			}
+		}
+	}
+
+	// sync src files to dest
+	srcFiles, err := util.ListDirRecursively(srcPath, &util.ListDirOptions{IncludeDir: true})
+	if err != nil {
+		return err
+	}
+	for _, srcFile := range srcFiles {
+		destFilePath := filepath.Join(destPath, srcFile)
+		// util.ListDirRecursively appends a slash to the directory name
+		if strings.HasSuffix(srcFile, "/") {
+			err = os.MkdirAll(destFilePath, os.ModePerm)
+		} else if filepath.Base(destFilePath) != keepFile {
+			err = SyncFile(filepath.Join(srcPath, srcFile), destFilePath)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

models/unittest/reflection.go

@@ -0,0 +1,40 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package unittest
+
+import (
+	"fmt"
+	"reflect"
+)
+
+func fieldByName(v reflect.Value, field string) reflect.Value {
+	if v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+	f := v.FieldByName(field)
+	if !f.IsValid() {
+		panic(fmt.Errorf("can not read %s for %v", field, v))
+	}
+	return f
+}
+
+type reflectionValue struct {
+	v reflect.Value
+}
+
+func reflectionWrap(v any) *reflectionValue {
+	return &reflectionValue{v: reflect.ValueOf(v)}
+}
+
+func (rv *reflectionValue) int(field string) int {
+	return int(fieldByName(rv.v, field).Int())
+}
+
+func (rv *reflectionValue) str(field string) string {
+	return fieldByName(rv.v, field).String()
+}
+
+func (rv *reflectionValue) bool(field string) bool {
+	return fieldByName(rv.v, field).Bool()
+}

models/unittest/testdb.go

@@ -0,0 +1,205 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package unittest
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/models/system"
+	"code.gitea.io/gitea/modules/auth/password/hash"
+	"code.gitea.io/gitea/modules/cache"
+	"code.gitea.io/gitea/modules/git"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/setting/config"
+	"code.gitea.io/gitea/modules/storage"
+	"code.gitea.io/gitea/modules/tempdir"
+	"code.gitea.io/gitea/modules/test"
+	"code.gitea.io/gitea/modules/util"
+
+	"github.com/stretchr/testify/assert"
+	"xorm.io/xorm"
+	"xorm.io/xorm/names"
+)
+
+var giteaRoot string
+
+func fatalTestError(fmtStr string, args ...any) {
+	_, _ = fmt.Fprintf(os.Stderr, fmtStr, args...)
+	os.Exit(1)
+}
+
+// InitSettingsForTesting initializes config provider and load common settings for tests
+func InitSettingsForTesting() {
+	setting.IsInTesting = true
+	log.OsExiter = func(code int) {
+		if code != 0 {
+			// non-zero exit code (log.Fatal) shouldn't occur during testing, if it happens, show a full stacktrace for more details
+			panic(fmt.Errorf("non-zero exit code during testing: %d", code))
+		}
+		os.Exit(0)
+	}
+	if setting.CustomConf == "" {
+		setting.CustomConf = filepath.Join(setting.CustomPath, "conf/app-unittest-tmp.ini")
+		_ = os.Remove(setting.CustomConf)
+	}
+	setting.InitCfgProvider(setting.CustomConf)
+	setting.LoadCommonSettings()
+
+	if err := setting.PrepareAppDataPath(); err != nil {
+		log.Fatal("Can not prepare APP_DATA_PATH: %v", err)
+	}
+	// register the dummy hash algorithm function used in the test fixtures
+	_ = hash.Register("dummy", hash.NewDummyHasher)
+
+	setting.PasswordHashAlgo, _ = hash.SetDefaultPasswordHashAlgorithm("dummy")
+	setting.InitGiteaEnvVarsForTesting()
+}
+
+// TestOptions represents test options
+type TestOptions struct {
+	FixtureFiles []string
+	SetUp        func() error // SetUp will be executed before all tests in this package
+	TearDown     func() error // TearDown will be executed after all tests in this package
+}
+
+// MainTest a reusable TestMain(..) function for unit tests that need to use a
+// test database. Creates the test database, and sets necessary settings.
+func MainTest(m *testing.M, testOptsArg ...*TestOptions) {
+	testOpts := util.OptionalArg(testOptsArg, &TestOptions{})
+	giteaRoot = test.SetupGiteaRoot()
+	setting.CustomPath = filepath.Join(giteaRoot, "custom")
+	InitSettingsForTesting()
+
+	fixturesOpts := FixturesOptions{Dir: filepath.Join(giteaRoot, "models", "fixtures"), Files: testOpts.FixtureFiles}
+	if err := CreateTestEngine(fixturesOpts); err != nil {
+		fatalTestError("Error creating test engine: %v\n", err)
+	}
+
+	setting.IsInTesting = true
+	setting.AppURL = "https://try.gitea.io/"
+	setting.Domain = "try.gitea.io"
+	setting.RunUser = "runuser"
+	setting.SSH.User = "sshuser"
+	setting.SSH.BuiltinServerUser = "builtinuser"
+	setting.SSH.Port = 3000
+	setting.SSH.Domain = "try.gitea.io"
+	setting.Database.Type = "sqlite3"
+	setting.Repository.DefaultBranch = "master" // many test code still assume that default branch is called "master"
+	repoRootPath, cleanup1, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("repos")
+	if err != nil {
+		fatalTestError("TempDir: %v\n", err)
+	}
+	defer cleanup1()
+
+	setting.RepoRootPath = repoRootPath
+	appDataPath, cleanup2, err := tempdir.OsTempDir("gitea-test").MkdirTempRandom("appdata")
+	if err != nil {
+		fatalTestError("TempDir: %v\n", err)
+	}
+	defer cleanup2()
+
+	setting.AppDataPath = appDataPath
+	setting.AppWorkPath = giteaRoot
+	setting.StaticRootPath = giteaRoot
+	setting.GravatarSource = "https://secure.gravatar.com/avatar/"
+
+	setting.Attachment.Storage.Path = filepath.Join(setting.AppDataPath, "attachments")
+
+	setting.LFS.Storage.Path = filepath.Join(setting.AppDataPath, "lfs")
+
+	setting.Avatar.Storage.Path = filepath.Join(setting.AppDataPath, "avatars")
+
+	setting.RepoAvatar.Storage.Path = filepath.Join(setting.AppDataPath, "repo-avatars")
+
+	setting.RepoArchive.Storage.Path = filepath.Join(setting.AppDataPath, "repo-archive")
+
+	setting.Packages.Storage.Path = filepath.Join(setting.AppDataPath, "packages")
+
+	setting.Actions.LogStorage.Path = filepath.Join(setting.AppDataPath, "actions_log")
+
+	setting.Git.HomePath = filepath.Join(setting.AppDataPath, "home")
+
+	setting.IncomingEmail.ReplyToAddress = "incoming+%{token}@localhost"
+
+	config.SetDynGetter(system.NewDatabaseDynKeyGetter())
+
+	if err = cache.Init(); err != nil {
+		fatalTestError("cache.Init: %v\n", err)
+	}
+	if err = storage.Init(); err != nil {
+		fatalTestError("storage.Init: %v\n", err)
+	}
+	if err = SyncDirs(filepath.Join(giteaRoot, "tests", "gitea-repositories-meta"), setting.RepoRootPath); err != nil {
+		fatalTestError("util.SyncDirs: %v\n", err)
+	}
+
+	if err = git.InitFull(context.Background()); err != nil {
+		fatalTestError("git.Init: %v\n", err)
+	}
+
+	if testOpts.SetUp != nil {
+		if err := testOpts.SetUp(); err != nil {
+			fatalTestError("set up failed: %v\n", err)
+		}
+	}
+
+	exitStatus := m.Run()
+
+	if testOpts.TearDown != nil {
+		if err := testOpts.TearDown(); err != nil {
+			fatalTestError("tear down failed: %v\n", err)
+		}
+	}
+	os.Exit(exitStatus)
+}
+
+// FixturesOptions fixtures needs to be loaded options
+type FixturesOptions struct {
+	Dir   string
+	Files []string
+}
+
+// CreateTestEngine creates a memory database and loads the fixture data from fixturesDir
+func CreateTestEngine(opts FixturesOptions) error {
+	x, err := xorm.NewEngine("sqlite3", "file::memory:?cache=shared&_txlock=immediate")
+	if err != nil {
+		if strings.Contains(err.Error(), "unknown driver") {
+			return fmt.Errorf(`sqlite3 requires: -tags sqlite,sqlite_unlock_notify%s%w`, "\n", err)
+		}
+		return err
+	}
+	x.SetMapper(names.GonicMapper{})
+	db.SetDefaultEngine(context.Background(), x)
+
+	if err = db.SyncAllTables(); err != nil {
+		return err
+	}
+	switch os.Getenv("GITEA_UNIT_TESTS_LOG_SQL") {
+	case "true", "1":
+		x.ShowSQL(true)
+	}
+
+	return InitFixtures(opts)
+}
+
+// PrepareTestDatabase load test fixtures into test database
+func PrepareTestDatabase() error {
+	return LoadFixtures()
+}
+
+// PrepareTestEnv prepares the environment for unit tests. Can only be called
+// by tests that use the above MainTest(..) function.
+func PrepareTestEnv(t testing.TB) {
+	assert.NoError(t, PrepareTestDatabase())
+	metaPath := filepath.Join(giteaRoot, "tests", "gitea-repositories-meta")
+	assert.NoError(t, SyncDirs(metaPath, setting.RepoRootPath))
+	test.SetupGiteaRoot() // Makes sure GITEA_ROOT is set
+}

models/unittest/unit_tests.go

@@ -0,0 +1,184 @@
+// Copyright 2016 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package unittest
+
+import (
+	"fmt"
+	"math"
+	"os"
+	"strings"
+
+	"code.gitea.io/gitea/models/db"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"xorm.io/builder"
+	"xorm.io/xorm"
+)
+
+// Code in this file is mainly used by unittest.CheckConsistencyFor, which is not in the unit test for various reasons.
+// In the future if we can decouple CheckConsistencyFor into separate unit test code, then this file can be moved into unittest package too.
+
+// NonexistentID an ID that will never exist
+const NonexistentID = int64(math.MaxInt64)
+
+type testCond struct {
+	query any
+	args  []any
+}
+
+type testOrderBy string
+
+// Cond create a condition with arguments for a test
+func Cond(query any, args ...any) any {
+	return &testCond{query: query, args: args}
+}
+
+// OrderBy creates "ORDER BY" a test query
+func OrderBy(orderBy string) any {
+	return testOrderBy(orderBy)
+}
+
+func whereOrderConditions(e db.Engine, conditions []any) db.Engine {
+	orderBy := "id" // query must have the "ORDER BY", otherwise the result is not deterministic
+	for _, condition := range conditions {
+		switch cond := condition.(type) {
+		case *testCond:
+			e = e.Where(cond.query, cond.args...)
+		case testOrderBy:
+			orderBy = string(cond)
+		default:
+			e = e.Where(cond)
+		}
+	}
+	return e.OrderBy(orderBy)
+}
+
+func getBeanIfExists(bean any, conditions ...any) (bool, error) {
+	e := db.GetEngine(db.DefaultContext)
+	return whereOrderConditions(e, conditions).Get(bean)
+}
+
+func GetBean[T any](t require.TestingT, bean T, conditions ...any) (ret T) {
+	exists, err := getBeanIfExists(bean, conditions...)
+	require.NoError(t, err)
+	if exists {
+		return bean
+	}
+	return ret
+}
+
+// AssertExistsAndLoadBean assert that a bean exists and load it from the test database
+func AssertExistsAndLoadBean[T any](t require.TestingT, bean T, conditions ...any) T {
+	exists, err := getBeanIfExists(bean, conditions...)
+	require.NoError(t, err)
+	require.True(t, exists,
+		"Expected to find %+v (of type %T, with conditions %+v), but did not",
+		bean, bean, conditions)
+	return bean
+}
+
+// AssertExistsAndLoadMap assert that a row exists and load it from the test database
+func AssertExistsAndLoadMap(t assert.TestingT, table string, conditions ...any) map[string]string {
+	e := db.GetEngine(db.DefaultContext).Table(table)
+	res, err := whereOrderConditions(e, conditions).Query()
+	assert.NoError(t, err)
+	assert.Len(t, res, 1,
+		"Expected to find one row in %s (with conditions %+v), but found %d",
+		table, conditions, len(res),
+	)
+
+	if len(res) == 1 {
+		rec := map[string]string{}
+		for k, v := range res[0] {
+			rec[k] = string(v)
+		}
+		return rec
+	}
+	return nil
+}
+
+// GetCount get the count of a bean
+func GetCount(t assert.TestingT, bean any, conditions ...any) int {
+	e := db.GetEngine(db.DefaultContext)
+	for _, condition := range conditions {
+		switch cond := condition.(type) {
+		case *testCond:
+			e = e.Where(cond.query, cond.args...)
+		default:
+			e = e.Where(cond)
+		}
+	}
+	count, err := e.Count(bean)
+	assert.NoError(t, err)
+	return int(count)
+}
+
+// AssertNotExistsBean assert that a bean does not exist in the test database
+func AssertNotExistsBean(t assert.TestingT, bean any, conditions ...any) {
+	exists, err := getBeanIfExists(bean, conditions...)
+	assert.NoError(t, err)
+	assert.False(t, exists)
+}
+
+// AssertCount assert the count of a bean
+func AssertCount(t assert.TestingT, bean, expected any) bool {
+	return assert.EqualValues(t, expected, GetCount(t, bean))
+}
+
+// AssertInt64InRange assert value is in range [low, high]
+func AssertInt64InRange(t assert.TestingT, low, high, value int64) {
+	assert.True(t, value >= low && value <= high,
+		"Expected value in range [%d, %d], found %d", low, high, value)
+}
+
+// GetCountByCond get the count of database entries matching bean
+func GetCountByCond(t assert.TestingT, tableName string, cond builder.Cond) int64 {
+	e := db.GetEngine(db.DefaultContext)
+	count, err := e.Table(tableName).Where(cond).Count()
+	assert.NoError(t, err)
+	return count
+}
+
+// AssertCountByCond test the count of database entries matching bean
+func AssertCountByCond(t assert.TestingT, tableName string, cond builder.Cond, expected int) bool {
+	return assert.EqualValues(t, expected, GetCountByCond(t, tableName, cond),
+		"Failed consistency test, the counted bean (of table %s) was %+v", tableName, cond)
+}
+
+// DumpQueryResult dumps the result of a query for debugging purpose
+func DumpQueryResult(t require.TestingT, sqlOrBean any, sqlArgs ...any) {
+	x := db.GetEngine(db.DefaultContext).(*xorm.Engine)
+	goDB := x.DB().DB
+	sql, ok := sqlOrBean.(string)
+	if !ok {
+		sql = "SELECT * FROM " + db.TableName(sqlOrBean)
+	} else if !strings.Contains(sql, " ") {
+		sql = "SELECT * FROM " + sql
+	}
+	rows, err := goDB.Query(sql, sqlArgs...)
+	require.NoError(t, err)
+	defer rows.Close()
+	columns, err := rows.Columns()
+	require.NoError(t, err)
+
+	_, _ = fmt.Fprintf(os.Stdout, "====== DumpQueryResult: %s ======\n", sql)
+	idx := 0
+	for rows.Next() {
+		row := make([]any, len(columns))
+		rowPointers := make([]any, len(columns))
+		for i := range row {
+			rowPointers[i] = &row[i]
+		}
+		require.NoError(t, rows.Scan(rowPointers...))
+		_, _ = fmt.Fprintf(os.Stdout, "- # row[%d]\n", idx)
+		for i, col := range columns {
+			_, _ = fmt.Fprintf(os.Stdout, "  %s: %v\n", col, row[i])
+		}
+		idx++
+	}
+	if idx == 0 {
+		_, _ = fmt.Fprintf(os.Stdout, "(no result, columns: %s)\n", strings.Join(columns, ", "))
+	}
+}