first-commit
This commit is contained in:
309
cmd/migrate_storage.go
Normal file
309
cmd/migrate_storage.go
Normal file
@@ -0,0 +1,309 @@
|
||||
// Copyright 2020 The Gitea Authors. All rights reserved.
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"strings"
|
||||
|
||||
actions_model "code.gitea.io/gitea/models/actions"
|
||||
"code.gitea.io/gitea/models/db"
|
||||
git_model "code.gitea.io/gitea/models/git"
|
||||
packages_model "code.gitea.io/gitea/models/packages"
|
||||
repo_model "code.gitea.io/gitea/models/repo"
|
||||
user_model "code.gitea.io/gitea/models/user"
|
||||
"code.gitea.io/gitea/modules/log"
|
||||
packages_module "code.gitea.io/gitea/modules/packages"
|
||||
"code.gitea.io/gitea/modules/setting"
|
||||
"code.gitea.io/gitea/modules/storage"
|
||||
"code.gitea.io/gitea/services/versioned_migration"
|
||||
|
||||
"github.com/urfave/cli/v3"
|
||||
)
|
||||
|
||||
// CmdMigrateStorage represents the available migrate storage sub-command.
|
||||
var CmdMigrateStorage = &cli.Command{
|
||||
Name: "migrate-storage",
|
||||
Usage: "Migrate the storage",
|
||||
Description: "Copies stored files from storage configured in app.ini to parameter-configured storage",
|
||||
Action: runMigrateStorage,
|
||||
Flags: []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "type",
|
||||
Aliases: []string{"t"},
|
||||
Value: "",
|
||||
Usage: "Type of stored files to copy. Allowed types: 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages', 'actions-log', 'actions-artifacts",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "storage",
|
||||
Aliases: []string{"s"},
|
||||
Value: "",
|
||||
Usage: "New storage type: local (default), minio or azureblob",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "path",
|
||||
Aliases: []string{"p"},
|
||||
Value: "",
|
||||
Usage: "New storage placement if store is local (leave blank for default)",
|
||||
},
|
||||
// Minio Storage special configurations
|
||||
&cli.StringFlag{
|
||||
Name: "minio-endpoint",
|
||||
Value: "",
|
||||
Usage: "Minio storage endpoint",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "minio-access-key-id",
|
||||
Value: "",
|
||||
Usage: "Minio storage accessKeyID",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "minio-secret-access-key",
|
||||
Value: "",
|
||||
Usage: "Minio storage secretAccessKey",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "minio-bucket",
|
||||
Value: "",
|
||||
Usage: "Minio storage bucket",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "minio-location",
|
||||
Value: "",
|
||||
Usage: "Minio storage location to create bucket",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "minio-base-path",
|
||||
Value: "",
|
||||
Usage: "Minio storage base path on the bucket",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "minio-use-ssl",
|
||||
Usage: "Enable SSL for minio",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "minio-insecure-skip-verify",
|
||||
Usage: "Skip SSL verification",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "minio-checksum-algorithm",
|
||||
Value: "",
|
||||
Usage: "Minio checksum algorithm (default/md5)",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "minio-bucket-lookup-type",
|
||||
Value: "",
|
||||
Usage: "Minio bucket lookup type",
|
||||
},
|
||||
// Azure Blob Storage special configurations
|
||||
&cli.StringFlag{
|
||||
Name: "azureblob-endpoint",
|
||||
Value: "",
|
||||
Usage: "Azure Blob storage endpoint",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "azureblob-account-name",
|
||||
Value: "",
|
||||
Usage: "Azure Blob storage account name",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "azureblob-account-key",
|
||||
Value: "",
|
||||
Usage: "Azure Blob storage account key",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "azureblob-container",
|
||||
Value: "",
|
||||
Usage: "Azure Blob storage container",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "azureblob-base-path",
|
||||
Value: "",
|
||||
Usage: "Azure Blob storage base path",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
||||
return db.Iterate(ctx, nil, func(ctx context.Context, attach *repo_model.Attachment) error {
|
||||
_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
||||
return db.Iterate(ctx, nil, func(ctx context.Context, mo *git_model.LFSMetaObject) error {
|
||||
_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
||||
return db.Iterate(ctx, nil, func(ctx context.Context, user *user_model.User) error {
|
||||
if user.CustomAvatarRelativePath() == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
||||
return db.Iterate(ctx, nil, func(ctx context.Context, repo *repo_model.Repository) error {
|
||||
if repo.CustomAvatarRelativePath() == "" {
|
||||
return nil
|
||||
}
|
||||
_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
||||
return db.Iterate(ctx, nil, func(ctx context.Context, archiver *repo_model.RepoArchiver) error {
|
||||
p := archiver.RelativePath()
|
||||
_, err := storage.Copy(dstStorage, p, storage.RepoArchives, p)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
||||
return db.Iterate(ctx, nil, func(ctx context.Context, pb *packages_model.PackageBlob) error {
|
||||
p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256))
|
||||
_, err := storage.Copy(dstStorage, p, storage.Packages, p)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func migrateActionsLog(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
||||
return db.Iterate(ctx, nil, func(ctx context.Context, task *actions_model.ActionTask) error {
|
||||
if task.LogExpired {
|
||||
// the log has been cleared
|
||||
return nil
|
||||
}
|
||||
if !task.LogInStorage {
|
||||
// running tasks store logs in DBFS
|
||||
return nil
|
||||
}
|
||||
p := task.LogFilename
|
||||
_, err := storage.Copy(dstStorage, p, storage.Actions, p)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func migrateActionsArtifacts(ctx context.Context, dstStorage storage.ObjectStorage) error {
|
||||
return db.Iterate(ctx, nil, func(ctx context.Context, artifact *actions_model.ActionArtifact) error {
|
||||
if artifact.Status == actions_model.ArtifactStatusExpired {
|
||||
return nil
|
||||
}
|
||||
|
||||
_, err := storage.Copy(dstStorage, artifact.StoragePath, storage.ActionsArtifacts, artifact.StoragePath)
|
||||
if err != nil {
|
||||
// ignore files that do not exist
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func runMigrateStorage(ctx context.Context, cmd *cli.Command) error {
|
||||
if err := initDB(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Info("AppPath: %s", setting.AppPath)
|
||||
log.Info("AppWorkPath: %s", setting.AppWorkPath)
|
||||
log.Info("Custom path: %s", setting.CustomPath)
|
||||
log.Info("Log path: %s", setting.Log.RootPath)
|
||||
log.Info("Configuration file: %s", setting.CustomConf)
|
||||
|
||||
if err := db.InitEngineWithMigration(context.Background(), versioned_migration.Migrate); err != nil {
|
||||
log.Fatal("Failed to initialize ORM engine: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
if err := storage.Init(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var dstStorage storage.ObjectStorage
|
||||
var err error
|
||||
switch strings.ToLower(cmd.String("storage")) {
|
||||
case "":
|
||||
fallthrough
|
||||
case string(setting.LocalStorageType):
|
||||
p := cmd.String("path")
|
||||
if p == "" {
|
||||
log.Fatal("Path must be given when storage is local")
|
||||
return nil
|
||||
}
|
||||
dstStorage, err = storage.NewLocalStorage(
|
||||
ctx,
|
||||
&setting.Storage{
|
||||
Path: p,
|
||||
})
|
||||
case string(setting.MinioStorageType):
|
||||
dstStorage, err = storage.NewMinioStorage(
|
||||
ctx,
|
||||
&setting.Storage{
|
||||
MinioConfig: setting.MinioStorageConfig{
|
||||
Endpoint: cmd.String("minio-endpoint"),
|
||||
AccessKeyID: cmd.String("minio-access-key-id"),
|
||||
SecretAccessKey: cmd.String("minio-secret-access-key"),
|
||||
Bucket: cmd.String("minio-bucket"),
|
||||
Location: cmd.String("minio-location"),
|
||||
BasePath: cmd.String("minio-base-path"),
|
||||
UseSSL: cmd.Bool("minio-use-ssl"),
|
||||
InsecureSkipVerify: cmd.Bool("minio-insecure-skip-verify"),
|
||||
ChecksumAlgorithm: cmd.String("minio-checksum-algorithm"),
|
||||
BucketLookUpType: cmd.String("minio-bucket-lookup-type"),
|
||||
},
|
||||
})
|
||||
case string(setting.AzureBlobStorageType):
|
||||
dstStorage, err = storage.NewAzureBlobStorage(
|
||||
ctx,
|
||||
&setting.Storage{
|
||||
AzureBlobConfig: setting.AzureBlobStorageConfig{
|
||||
Endpoint: cmd.String("azureblob-endpoint"),
|
||||
AccountName: cmd.String("azureblob-account-name"),
|
||||
AccountKey: cmd.String("azureblob-account-key"),
|
||||
Container: cmd.String("azureblob-container"),
|
||||
BasePath: cmd.String("azureblob-base-path"),
|
||||
},
|
||||
})
|
||||
default:
|
||||
return fmt.Errorf("unsupported storage type: %s", cmd.String("storage"))
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
migratedMethods := map[string]func(context.Context, storage.ObjectStorage) error{
|
||||
"attachments": migrateAttachments,
|
||||
"lfs": migrateLFS,
|
||||
"avatars": migrateAvatars,
|
||||
"repo-avatars": migrateRepoAvatars,
|
||||
"repo-archivers": migrateRepoArchivers,
|
||||
"packages": migratePackages,
|
||||
"actions-log": migrateActionsLog,
|
||||
"actions-artifacts": migrateActionsArtifacts,
|
||||
}
|
||||
|
||||
tp := strings.ToLower(cmd.String("type"))
|
||||
if m, ok := migratedMethods[tp]; ok {
|
||||
if err := m(ctx, dstStorage); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Info("%s files have successfully been copied to the new storage.", tp)
|
||||
return nil
|
||||
}
|
||||
|
||||
return fmt.Errorf("unsupported storage: %s", cmd.String("type"))
|
||||
}
|
||||
Reference in New Issue
Block a user