summaryrefslogtreecommitdiffstats
path: root/cmd/migrate_storage.go
diff options
context:
space:
mode:
Diffstat (limited to 'cmd/migrate_storage.go')
-rw-r--r--cmd/migrate_storage.go267
1 files changed, 267 insertions, 0 deletions
diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go
new file mode 100644
index 00000000..3a69b555
--- /dev/null
+++ b/cmd/migrate_storage.go
@@ -0,0 +1,267 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io/fs"
+ "strings"
+
+ actions_model "code.gitea.io/gitea/models/actions"
+ "code.gitea.io/gitea/models/db"
+ git_model "code.gitea.io/gitea/models/git"
+ "code.gitea.io/gitea/models/migrations"
+ packages_model "code.gitea.io/gitea/models/packages"
+ repo_model "code.gitea.io/gitea/models/repo"
+ user_model "code.gitea.io/gitea/models/user"
+ "code.gitea.io/gitea/modules/log"
+ packages_module "code.gitea.io/gitea/modules/packages"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/storage"
+
+ "github.com/urfave/cli/v2"
+)
+
+// CmdMigrateStorage represents the available migrate storage sub-command.
+var CmdMigrateStorage = &cli.Command{
+ Name: "migrate-storage",
+ Usage: "Migrate the storage",
+ Description: "Copies stored files from storage configured in app.ini to parameter-configured storage",
+ Action: runMigrateStorage,
+ Flags: []cli.Flag{
+ &cli.StringFlag{
+ Name: "type",
+ Aliases: []string{"t"},
+ Value: "",
+ Usage: "Type of stored files to copy. Allowed types: 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages', 'actions-log', 'actions-artifacts'",
+ },
+ &cli.StringFlag{
+ Name: "storage",
+ Aliases: []string{"s"},
+ Value: "",
+ Usage: "New storage type: local (default) or minio",
+ },
+ &cli.StringFlag{
+ Name: "path",
+ Aliases: []string{"p"},
+ Value: "",
+ Usage: "New storage placement if store is local (leave blank for default)",
+ },
+ &cli.StringFlag{
+ Name: "minio-endpoint",
+ Value: "",
+ Usage: "Minio storage endpoint",
+ },
+ &cli.StringFlag{
+ Name: "minio-access-key-id",
+ Value: "",
+ Usage: "Minio storage accessKeyID",
+ },
+ &cli.StringFlag{
+ Name: "minio-secret-access-key",
+ Value: "",
+ Usage: "Minio storage secretAccessKey",
+ },
+ &cli.StringFlag{
+ Name: "minio-bucket",
+ Value: "",
+ Usage: "Minio storage bucket",
+ },
+ &cli.StringFlag{
+ Name: "minio-location",
+ Value: "",
+ Usage: "Minio storage location to create bucket",
+ },
+ &cli.StringFlag{
+ Name: "minio-base-path",
+ Value: "",
+ Usage: "Minio storage base path on the bucket",
+ },
+ &cli.BoolFlag{
+ Name: "minio-use-ssl",
+ Usage: "Enable SSL for minio",
+ },
+ &cli.BoolFlag{
+ Name: "minio-insecure-skip-verify",
+ Usage: "Skip SSL verification",
+ },
+ &cli.StringFlag{
+ Name: "minio-checksum-algorithm",
+ Value: "",
+ Usage: "Minio checksum algorithm (default/md5)",
+ },
+ },
+}
+
+func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error {
+ return db.Iterate(ctx, nil, func(ctx context.Context, attach *repo_model.Attachment) error {
+ _, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
+ return err
+ })
+}
+
+func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error {
+ return db.Iterate(ctx, nil, func(ctx context.Context, mo *git_model.LFSMetaObject) error {
+ _, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
+ return err
+ })
+}
+
+func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
+ return db.Iterate(ctx, nil, func(ctx context.Context, user *user_model.User) error {
+ if user.CustomAvatarRelativePath() == "" {
+ return nil
+ }
+ _, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
+ return err
+ })
+}
+
+func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
+ return db.Iterate(ctx, nil, func(ctx context.Context, repo *repo_model.Repository) error {
+ if repo.CustomAvatarRelativePath() == "" {
+ return nil
+ }
+ _, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
+ return err
+ })
+}
+
+func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error {
+ return db.Iterate(ctx, nil, func(ctx context.Context, archiver *repo_model.RepoArchiver) error {
+ p := archiver.RelativePath()
+ _, err := storage.Copy(dstStorage, p, storage.RepoArchives, p)
+ return err
+ })
+}
+
+func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error {
+ return db.Iterate(ctx, nil, func(ctx context.Context, pb *packages_model.PackageBlob) error {
+ p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256))
+ _, err := storage.Copy(dstStorage, p, storage.Packages, p)
+ return err
+ })
+}
+
+func migrateActionsLog(ctx context.Context, dstStorage storage.ObjectStorage) error {
+ return db.Iterate(ctx, nil, func(ctx context.Context, task *actions_model.ActionTask) error {
+ if task.LogExpired {
+ // the log has been cleared
+ return nil
+ }
+ if !task.LogInStorage {
+ // running tasks store logs in DBFS
+ return nil
+ }
+ p := task.LogFilename
+ _, err := storage.Copy(dstStorage, p, storage.Actions, p)
+ return err
+ })
+}
+
+func migrateActionsArtifacts(ctx context.Context, dstStorage storage.ObjectStorage) error {
+ return db.Iterate(ctx, nil, func(ctx context.Context, artifact *actions_model.ActionArtifact) error {
+ if artifact.Status == int64(actions_model.ArtifactStatusExpired) {
+ return nil
+ }
+
+ _, err := storage.Copy(dstStorage, artifact.StoragePath, storage.ActionsArtifacts, artifact.StoragePath)
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ log.Warn("ignored: actions artifact %s exists in the database but not in storage", artifact.StoragePath)
+ return nil
+ }
+ return err
+ }
+
+ return nil
+ })
+}
+
+func runMigrateStorage(ctx *cli.Context) error {
+ stdCtx, cancel := installSignals()
+ defer cancel()
+
+ if err := initDB(stdCtx); err != nil {
+ return err
+ }
+
+ log.Info("AppPath: %s", setting.AppPath)
+ log.Info("AppWorkPath: %s", setting.AppWorkPath)
+ log.Info("Custom path: %s", setting.CustomPath)
+ log.Info("Log path: %s", setting.Log.RootPath)
+ log.Info("Configuration file: %s", setting.CustomConf)
+
+ if err := db.InitEngineWithMigration(context.Background(), migrations.Migrate); err != nil {
+ log.Fatal("Failed to initialize ORM engine: %v", err)
+ return err
+ }
+
+ if err := storage.Init(); err != nil {
+ return err
+ }
+
+ var dstStorage storage.ObjectStorage
+ var err error
+ switch strings.ToLower(ctx.String("storage")) {
+ case "":
+ fallthrough
+ case string(setting.LocalStorageType):
+ p := ctx.String("path")
+ if p == "" {
+ log.Fatal("Path must be given when storage is local")
+ return nil
+ }
+ dstStorage, err = storage.NewLocalStorage(
+ stdCtx,
+ &setting.Storage{
+ Path: p,
+ })
+ case string(setting.MinioStorageType):
+ dstStorage, err = storage.NewMinioStorage(
+ stdCtx,
+ &setting.Storage{
+ MinioConfig: setting.MinioStorageConfig{
+ Endpoint: ctx.String("minio-endpoint"),
+ AccessKeyID: ctx.String("minio-access-key-id"),
+ SecretAccessKey: ctx.String("minio-secret-access-key"),
+ Bucket: ctx.String("minio-bucket"),
+ Location: ctx.String("minio-location"),
+ BasePath: ctx.String("minio-base-path"),
+ UseSSL: ctx.Bool("minio-use-ssl"),
+ InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"),
+ ChecksumAlgorithm: ctx.String("minio-checksum-algorithm"),
+ },
+ })
+ default:
+ return fmt.Errorf("unsupported storage type: %s", ctx.String("storage"))
+ }
+ if err != nil {
+ return err
+ }
+
+ migratedMethods := map[string]func(context.Context, storage.ObjectStorage) error{
+ "attachments": migrateAttachments,
+ "lfs": migrateLFS,
+ "avatars": migrateAvatars,
+ "repo-avatars": migrateRepoAvatars,
+ "repo-archivers": migrateRepoArchivers,
+ "packages": migratePackages,
+ "actions-log": migrateActionsLog,
+ "actions-artifacts": migrateActionsArtifacts,
+ }
+
+ tp := strings.ToLower(ctx.String("type"))
+ if m, ok := migratedMethods[tp]; ok {
+ if err := m(stdCtx, dstStorage); err != nil {
+ return err
+ }
+ log.Info("%s files have successfully been copied to the new storage.", tp)
+ return nil
+ }
+
+ return fmt.Errorf("unsupported storage: %s", ctx.String("type"))
+}
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-27 14:49:21 +0000