Adding upstream version 8.0.3.HEADupstream/8.0.3upstreamdebian

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 267 insertions, 0 deletions

diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go
new file mode 100644
index 00000000..3a69b555
--- /dev/null
+++ b/cmd/migrate_storage.go
@@ -0,0 +1,267 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io/fs"
+	"strings"
+
+	actions_model "code.gitea.io/gitea/models/actions"
+	"code.gitea.io/gitea/models/db"
+	git_model "code.gitea.io/gitea/models/git"
+	"code.gitea.io/gitea/models/migrations"
+	packages_model "code.gitea.io/gitea/models/packages"
+	repo_model "code.gitea.io/gitea/models/repo"
+	user_model "code.gitea.io/gitea/models/user"
+	"code.gitea.io/gitea/modules/log"
+	packages_module "code.gitea.io/gitea/modules/packages"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
+
+	"github.com/urfave/cli/v2"
+)
+
+// CmdMigrateStorage represents the available migrate storage sub-command.
+var CmdMigrateStorage = &cli.Command{
+	Name:        "migrate-storage",
+	Usage:       "Migrate the storage",
+	Description: "Copies stored files from storage configured in app.ini to parameter-configured storage",
+	Action:      runMigrateStorage,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:    "type",
+			Aliases: []string{"t"},
+			Value:   "",
+			Usage:   "Type of stored files to copy.  Allowed types: 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages', 'actions-log', 'actions-artifacts'",
+		},
+		&cli.StringFlag{
+			Name:    "storage",
+			Aliases: []string{"s"},
+			Value:   "",
+			Usage:   "New storage type: local (default) or minio",
+		},
+		&cli.StringFlag{
+			Name:    "path",
+			Aliases: []string{"p"},
+			Value:   "",
+			Usage:   "New storage placement if store is local (leave blank for default)",
+		},
+		&cli.StringFlag{
+			Name:  "minio-endpoint",
+			Value: "",
+			Usage: "Minio storage endpoint",
+		},
+		&cli.StringFlag{
+			Name:  "minio-access-key-id",
+			Value: "",
+			Usage: "Minio storage accessKeyID",
+		},
+		&cli.StringFlag{
+			Name:  "minio-secret-access-key",
+			Value: "",
+			Usage: "Minio storage secretAccessKey",
+		},
+		&cli.StringFlag{
+			Name:  "minio-bucket",
+			Value: "",
+			Usage: "Minio storage bucket",
+		},
+		&cli.StringFlag{
+			Name:  "minio-location",
+			Value: "",
+			Usage: "Minio storage location to create bucket",
+		},
+		&cli.StringFlag{
+			Name:  "minio-base-path",
+			Value: "",
+			Usage: "Minio storage base path on the bucket",
+		},
+		&cli.BoolFlag{
+			Name:  "minio-use-ssl",
+			Usage: "Enable SSL for minio",
+		},
+		&cli.BoolFlag{
+			Name:  "minio-insecure-skip-verify",
+			Usage: "Skip SSL verification",
+		},
+		&cli.StringFlag{
+			Name:  "minio-checksum-algorithm",
+			Value: "",
+			Usage: "Minio checksum algorithm (default/md5)",
+		},
+	},
+}
+
+func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error {
+	return db.Iterate(ctx, nil, func(ctx context.Context, attach *repo_model.Attachment) error {
+		_, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath())
+		return err
+	})
+}
+
+func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error {
+	return db.Iterate(ctx, nil, func(ctx context.Context, mo *git_model.LFSMetaObject) error {
+		_, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath())
+		return err
+	})
+}
+
+func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
+	return db.Iterate(ctx, nil, func(ctx context.Context, user *user_model.User) error {
+		if user.CustomAvatarRelativePath() == "" {
+			return nil
+		}
+		_, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath())
+		return err
+	})
+}
+
+func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error {
+	return db.Iterate(ctx, nil, func(ctx context.Context, repo *repo_model.Repository) error {
+		if repo.CustomAvatarRelativePath() == "" {
+			return nil
+		}
+		_, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath())
+		return err
+	})
+}
+
+func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error {
+	return db.Iterate(ctx, nil, func(ctx context.Context, archiver *repo_model.RepoArchiver) error {
+		p := archiver.RelativePath()
+		_, err := storage.Copy(dstStorage, p, storage.RepoArchives, p)
+		return err
+	})
+}
+
+func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error {
+	return db.Iterate(ctx, nil, func(ctx context.Context, pb *packages_model.PackageBlob) error {
+		p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256))
+		_, err := storage.Copy(dstStorage, p, storage.Packages, p)
+		return err
+	})
+}
+
+func migrateActionsLog(ctx context.Context, dstStorage storage.ObjectStorage) error {
+	return db.Iterate(ctx, nil, func(ctx context.Context, task *actions_model.ActionTask) error {
+		if task.LogExpired {
+			// the log has been cleared
+			return nil
+		}
+		if !task.LogInStorage {
+			// running tasks store logs in DBFS
+			return nil
+		}
+		p := task.LogFilename
+		_, err := storage.Copy(dstStorage, p, storage.Actions, p)
+		return err
+	})
+}
+
+func migrateActionsArtifacts(ctx context.Context, dstStorage storage.ObjectStorage) error {
+	return db.Iterate(ctx, nil, func(ctx context.Context, artifact *actions_model.ActionArtifact) error {
+		if artifact.Status == int64(actions_model.ArtifactStatusExpired) {
+			return nil
+		}
+
+		_, err := storage.Copy(dstStorage, artifact.StoragePath, storage.ActionsArtifacts, artifact.StoragePath)
+		if err != nil {
+			if errors.Is(err, fs.ErrNotExist) {
+				log.Warn("ignored: actions artifact %s exists in the database but not in storage", artifact.StoragePath)
+				return nil
+			}
+			return err
+		}
+
+		return nil
+	})
+}
+
+func runMigrateStorage(ctx *cli.Context) error {
+	stdCtx, cancel := installSignals()
+	defer cancel()
+
+	if err := initDB(stdCtx); err != nil {
+		return err
+	}
+
+	log.Info("AppPath: %s", setting.AppPath)
+	log.Info("AppWorkPath: %s", setting.AppWorkPath)
+	log.Info("Custom path: %s", setting.CustomPath)
+	log.Info("Log path: %s", setting.Log.RootPath)
+	log.Info("Configuration file: %s", setting.CustomConf)
+
+	if err := db.InitEngineWithMigration(context.Background(), migrations.Migrate); err != nil {
+		log.Fatal("Failed to initialize ORM engine: %v", err)
+		return err
+	}
+
+	if err := storage.Init(); err != nil {
+		return err
+	}
+
+	var dstStorage storage.ObjectStorage
+	var err error
+	switch strings.ToLower(ctx.String("storage")) {
+	case "":
+		fallthrough
+	case string(setting.LocalStorageType):
+		p := ctx.String("path")
+		if p == "" {
+			log.Fatal("Path must be given when storage is local")
+			return nil
+		}
+		dstStorage, err = storage.NewLocalStorage(
+			stdCtx,
+			&setting.Storage{
+				Path: p,
+			})
+	case string(setting.MinioStorageType):
+		dstStorage, err = storage.NewMinioStorage(
+			stdCtx,
+			&setting.Storage{
+				MinioConfig: setting.MinioStorageConfig{
+					Endpoint:           ctx.String("minio-endpoint"),
+					AccessKeyID:        ctx.String("minio-access-key-id"),
+					SecretAccessKey:    ctx.String("minio-secret-access-key"),
+					Bucket:             ctx.String("minio-bucket"),
+					Location:           ctx.String("minio-location"),
+					BasePath:           ctx.String("minio-base-path"),
+					UseSSL:             ctx.Bool("minio-use-ssl"),
+					InsecureSkipVerify: ctx.Bool("minio-insecure-skip-verify"),
+					ChecksumAlgorithm:  ctx.String("minio-checksum-algorithm"),
+				},
+			})
+	default:
+		return fmt.Errorf("unsupported storage type: %s", ctx.String("storage"))
+	}
+	if err != nil {
+		return err
+	}
+
+	migratedMethods := map[string]func(context.Context, storage.ObjectStorage) error{
+		"attachments":       migrateAttachments,
+		"lfs":               migrateLFS,
+		"avatars":           migrateAvatars,
+		"repo-avatars":      migrateRepoAvatars,
+		"repo-archivers":    migrateRepoArchivers,
+		"packages":          migratePackages,
+		"actions-log":       migrateActionsLog,
+		"actions-artifacts": migrateActionsArtifacts,
+	}
+
+	tp := strings.ToLower(ctx.String("type"))
+	if m, ok := migratedMethods[tp]; ok {
+		if err := m(stdCtx, dstStorage); err != nil {
+			return err
+		}
+		log.Info("%s files have successfully been copied to the new storage.", tp)
+		return nil
+	}
+
+	return fmt.Errorf("unsupported storage: %s", ctx.String("type"))
+}