From e283d58a2f20124836e78603c1f4b04c42ae97f0 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 11 Aug 2022 10:21:43 +0800 Subject: [PATCH 01/11] Add migrate repo archiver and packages storage support on command line --- cmd/migrate_storage.go | 63 +++++++++++++++++++++---------- models/db/iterate.go | 34 +++++++++++++++++ models/git/lfs.go | 23 ----------- models/repo/attachment.go | 22 ----------- models/repo/repo_list.go | 24 ------------ models/user/search.go | 23 ----------- modules/packages/content_store.go | 10 ++--- routers/private/mail.go | 3 +- services/repository/avatar.go | 2 +- 9 files changed, 85 insertions(+), 119 deletions(-) create mode 100644 models/db/iterate.go diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index 93fb64a4d3a58..570f284d2bd4f 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -12,9 +12,11 @@ import ( "code.gitea.io/gitea/models/db" git_model "code.gitea.io/gitea/models/git" "code.gitea.io/gitea/models/migrations" + packages_model "code.gitea.io/gitea/models/packages" repo_model "code.gitea.io/gitea/models/repo" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/log" + packages_module "code.gitea.io/gitea/modules/packages" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" @@ -81,33 +83,58 @@ var CmdMigrateStorage = cli.Command{ } func migrateAttachments(dstStorage storage.ObjectStorage) error { - return repo_model.IterateAttachment(func(attach *repo_model.Attachment) error { + return db.IterateObjects(db.DefaultContext, func(attach *repo_model.Attachment) error { _, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath()) return err }) } func migrateLFS(dstStorage storage.ObjectStorage) error { - return git_model.IterateLFS(func(mo *git_model.LFSMetaObject) error { + return db.IterateObjects(db.DefaultContext, func(mo *git_model.LFSMetaObject) error { _, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath()) return err }) } func migrateAvatars(dstStorage storage.ObjectStorage) error { - return user_model.IterateUser(func(user *user_model.User) error { + return db.IterateObjects(db.DefaultContext, func(user *user_model.User) error { _, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath()) return err }) } func migrateRepoAvatars(dstStorage storage.ObjectStorage) error { - return repo_model.IterateRepository(func(repo *repo_model.Repository) error { + return db.IterateObjects(db.DefaultContext, func(repo *repo_model.Repository) error { _, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath()) return err }) } +func migrateRepoArchivers(dstStorage storage.ObjectStorage) error { + return db.IterateObjects(db.DefaultContext, func(archiver *repo_model.RepoArchiver) error { + p, err := archiver.RelativePath() + if err != nil { + return err + } + _, err = storage.Copy(dstStorage, p, storage.RepoArchives, p) + return err + }) +} + +func migratePackages(dstStorage storage.ObjectStorage) error { + ctx := db.DefaultContext + return db.IterateObjects(ctx, func(pf *packages_model.PackageFile) error { + pb, err := packages_model.GetBlobByID(ctx, pf.BlobID) + if err != nil { + return err + } + + p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256)) + _, err = storage.Copy(dstStorage, p, storage.RepoAvatars, p) + return err + }) +} + func runMigrateStorage(ctx *cli.Context) error { stdCtx, cancel := installSignals() defer cancel() @@ -168,25 +195,21 @@ func runMigrateStorage(ctx *cli.Context) error { return err } + migratedMethods := map[string]func(storage.ObjectStorage) error{ + "attachments": migrateAttachments, + "lfs": migrateLFS, + "avatars": migrateAvatars, + "repo-avatars": migrateRepoAvatars, + "repo-archivers": migrateRepoArchivers, + "packages": migratePackages, + } + tp := strings.ToLower(ctx.String("type")) - switch tp { - case "attachments": - if err := migrateAttachments(dstStorage); err != nil { - return err - } - case "lfs": - if err := migrateLFS(dstStorage); err != nil { + if m, ok := migratedMethods[tp]; ok { + if err := m(dstStorage); err != nil { return err } - case "avatars": - if err := migrateAvatars(dstStorage); err != nil { - return err - } - case "repo-avatars": - if err := migrateRepoAvatars(dstStorage); err != nil { - return err - } - default: + } else { return fmt.Errorf("Unsupported storage: %s", ctx.String("type")) } diff --git a/models/db/iterate.go b/models/db/iterate.go new file mode 100644 index 0000000000000..3d4fa06eeb96e --- /dev/null +++ b/models/db/iterate.go @@ -0,0 +1,34 @@ +// Copyright 2022 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package db + +import ( + "context" + + "code.gitea.io/gitea/modules/setting" +) + +// IterateObjects iterate all the Bean object +func IterateObjects[Object any](ctx context.Context, f func(repo *Object) error) error { + var start int + batchSize := setting.Database.IterateBufferSize + sess := GetEngine(ctx) + for { + repos := make([]*Object, 0, batchSize) + if err := sess.Limit(batchSize, start).Find(&repos); err != nil { + return err + } + if len(repos) == 0 { + return nil + } + start += len(repos) + + for _, repo := range repos { + if err := f(repo); err != nil { + return err + } + } + } +} diff --git a/models/git/lfs.go b/models/git/lfs.go index ec963cf593582..179da3120ae1c 100644 --- a/models/git/lfs.go +++ b/models/git/lfs.go @@ -278,29 +278,6 @@ func LFSAutoAssociate(metas []*LFSMetaObject, user *user_model.User, repoID int6 return committer.Commit() } -// IterateLFS iterates lfs object -func IterateLFS(f func(mo *LFSMetaObject) error) error { - var start int - const batchSize = 100 - e := db.GetEngine(db.DefaultContext) - for { - mos := make([]*LFSMetaObject, 0, batchSize) - if err := e.Limit(batchSize, start).Find(&mos); err != nil { - return err - } - if len(mos) == 0 { - return nil - } - start += len(mos) - - for _, mo := range mos { - if err := f(mo); err != nil { - return err - } - } - } -} - // CopyLFS copies LFS data from one repo to another func CopyLFS(ctx context.Context, newRepo, oldRepo *repo_model.Repository) error { var lfsObjects []*LFSMetaObject diff --git a/models/repo/attachment.go b/models/repo/attachment.go index ddddac2c3dcf0..afec78a4254b7 100644 --- a/models/repo/attachment.go +++ b/models/repo/attachment.go @@ -226,28 +226,6 @@ func DeleteAttachmentsByRelease(releaseID int64) error { return err } -// IterateAttachment iterates attachments; it should not be used when Gitea is servicing users. -func IterateAttachment(f func(attach *Attachment) error) error { - var start int - const batchSize = 100 - for { - attachments := make([]*Attachment, 0, batchSize) - if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&attachments); err != nil { - return err - } - if len(attachments) == 0 { - return nil - } - start += len(attachments) - - for _, attach := range attachments { - if err := f(attach); err != nil { - return err - } - } - } -} - // CountOrphanedAttachments returns the number of bad attachments func CountOrphanedAttachments() (int64, error) { return db.GetEngine(db.DefaultContext).Where("(issue_id > 0 and issue_id not in (select id from issue)) or (release_id > 0 and release_id not in (select id from `release`))"). diff --git a/models/repo/repo_list.go b/models/repo/repo_list.go index 9de76fa5ffa14..1fa469fcfeb33 100644 --- a/models/repo/repo_list.go +++ b/models/repo/repo_list.go @@ -15,36 +15,12 @@ import ( "code.gitea.io/gitea/models/unit" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/container" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" "xorm.io/builder" ) -// IterateRepository iterate repositories -func IterateRepository(f func(repo *Repository) error) error { - var start int - batchSize := setting.Database.IterateBufferSize - sess := db.GetEngine(db.DefaultContext) - for { - repos := make([]*Repository, 0, batchSize) - if err := sess.Limit(batchSize, start).Find(&repos); err != nil { - return err - } - if len(repos) == 0 { - return nil - } - start += len(repos) - - for _, repo := range repos { - if err := f(repo); err != nil { - return err - } - } - } -} - // FindReposMapByIDs find repos as map func FindReposMapByIDs(repoIDs []int64, res map[int64]*Repository) error { return db.GetEngine(db.DefaultContext).In("id", repoIDs).Find(&res) diff --git a/models/user/search.go b/models/user/search.go index f8e6c89f06804..0aa9949367afb 100644 --- a/models/user/search.go +++ b/models/user/search.go @@ -9,7 +9,6 @@ import ( "strings" "code.gitea.io/gitea/models/db" - "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/structs" "code.gitea.io/gitea/modules/util" @@ -125,28 +124,6 @@ func SearchUsers(opts *SearchUserOptions) (users []*User, _ int64, _ error) { return users, count, sessQuery.Find(&users) } -// IterateUser iterate users -func IterateUser(f func(user *User) error) error { - var start int - batchSize := setting.Database.IterateBufferSize - for { - users := make([]*User, 0, batchSize) - if err := db.GetEngine(db.DefaultContext).Limit(batchSize, start).Find(&users); err != nil { - return err - } - if len(users) == 0 { - return nil - } - start += len(users) - - for _, user := range users { - if err := f(user); err != nil { - return err - } - } - } -} - // BuildCanSeeUserCondition creates a condition which can be used to restrict results to users/orgs the actor can see func BuildCanSeeUserCondition(actor *User) builder.Cond { if actor != nil { diff --git a/modules/packages/content_store.go b/modules/packages/content_store.go index 64c3eedc2328e..a3a5d1a6663c8 100644 --- a/modules/packages/content_store.go +++ b/modules/packages/content_store.go @@ -27,21 +27,21 @@ func NewContentStore() *ContentStore { // Get gets a package blob func (s *ContentStore) Get(key BlobHash256Key) (storage.Object, error) { - return s.store.Open(keyToRelativePath(key)) + return s.store.Open(KeyToRelativePath(key)) } // Save stores a package blob func (s *ContentStore) Save(key BlobHash256Key, r io.Reader, size int64) error { - _, err := s.store.Save(keyToRelativePath(key), r, size) + _, err := s.store.Save(KeyToRelativePath(key), r, size) return err } // Delete deletes a package blob func (s *ContentStore) Delete(key BlobHash256Key) error { - return s.store.Delete(keyToRelativePath(key)) + return s.store.Delete(KeyToRelativePath(key)) } -// keyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000... -func keyToRelativePath(key BlobHash256Key) string { +// KeyToRelativePath converts the sha256 key aabb000000... to aa/bb/aabb000000... +func KeyToRelativePath(key BlobHash256Key) string { return path.Join(string(key)[0:2], string(key)[2:4], string(key)) } diff --git a/routers/private/mail.go b/routers/private/mail.go index 966a838168006..e858992aee13b 100644 --- a/routers/private/mail.go +++ b/routers/private/mail.go @@ -9,6 +9,7 @@ import ( "net/http" "strconv" + "code.gitea.io/gitea/models/db" user_model "code.gitea.io/gitea/models/user" "code.gitea.io/gitea/modules/context" "code.gitea.io/gitea/modules/json" @@ -59,7 +60,7 @@ func SendEmail(ctx *context.PrivateContext) { } } } else { - err := user_model.IterateUser(func(user *user_model.User) error { + err := db.IterateObjects(ctx, func(user *user_model.User) error { if len(user.Email) > 0 && user.IsActive { emails = append(emails, user.Email) } diff --git a/services/repository/avatar.go b/services/repository/avatar.go index dcf04c7e547c5..b9bd36ab66588 100644 --- a/services/repository/avatar.go +++ b/services/repository/avatar.go @@ -96,7 +96,7 @@ func DeleteAvatar(repo *repo_model.Repository) error { // RemoveRandomAvatars removes the randomly generated avatars that were created for repositories func RemoveRandomAvatars(ctx context.Context) error { - return repo_model.IterateRepository(func(repository *repo_model.Repository) error { + return db.IterateObjects(ctx, func(repository *repo_model.Repository) error { select { case <-ctx.Done(): return db.ErrCancelledf("before random avatars removed for %s", repository.FullName()) From d3139293c4d13c4bdfe586ebac7ae8e6ac7b3b78 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 11 Aug 2022 10:25:45 +0800 Subject: [PATCH 02/11] Fix typo --- cmd/migrate_storage.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index 570f284d2bd4f..deb91610f2216 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -130,7 +130,7 @@ func migratePackages(dstStorage storage.ObjectStorage) error { } p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256)) - _, err = storage.Copy(dstStorage, p, storage.RepoAvatars, p) + _, err = storage.Copy(dstStorage, p, storage.Packages, p) return err }) } @@ -189,7 +189,7 @@ func runMigrateStorage(ctx *cli.Context) error { UseSSL: ctx.Bool("minio-use-ssl"), }) default: - return fmt.Errorf("Unsupported storage type: %s", ctx.String("storage")) + return fmt.Errorf("unsupported storage type: %s", ctx.String("storage")) } if err != nil { return err @@ -209,11 +209,9 @@ func runMigrateStorage(ctx *cli.Context) error { if err := m(dstStorage); err != nil { return err } - } else { - return fmt.Errorf("Unsupported storage: %s", ctx.String("type")) + log.Warn("All files have been copied to the new placement but old files are still on the original placement.") + return nil } - log.Warn("All files have been copied to the new placement but old files are still on the original placement.") - - return nil + return fmt.Errorf("unsupported storage: %s", ctx.String("type")) } From 76b5c41b7765f4734dad48b215802641db03e412 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 11 Aug 2022 10:33:09 +0800 Subject: [PATCH 03/11] Use stdCtx --- cmd/migrate_storage.go | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index deb91610f2216..57a2cb7a9b80d 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -82,36 +82,36 @@ var CmdMigrateStorage = cli.Command{ }, } -func migrateAttachments(dstStorage storage.ObjectStorage) error { - return db.IterateObjects(db.DefaultContext, func(attach *repo_model.Attachment) error { +func migrateAttachments(ctx context.Context, dstStorage storage.ObjectStorage) error { + return db.IterateObjects(ctx, func(attach *repo_model.Attachment) error { _, err := storage.Copy(dstStorage, attach.RelativePath(), storage.Attachments, attach.RelativePath()) return err }) } -func migrateLFS(dstStorage storage.ObjectStorage) error { - return db.IterateObjects(db.DefaultContext, func(mo *git_model.LFSMetaObject) error { +func migrateLFS(ctx context.Context, dstStorage storage.ObjectStorage) error { + return db.IterateObjects(ctx, func(mo *git_model.LFSMetaObject) error { _, err := storage.Copy(dstStorage, mo.RelativePath(), storage.LFS, mo.RelativePath()) return err }) } -func migrateAvatars(dstStorage storage.ObjectStorage) error { - return db.IterateObjects(db.DefaultContext, func(user *user_model.User) error { +func migrateAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error { + return db.IterateObjects(ctx, func(user *user_model.User) error { _, err := storage.Copy(dstStorage, user.CustomAvatarRelativePath(), storage.Avatars, user.CustomAvatarRelativePath()) return err }) } -func migrateRepoAvatars(dstStorage storage.ObjectStorage) error { - return db.IterateObjects(db.DefaultContext, func(repo *repo_model.Repository) error { +func migrateRepoAvatars(ctx context.Context, dstStorage storage.ObjectStorage) error { + return db.IterateObjects(ctx, func(repo *repo_model.Repository) error { _, err := storage.Copy(dstStorage, repo.CustomAvatarRelativePath(), storage.RepoAvatars, repo.CustomAvatarRelativePath()) return err }) } -func migrateRepoArchivers(dstStorage storage.ObjectStorage) error { - return db.IterateObjects(db.DefaultContext, func(archiver *repo_model.RepoArchiver) error { +func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) error { + return db.IterateObjects(ctx, func(archiver *repo_model.RepoArchiver) error { p, err := archiver.RelativePath() if err != nil { return err @@ -121,8 +121,7 @@ func migrateRepoArchivers(dstStorage storage.ObjectStorage) error { }) } -func migratePackages(dstStorage storage.ObjectStorage) error { - ctx := db.DefaultContext +func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error { return db.IterateObjects(ctx, func(pf *packages_model.PackageFile) error { pb, err := packages_model.GetBlobByID(ctx, pf.BlobID) if err != nil { @@ -154,8 +153,6 @@ func runMigrateStorage(ctx *cli.Context) error { return err } - goCtx := context.Background() - if err := storage.Init(); err != nil { return err } @@ -172,13 +169,13 @@ func runMigrateStorage(ctx *cli.Context) error { return nil } dstStorage, err = storage.NewLocalStorage( - goCtx, + stdCtx, storage.LocalStorageConfig{ Path: p, }) case string(storage.MinioStorageType): dstStorage, err = storage.NewMinioStorage( - goCtx, + stdCtx, storage.MinioStorageConfig{ Endpoint: ctx.String("minio-endpoint"), AccessKeyID: ctx.String("minio-access-key-id"), @@ -195,7 +192,7 @@ func runMigrateStorage(ctx *cli.Context) error { return err } - migratedMethods := map[string]func(storage.ObjectStorage) error{ + migratedMethods := map[string]func(context.Context, storage.ObjectStorage) error{ "attachments": migrateAttachments, "lfs": migrateLFS, "avatars": migrateAvatars, @@ -206,7 +203,7 @@ func runMigrateStorage(ctx *cli.Context) error { tp := strings.ToLower(ctx.String("type")) if m, ok := migratedMethods[tp]; ok { - if err := m(dstStorage); err != nil { + if err := m(stdCtx, dstStorage); err != nil { return err } log.Warn("All files have been copied to the new placement but old files are still on the original placement.") From d7697ec191704c1d679c98ebcc1814ffa8044b98 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 11 Aug 2022 17:54:33 +0800 Subject: [PATCH 04/11] Use packageblob and fix command description --- cmd/migrate_storage.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index 57a2cb7a9b80d..53928fd7e1ee2 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -27,13 +27,13 @@ import ( var CmdMigrateStorage = cli.Command{ Name: "migrate-storage", Usage: "Migrate the storage", - Description: "This is a command for migrating storage.", + Description: "This is a command for migrating current storage to external storage.", Action: runMigrateStorage, Flags: []cli.Flag{ cli.StringFlag{ Name: "type, t", Value: "", - Usage: "Kinds of files to migrate, currently only 'attachments' is supported", + Usage: "Kinds of files to migrate, could be one of 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages'", }, cli.StringFlag{ Name: "storage, s", @@ -122,14 +122,9 @@ func migrateRepoArchivers(ctx context.Context, dstStorage storage.ObjectStorage) } func migratePackages(ctx context.Context, dstStorage storage.ObjectStorage) error { - return db.IterateObjects(ctx, func(pf *packages_model.PackageFile) error { - pb, err := packages_model.GetBlobByID(ctx, pf.BlobID) - if err != nil { - return err - } - + return db.IterateObjects(ctx, func(pb *packages_model.PackageBlob) error { p := packages_module.KeyToRelativePath(packages_module.BlobHash256Key(pb.HashSHA256)) - _, err = storage.Copy(dstStorage, p, storage.Packages, p) + _, err := storage.Copy(dstStorage, p, storage.Packages, p) return err }) } From 7fcba32c10628998c36b464cee9d11b0acd974c4 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 11 Aug 2022 18:05:56 +0800 Subject: [PATCH 05/11] Add migrate packages unit tests --- cmd/main_test.go | 23 ++++++++++++ cmd/migrate_storage_test.go | 74 +++++++++++++++++++++++++++++++++++++ 2 files changed, 97 insertions(+) create mode 100644 cmd/main_test.go create mode 100644 cmd/migrate_storage_test.go diff --git a/cmd/main_test.go b/cmd/main_test.go new file mode 100644 index 0000000000000..9e749dce774bf --- /dev/null +++ b/cmd/main_test.go @@ -0,0 +1,23 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "testing" + + "code.gitea.io/gitea/models/unittest" + "code.gitea.io/gitea/modules/setting" +) + +func init() { + setting.SetCustomPathAndConf("", "", "") + setting.LoadForTest() +} + +func TestMain(m *testing.M) { + unittest.MainTest(m, &unittest.TestOptions{ + GiteaRootPath: "..", + }) +} diff --git a/cmd/migrate_storage_test.go b/cmd/migrate_storage_test.go new file mode 100644 index 0000000000000..9c9b12566303e --- /dev/null +++ b/cmd/migrate_storage_test.go @@ -0,0 +1,74 @@ +// Copyright 2020 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "context" + "os" + "strings" + "testing" + + "code.gitea.io/gitea/models/packages" + "code.gitea.io/gitea/models/unittest" + user_model "code.gitea.io/gitea/models/user" + packages_module "code.gitea.io/gitea/modules/packages" + "code.gitea.io/gitea/modules/storage" + packages_service "code.gitea.io/gitea/services/packages" + + "github.com/stretchr/testify/assert" +) + +func TestMigratePackages(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + creator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}).(*user_model.User) + + content := "package main\n\nfunc main() {\nfmt.Println(\"hi\")\n}\n" + buf, err := packages_module.CreateHashedBufferFromReader(strings.NewReader(content), 1024) + assert.NoError(t, err) + defer buf.Close() + + v, f, err := packages_service.CreatePackageAndAddFile(&packages_service.PackageCreationInfo{ + PackageInfo: packages_service.PackageInfo{ + Owner: creator, + PackageType: packages.TypeGeneric, + Name: "test", + Version: "1.0.0", + }, + Creator: creator, + SemverCompatible: true, + VersionProperties: map[string]string{}, + }, &packages_service.PackageFileCreationInfo{ + PackageFileInfo: packages_service.PackageFileInfo{ + Filename: "a.go", + }, + Data: buf, + IsLead: true, + }) + assert.NoError(t, err) + assert.NotNil(t, v) + assert.NotNil(t, f) + + ctx := context.Background() + + p, err := os.MkdirTemp(os.TempDir(), "migrated_packages") + assert.NoError(t, err) + + dstStorage, err := storage.NewLocalStorage( + ctx, + storage.LocalStorageConfig{ + Path: p, + }) + assert.NoError(t, err) + + err = migratePackages(ctx, dstStorage) + assert.NoError(t, err) + + entries, err := os.ReadDir(p) + assert.NoError(t, err) + assert.EqualValues(t, 2, len(entries)) + assert.EqualValues(t, "01", entries[0].Name()) + assert.EqualValues(t, "tmp", entries[1].Name()) +} From 9f0e37ac6feb726f87400ac9e5da3e3f704fdb13 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 12 Aug 2022 13:43:18 +0800 Subject: [PATCH 06/11] Fix comment year --- cmd/main_test.go | 2 +- cmd/migrate_storage_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/main_test.go b/cmd/main_test.go index 9e749dce774bf..9cce0ef0360b5 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. +// Copyright 2022 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. diff --git a/cmd/migrate_storage_test.go b/cmd/migrate_storage_test.go index 9c9b12566303e..1297ba69a81eb 100644 --- a/cmd/migrate_storage_test.go +++ b/cmd/migrate_storage_test.go @@ -1,4 +1,4 @@ -// Copyright 2020 The Gitea Authors. All rights reserved. +// Copyright 2022 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. From 404a32beb86bc6cf2845025be32b5a434bab216b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 15 Aug 2022 12:00:23 +0800 Subject: [PATCH 07/11] Fix the migrate storage command line description --- cmd/migrate_storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index 53928fd7e1ee2..7b53afe178d83 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -27,7 +27,7 @@ import ( var CmdMigrateStorage = cli.Command{ Name: "migrate-storage", Usage: "Migrate the storage", - Description: "This is a command for migrating current storage to external storage.", + Description: "This is a command for migrating Gitea configured storage to a parameter indicated storage.", Action: runMigrateStorage, Flags: []cli.Flag{ cli.StringFlag{ From 8458feb8e91be4f9c51816acf2c68b78e10eb158 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 16 Aug 2022 09:36:26 +0800 Subject: [PATCH 08/11] Update cmd/migrate_storage.go Co-authored-by: zeripath --- cmd/migrate_storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index 7b53afe178d83..75652515f7ed6 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -27,7 +27,7 @@ import ( var CmdMigrateStorage = cli.Command{ Name: "migrate-storage", Usage: "Migrate the storage", - Description: "This is a command for migrating Gitea configured storage to a parameter indicated storage.", + Description: "Copies stored files from storage configured in app.ini to parameter-configured storage", Action: runMigrateStorage, Flags: []cli.Flag{ cli.StringFlag{ From 97a1ed2abf0e2d5fdf4e672247331eaba42e36f4 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 16 Aug 2022 10:46:29 +0800 Subject: [PATCH 09/11] Update cmd/migrate_storage.go Co-authored-by: zeripath --- cmd/migrate_storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index 75652515f7ed6..0fc1790b8f5f6 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -33,7 +33,7 @@ var CmdMigrateStorage = cli.Command{ cli.StringFlag{ Name: "type, t", Value: "", - Usage: "Kinds of files to migrate, could be one of 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages'", + Usage: "Type of stored files to copy. Allowed types: 'attachments', 'lfs', 'avatars', 'repo-avatars', 'repo-archivers', 'packages'", }, cli.StringFlag{ Name: "storage, s", From eb08df9226c85d9cedcc10ea47f5605e2a296f01 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 16 Aug 2022 10:46:48 +0800 Subject: [PATCH 10/11] Update cmd/migrate_storage.go Co-authored-by: zeripath --- cmd/migrate_storage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/migrate_storage.go b/cmd/migrate_storage.go index 0fc1790b8f5f6..f11cf9b11f373 100644 --- a/cmd/migrate_storage.go +++ b/cmd/migrate_storage.go @@ -201,7 +201,7 @@ func runMigrateStorage(ctx *cli.Context) error { if err := m(stdCtx, dstStorage); err != nil { return err } - log.Warn("All files have been copied to the new placement but old files are still on the original placement.") + log.Info("%s files have successfully been copied to the new storage.", tp) return nil } From a7ca5f6a85842b33f5ef4c0ac61dbd27699607f5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Tue, 16 Aug 2022 11:35:55 +0800 Subject: [PATCH 11/11] Fix test --- cmd/migrate_storage_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/migrate_storage_test.go b/cmd/migrate_storage_test.go index 1297ba69a81eb..e6d205e4109d8 100644 --- a/cmd/migrate_storage_test.go +++ b/cmd/migrate_storage_test.go @@ -23,7 +23,7 @@ import ( func TestMigratePackages(t *testing.T) { assert.NoError(t, unittest.PrepareTestDatabase()) - creator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}).(*user_model.User) + creator := unittest.AssertExistsAndLoadBean(t, &user_model.User{ID: 1}) content := "package main\n\nfunc main() {\nfmt.Println(\"hi\")\n}\n" buf, err := packages_module.CreateHashedBufferFromReader(strings.NewReader(content), 1024)