diff --git a/cmd/backup.go b/cmd/backup.go new file mode 100644 index 0000000000000..fcb9fe6a1e073 --- /dev/null +++ b/cmd/backup.go @@ -0,0 +1,75 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cmd + +import ( + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + + "github.com/urfave/cli/v2" +) + +// CmdBackup backup all data from database to fixtures files on dirPath +var CmdBackup = &cli.Command{ + Name: "backup", + Usage: "Backup the Gitea database", + Description: "A command to backup all data from database to fixtures files on dirPath", + Action: runBackup, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "dir-path", + Value: "", + Usage: "Directory path to save fixtures files", + }, + }, +} + +func runBackup(ctx *cli.Context) error { + stdCtx, cancel := installSignals() + defer cancel() + + if err := initDB(stdCtx); err != nil { + return err + } + + log.Info("AppPath: %s", setting.AppPath) + log.Info("AppWorkPath: %s", setting.AppWorkPath) + log.Info("Custom path: %s", setting.CustomPath) + log.Info("Log path: %s", setting.Log.RootPath) + log.Info("Configuration file: %s", setting.CustomConf) + + return db.BackupDatabaseAsFixtures(ctx.String("dir-path")) +} + +var CmdRestore = &cli.Command{ + Name: "restore", + Usage: "Restore the Gitea database", + Description: "A command to restore all data from fixtures files on dirPath to database", + Action: runRestore, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "dir-path", + Value: "", + Usage: "Directory path to load fixtures files", + }, + }, +} + +func runRestore(ctx *cli.Context) error { + stdCtx, cancel := installSignals() + defer cancel() + + if err := initDB(stdCtx); err != nil { + return err + } + + log.Info("AppPath: %s", setting.AppPath) + log.Info("AppWorkPath: %s", setting.AppWorkPath) + log.Info("Custom path: %s", setting.CustomPath) + log.Info("Log path: %s", setting.Log.RootPath) + log.Info("Configuration file: %s", setting.CustomConf) + + return db.RestoreDatabase(ctx.String("dir-path")) +} diff --git a/cmd/main.go b/cmd/main.go index feda41e68b24a..3f72804022ecf 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -142,6 +142,8 @@ func NewMainApp(version, versionExtra string) *cli.App { CmdDumpRepository, CmdRestoreRepository, CmdActions, + CmdBackup, + CmdRestore, cmdHelp(), // the "help" sub-command was used to show the more information for "work path" and "custom config" } diff --git a/models/db/backup.go b/models/db/backup.go new file mode 100644 index 0000000000000..9a4073f04555c --- /dev/null +++ b/models/db/backup.go @@ -0,0 +1,150 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package db + +import ( + "encoding/base64" + "fmt" + "os" + "path/filepath" + + "gopkg.in/yaml.v3" + "xorm.io/xorm" +) + +// BackupDatabaseAsFixtures backup all data from database to fixtures files on dirPath +func BackupDatabaseAsFixtures(dirPath string) error { + if err := os.MkdirAll(dirPath, os.ModePerm); err != nil { + return err + } + + for _, t := range tables { + if err := backupTableFixtures(x, t, dirPath); err != nil { + return err + } + } + return nil +} + +func toNode(tableName, col string, v interface{}) *yaml.Node { + if v == nil { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "", + } + } + switch vv := v.(type) { + case string: + if tableName == "action_task" && col == "log_indexes" { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!binary", + Value: base64.StdEncoding.EncodeToString([]byte(vv)), + } + } + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: vv, + } + case []byte: + if tableName == "action_task" && col == "log_indexes" { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!binary", + Value: base64.StdEncoding.EncodeToString(vv), + } + } + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: string(vv), + } + case int, int64, int32, int8, int16, uint, uint64: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!int", + Value: fmt.Sprintf("%d", vv), + } + case float64, float32: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!float", + Value: fmt.Sprintf("%f", vv), + } + default: + panic(fmt.Sprintf("unknow type %#v", v)) + } +} + +func backupTableFixtures(e *xorm.Engine, bean interface{}, dirPath string) error { + table, err := e.TableInfo(bean) + if err != nil { + return err + } + if isEmpty, err := e.IsTableEmpty(table.Name); err != nil { + return err + } else if isEmpty { + return nil + } + + f, err := os.Create(filepath.Join(dirPath, table.Name+".yml")) + if err != nil { + return err + } + defer f.Close() + + const bufferSize = 100 + start := 0 + + for { + objs, err := e.Table(table.Name).Limit(bufferSize, start).QueryInterface() + if err != nil { + return err + } + if len(objs) == 0 { + break + } + + for _, obj := range objs { + node := yaml.Node{ + Kind: yaml.MappingNode, + } + for _, col := range table.ColumnsSeq() { + v, ok := obj[col] + if !ok { + return fmt.Errorf("column %s has no value from database", col) + } + + node.Content = append(node.Content, + &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: col, + }, + toNode(table.Name, col, v), + ) + } + + bs, err := yaml.Marshal([]*yaml.Node{&node}) // with []any{} to ensure generated a list + if err != nil { + return fmt.Errorf("marshal table %s record %#v %#v failed: %v", table.Name, obj, node.Content[1], err) + } + if _, err := f.Write(bs); err != nil { + return err + } + if _, err := f.Write([]byte{'\n'}); err != nil { // generate a blank line for human readable + return err + } + } + + if len(objs) < bufferSize { + break + } + start += len(objs) + } + + return nil +} diff --git a/models/db/backup_test.go b/models/db/backup_test.go new file mode 100644 index 0000000000000..6bb8e1e6b7e74 --- /dev/null +++ b/models/db/backup_test.go @@ -0,0 +1,111 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package db_test + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "testing" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/unittest" + + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" +) + +func TestBackupRestore(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + d, err := os.MkdirTemp(os.TempDir(), "backup_restore") + assert.NoError(t, err) + + assert.NoError(t, db.BackupDatabaseAsFixtures(d)) + + f, err := os.Open(d) + assert.NoError(t, err) + defer f.Close() + + entries, err := f.ReadDir(0) + assert.NoError(t, err) + for _, entry := range entries { + if entry.IsDir() { + continue + } + + fileEqual(t, filepath.Join("..", "fixtures", entry.Name()), filepath.Join(d, entry.Name())) + } + + // assert.NoError(t, db.RestoreDatabase(d)) +} + +func sortTable(tablename string, data []map[string]any) { + sort.Slice(data, func(i, j int) bool { + if tablename == "issue_index" { + return data[i]["group_id"].(int) < data[j]["group_id"].(int) + } + if tablename == "repo_topic" { + return data[i]["repo_id"].(int) < data[j]["repo_id"].(int) + } + return data[i]["id"].(int) < data[j]["id"].(int) + }) +} + +func convertBool(b any) bool { + switch rr := b.(type) { + case bool: + return rr + case int: + return rr != 0 + default: + r, _ := strconv.ParseBool(b.(string)) + return r + } +} + +func fileEqual(t *testing.T, a, b string) { + filename := filepath.Base(a) + tablename := filename[:len(filename)-len(filepath.Ext(filename))] + t.Run(filename, func(t *testing.T) { + bs1, err := os.ReadFile(a) + assert.NoError(t, err) + + var data1 []map[string]any + assert.NoError(t, yaml.Unmarshal(bs1, &data1)) + + sortTable(tablename, data1) + + bs2, err := os.ReadFile(b) + assert.NoError(t, err) + + var data2 []map[string]any + assert.NoError(t, yaml.Unmarshal(bs2, &data2)) + + sortTable(tablename, data2) + + assert.EqualValues(t, len(data1), len(data2), fmt.Sprintf("compare %s with %s", a, b)) + for i := range data1 { + assert.LessOrEqual(t, len(data1[i]), len(data2[i]), fmt.Sprintf("compare %s with %s", a, b)) + for k, v := range data1[i] { + switch vv := v.(type) { + case bool: + assert.EqualValues(t, vv, convertBool(data2[i][k]), fmt.Sprintf("compare %s with %s", a, b)) + case nil: + switch data2[i][k].(type) { + case nil: + case string: + assert.Empty(t, data2[i][k]) + default: + panic(fmt.Sprintf("%#v", data2[i][k])) + } + default: + assert.EqualValues(t, v, data2[i][k], fmt.Sprintf("compare %#v with %#v", v, data2[i][k])) + } + } + } + }) +} diff --git a/models/db/restore.go b/models/db/restore.go new file mode 100644 index 0000000000000..cf03faf85a34a --- /dev/null +++ b/models/db/restore.go @@ -0,0 +1,45 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package db + +import ( + "fmt" + + "github.com/go-testfixtures/testfixtures/v3" + "xorm.io/xorm/schemas" +) + +func RestoreDatabase(dirPath string) error { + var dialect string + switch x.Dialect().URI().DBType { + case schemas.POSTGRES: + dialect = "postgres" + case schemas.MYSQL: + dialect = "mysql" + case schemas.MSSQL: + dialect = "mssql" + case schemas.SQLITE: + dialect = "sqlite3" + default: + return fmt.Errorf("unsupported RDBMS for integration tests") + } + + loaderOptions := []func(loader *testfixtures.Loader) error{ + testfixtures.Database(x.DB().DB), + testfixtures.Dialect(dialect), + testfixtures.DangerousSkipTestDatabaseCheck(), + testfixtures.Directory(dirPath), + } + + if x.Dialect().URI().DBType == schemas.POSTGRES { + loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences()) + } + + fixtures, err := testfixtures.New(loaderOptions...) + if err != nil { + return err + } + + return fixtures.Load() +}