WIP: Add backup & restore commands

cmd/backup.go

@@ -0,0 +1,75 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package cmd
+
+import (
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/setting"
+
+	"github.com/urfave/cli/v2"
+)
+
+// CmdBackup backup all data from database to fixtures files on dirPath
+var CmdBackup = &cli.Command{
+	Name:        "backup",
+	Usage:       "Backup the Gitea database",
+	Description: "A command to backup all data from database to fixtures files on dirPath",
+	Action:      runBackup,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "dir-path",
+			Value: "",
+			Usage: "Directory path to save fixtures files",
+		},
+	},
+}
+
+func runBackup(ctx *cli.Context) error {
+	stdCtx, cancel := installSignals()
+	defer cancel()
+
+	if err := initDB(stdCtx); err != nil {
+		return err
+	}
+
+	log.Info("AppPath: %s", setting.AppPath)
+	log.Info("AppWorkPath: %s", setting.AppWorkPath)
+	log.Info("Custom path: %s", setting.CustomPath)
+	log.Info("Log path: %s", setting.Log.RootPath)
+	log.Info("Configuration file: %s", setting.CustomConf)
+
+	return db.BackupDatabaseAsFixtures(ctx.String("dir-path"))
+}
+
+var CmdRestore = &cli.Command{
+	Name:        "restore",
+	Usage:       "Restore the Gitea database",
+	Description: "A command to restore all data from fixtures files on dirPath to database",
+	Action:      runRestore,
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:  "dir-path",
+			Value: "",
+			Usage: "Directory path to load fixtures files",
+		},
+	},
+}
+
+func runRestore(ctx *cli.Context) error {
+	stdCtx, cancel := installSignals()
+	defer cancel()
+
+	if err := initDB(stdCtx); err != nil {
+		return err
+	}
+
+	log.Info("AppPath: %s", setting.AppPath)
+	log.Info("AppWorkPath: %s", setting.AppWorkPath)
+	log.Info("Custom path: %s", setting.CustomPath)
+	log.Info("Log path: %s", setting.Log.RootPath)
+	log.Info("Configuration file: %s", setting.CustomConf)
+
+	return db.RestoreDatabase(ctx.String("dir-path"))
+}

cmd/main.go

@@ -142,6 +142,8 @@ func NewMainApp(version, versionExtra string) *cli.App {
 		CmdDumpRepository,
 		CmdRestoreRepository,
 		CmdActions,
+		CmdBackup,
+		CmdRestore,
 		cmdHelp(), // the "help" sub-command was used to show the more information for "work path" and "custom config"
 	}
 

models/db/backup.go

@@ -0,0 +1,150 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package db
+
+import (
+	"encoding/base64"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"gopkg.in/yaml.v3"
+	"xorm.io/xorm"
+)
+
+// BackupDatabaseAsFixtures backup all data from database to fixtures files on dirPath
+func BackupDatabaseAsFixtures(dirPath string) error {
+	if err := os.MkdirAll(dirPath, os.ModePerm); err != nil {
+		return err
+	}
+
+	for _, t := range tables {
+		if err := backupTableFixtures(x, t, dirPath); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func toNode(tableName, col string, v interface{}) *yaml.Node {
+	if v == nil {
+		return &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Value: "",
+		}
+	}
+	switch vv := v.(type) {
+	case string:
+		if tableName == "action_task" && col == "log_indexes" {
+			return &yaml.Node{
+				Kind:  yaml.ScalarNode,
+				Tag:   "!!binary",
+				Value: base64.StdEncoding.EncodeToString([]byte(vv)),
+			}
+		}
+		return &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Value: vv,
+		}
+	case []byte:
+		if tableName == "action_task" && col == "log_indexes" {
+			return &yaml.Node{
+				Kind:  yaml.ScalarNode,
+				Tag:   "!!binary",
+				Value: base64.StdEncoding.EncodeToString(vv),
+			}
+		}
+		return &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!str",
+			Value: string(vv),
+		}
+	case int, int64, int32, int8, int16, uint, uint64:
+		return &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!int",
+			Value: fmt.Sprintf("%d", vv),
+		}
+	case float64, float32:
+		return &yaml.Node{
+			Kind:  yaml.ScalarNode,
+			Tag:   "!!float",
+			Value: fmt.Sprintf("%f", vv),
+		}
+	default:
+		panic(fmt.Sprintf("unknow type %#v", v))
+	}
+}
+
+func backupTableFixtures(e *xorm.Engine, bean interface{}, dirPath string) error {
+	table, err := e.TableInfo(bean)
+	if err != nil {
+		return err
+	}
+	if isEmpty, err := e.IsTableEmpty(table.Name); err != nil {
+		return err
+	} else if isEmpty {
+		return nil
+	}
+
+	f, err := os.Create(filepath.Join(dirPath, table.Name+".yml"))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	const bufferSize = 100
+	start := 0
+
+	for {
+		objs, err := e.Table(table.Name).Limit(bufferSize, start).QueryInterface()
+		if err != nil {
+			return err
+		}
+		if len(objs) == 0 {
+			break
+		}
+
+		for _, obj := range objs {
+			node := yaml.Node{
+				Kind: yaml.MappingNode,
+			}
+			for _, col := range table.ColumnsSeq() {
+				v, ok := obj[col]
+				if !ok {
+					return fmt.Errorf("column %s has no value from database", col)
+				}
+
+				node.Content = append(node.Content,
+					&yaml.Node{
+						Kind:  yaml.ScalarNode,
+						Tag:   "!!str",
+						Value: col,
+					},
+					toNode(table.Name, col, v),
+				)
+			}
+
+			bs, err := yaml.Marshal([]*yaml.Node{&node}) // with []any{} to ensure generated a list
+			if err != nil {
+				return fmt.Errorf("marshal table %s record %#v %#v failed: %v", table.Name, obj, node.Content[1], err)
+			}
+			if _, err := f.Write(bs); err != nil {
+				return err
+			}
+			if _, err := f.Write([]byte{'\n'}); err != nil { // generate a blank line for human readable
+				return err
+			}
+		}
+
+		if len(objs) < bufferSize {
+			break
+		}
+		start += len(objs)
+	}
+
+	return nil
+}

models/db/backup_test.go

@@ -0,0 +1,111 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package db_test
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"testing"
+
+	"code.gitea.io/gitea/models/db"
+	"code.gitea.io/gitea/models/unittest"
+
+	"github.com/stretchr/testify/assert"
+	"gopkg.in/yaml.v3"
+)
+
+func TestBackupRestore(t *testing.T) {
+	assert.NoError(t, unittest.PrepareTestDatabase())
+
+	d, err := os.MkdirTemp(os.TempDir(), "backup_restore")
+	assert.NoError(t, err)
+
+	assert.NoError(t, db.BackupDatabaseAsFixtures(d))
+
+	f, err := os.Open(d)
+	assert.NoError(t, err)
+	defer f.Close()
+
+	entries, err := f.ReadDir(0)
+	assert.NoError(t, err)
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+
+		fileEqual(t, filepath.Join("..", "fixtures", entry.Name()), filepath.Join(d, entry.Name()))
+	}
+
+	// assert.NoError(t, db.RestoreDatabase(d))
+}
+
+func sortTable(tablename string, data []map[string]any) {
+	sort.Slice(data, func(i, j int) bool {
+		if tablename == "issue_index" {
+			return data[i]["group_id"].(int) < data[j]["group_id"].(int)
+		}
+		if tablename == "repo_topic" {
+			return data[i]["repo_id"].(int) < data[j]["repo_id"].(int)
+		}
+		return data[i]["id"].(int) < data[j]["id"].(int)
+	})
+}
+
+func convertBool(b any) bool {
+	switch rr := b.(type) {
+	case bool:
+		return rr
+	case int:
+		return rr != 0
+	default:
+		r, _ := strconv.ParseBool(b.(string))
+		return r
+	}
+}
+
+func fileEqual(t *testing.T, a, b string) {
+	filename := filepath.Base(a)
+	tablename := filename[:len(filename)-len(filepath.Ext(filename))]
+	t.Run(filename, func(t *testing.T) {
+		bs1, err := os.ReadFile(a)
+		assert.NoError(t, err)
+
+		var data1 []map[string]any
+		assert.NoError(t, yaml.Unmarshal(bs1, &data1))
+
+		sortTable(tablename, data1)
+
+		bs2, err := os.ReadFile(b)
+		assert.NoError(t, err)
+
+		var data2 []map[string]any
+		assert.NoError(t, yaml.Unmarshal(bs2, &data2))
+
+		sortTable(tablename, data2)
+
+		assert.EqualValues(t, len(data1), len(data2), fmt.Sprintf("compare %s with %s", a, b))
+		for i := range data1 {
+			assert.LessOrEqual(t, len(data1[i]), len(data2[i]), fmt.Sprintf("compare %s with %s", a, b))
+			for k, v := range data1[i] {
+				switch vv := v.(type) {
+				case bool:
+					assert.EqualValues(t, vv, convertBool(data2[i][k]), fmt.Sprintf("compare %s with %s", a, b))
+				case nil:
+					switch data2[i][k].(type) {
+					case nil:
+					case string:
+						assert.Empty(t, data2[i][k])
+					default:
+						panic(fmt.Sprintf("%#v", data2[i][k]))
+					}
+				default:
+					assert.EqualValues(t, v, data2[i][k], fmt.Sprintf("compare %#v with %#v", v, data2[i][k]))
+				}
+			}
+		}
+	})
+}

models/db/restore.go

@@ -0,0 +1,45 @@
+// Copyright 2023 The Gitea Authors. All rights reserved.
+// SPDX-License-Identifier: MIT
+
+package db
+
+import (
+	"fmt"
+
+	"github.com/go-testfixtures/testfixtures/v3"
+	"xorm.io/xorm/schemas"
+)
+
+func RestoreDatabase(dirPath string) error {
+	var dialect string
+	switch x.Dialect().URI().DBType {
+	case schemas.POSTGRES:
+		dialect = "postgres"
+	case schemas.MYSQL:
+		dialect = "mysql"
+	case schemas.MSSQL:
+		dialect = "mssql"
+	case schemas.SQLITE:
+		dialect = "sqlite3"
+	default:
+		return fmt.Errorf("unsupported RDBMS for integration tests")
+	}
+
+	loaderOptions := []func(loader *testfixtures.Loader) error{
+		testfixtures.Database(x.DB().DB),
+		testfixtures.Dialect(dialect),
+		testfixtures.DangerousSkipTestDatabaseCheck(),
+		testfixtures.Directory(dirPath),
+	}
+
+	if x.Dialect().URI().DBType == schemas.POSTGRES {
+		loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences())
+	}
+
+	fixtures, err := testfixtures.New(loaderOptions...)
+	if err != nil {
+		return err
+	}
+
+	return fixtures.Load()
+}