diff --git a/cmd/dump.go b/cmd/dump.go index a895785295939..3530c63e545c8 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -44,10 +44,6 @@ It can be used for backup and capture Gitea server image to send to maintainer`, Value: os.TempDir(), Usage: "Temporary dir path", }, - cli.StringFlag{ - Name: "database, d", - Usage: "Specify the database SQL syntax", - }, cli.BoolFlag{ Name: "skip-repository, R", Usage: "Skip the repository dumping", @@ -83,10 +79,9 @@ func runDump(ctx *cli.Context) error { os.Setenv("TMPDIR", tmpWorkDir) } - dbDump := path.Join(tmpWorkDir, "gitea-db.sql") + log.Printf("Packing dump files...") fileName := fmt.Sprintf("gitea-dump-%d.zip", time.Now().Unix()) - log.Printf("Packing dump files...") z, err := zip.Create(fileName) if err != nil { log.Fatalf("Failed to create %s: %v", fileName, err) @@ -106,20 +101,23 @@ func runDump(ctx *cli.Context) error { } } - targetDBType := ctx.String("database") - if len(targetDBType) > 0 && targetDBType != models.DbCfg.Type { - log.Printf("Dumping database %s => %s...", models.DbCfg.Type, targetDBType) - } else { - log.Printf("Dumping database...") + log.Printf("Dumping database...") + + dbDump := path.Join(tmpWorkDir, "database") + if err := os.MkdirAll(dbDump, os.ModePerm); err != nil { + log.Fatalf("Failed to create database dir: %v", err) } - if err := models.DumpDatabase(dbDump, targetDBType); err != nil { + if err := models.DumpDatabaseFixtures(dbDump); err != nil { log.Fatalf("Failed to dump database: %v", err) } - if err := z.AddFile("gitea-db.sql", dbDump); err != nil { - log.Fatalf("Failed to include gitea-db.sql: %v", err) + if err := z.AddDir("database", dbDump); err != nil { + log.Fatalf("Failed to include database: %v", err) } + + log.Printf("Dumping custom directory ... %s", setting.CustomPath) + customDir, err := os.Stat(setting.CustomPath) if err == nil && customDir.IsDir() { if err := z.AddDir("custom", setting.CustomPath); err != nil { @@ -130,7 +128,7 @@ func runDump(ctx *cli.Context) error { } if com.IsExist(setting.AppDataPath) { - log.Printf("Packing data directory...%s", setting.AppDataPath) + log.Printf("Dumping data directory ... %s", setting.AppDataPath) var sessionAbsPath string if setting.SessionConfig.Provider == "file" { @@ -141,8 +139,19 @@ func runDump(ctx *cli.Context) error { } } - if err := z.AddDir("log", setting.LogRootPath); err != nil { - log.Fatalf("Failed to include log: %v", err) + verPath := filepath.Join(tmpWorkDir, "VERSION") + verf, err := os.Create(verPath) + if err != nil { + log.Fatalf("Failed to create version file: %v", err) + } + _, err = verf.WriteString(setting.AppVer) + verf.Close() + if err != nil { + log.Fatalf("Failed to write version to file: %v", err) + } + + if err = z.AddFile("VERSION", verPath); err != nil { + log.Fatalf("Failed to add version file: %v", err) } if err = z.Close(); err != nil { diff --git a/cmd/restore.go b/cmd/restore.go new file mode 100644 index 0000000000000..f6d26047afac9 --- /dev/null +++ b/cmd/restore.go @@ -0,0 +1,153 @@ +// Copyright 2017 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package cmd + +import ( + "errors" + "io/ioutil" + "log" + "os" + "path/filepath" + + "code.gitea.io/gitea/models" + "code.gitea.io/gitea/modules/setting" + + "github.com/Unknwon/cae/zip" + "github.com/Unknwon/com" + "github.com/urfave/cli" +) + +// CmdRestore represents the available restore sub-command. +var CmdRestore = cli.Command{ + Name: "restore", + Usage: "Restore Gitea files and database", + Description: `Restore will restore all data from zip file which dumped from gitea. It will use +the custom config in this dump zip file, this operation will remove all the dest database and repositories.`, + Action: runRestore, + Flags: []cli.Flag{ + cli.StringFlag{ + Name: "config, c", + Value: "custom/conf/app.ini", + Usage: "Custom configuration file path, if empty will use dumped config file", + }, + cli.BoolFlag{ + Name: "verbose, v", + Usage: "Show process details", + }, + cli.StringFlag{ + Name: "tempdir, t", + Value: os.TempDir(), + Usage: "Temporary dir path", + }, + }, +} + +func runRestore(ctx *cli.Context) error { + if len(os.Args) < 3 { + return errors.New("need zip file path") + } + + tmpDir := ctx.String("tempdir") + if _, err := os.Stat(tmpDir); os.IsNotExist(err) { + log.Fatalf("Path does not exist: %s", tmpDir) + } + tmpWorkDir, err := ioutil.TempDir(tmpDir, "gitea-restore-") + if err != nil { + log.Fatalf("Failed to create tmp work directory: %v", err) + } + log.Printf("Creating tmp work dir: %s", tmpWorkDir) + + // work-around #1103 + if os.Getenv("TMPDIR") == "" { + os.Setenv("TMPDIR", tmpWorkDir) + } + + srcPath := os.Args[2] + + zip.Verbose = ctx.Bool("verbose") + log.Printf("Extracting %s to %s", srcPath, tmpWorkDir) + err = zip.ExtractTo(srcPath, tmpWorkDir) + if err != nil { + log.Fatalf("Failed to extract %s to tmp work directory: %v", srcPath, err) + } + + verData, err := ioutil.ReadFile(filepath.Join(tmpWorkDir, "VERSION")) + if err != nil { + log.Fatalf("Failed to extract %s to tmp work directory: %v", srcPath, err) + } + + if setting.AppVer != string(verData) { + log.Fatalf("Expected gitea version to restore is %s, but get %s", string(verData), setting.AppVer) + } + + if ctx.IsSet("config") { + setting.CustomConf = ctx.String("config") + } else { + setting.CustomConf = filepath.Join(tmpWorkDir, "custom", "conf", "app.ini") + } + if !com.IsExist(setting.CustomConf) { + log.Fatalf("Failed to load ini config file from %s", setting.CustomConf) + } + + setting.NewContext() + //setting.CustomPath = filepath.Join(tmpWorkDir, "custom") + setting.NewXORMLogService(false) + models.LoadConfigs() + + err = models.SetEngine() + if err != nil { + log.Fatalf("Failed to SetEngine: %v", err) + } + + err = models.SyncDBStructs() + if err != nil { + log.Fatalf("Failed to SyncDBStructs: %v", err) + } + + log.Printf("Restoring repo dir to %s ...", setting.RepoRootPath) + repoPath := filepath.Join(tmpWorkDir, "repositories") + err = os.RemoveAll(setting.RepoRootPath) + if err != nil { + log.Fatalf("Failed to Remove repo root path %s: %v", setting.RepoRootPath, err) + } + + err = os.Rename(repoPath, setting.RepoRootPath) + if err != nil { + log.Fatalf("Failed to move %s to %s: %v", repoPath, setting.RepoRootPath, err) + } + + log.Printf("Restoring custom dir to %s ...", setting.CustomPath) + customPath := filepath.Join(tmpWorkDir, "custom") + err = os.RemoveAll(setting.CustomPath) + if err != nil { + log.Fatalf("Failed to Remove repo root path %s: %v", setting.CustomPath, err) + } + + err = os.Rename(customPath, setting.CustomPath) + if err != nil { + log.Fatalf("Failed to move %s to %s: %v", customPath, setting.CustomPath, err) + } + + log.Printf("Restoring data dir to %s ...", setting.AppDataPath) + dataPath := filepath.Join(tmpWorkDir, "data") + err = os.RemoveAll(setting.AppDataPath) + if err != nil { + log.Fatalf("Failed to Remove data root path %s: %v", setting.AppDataPath, err) + } + + err = os.Rename(dataPath, setting.AppDataPath) + if err != nil { + log.Fatalf("Failed to move %s to %s: %v", dataPath, setting.AppDataPath, err) + } + + dbPath := filepath.Join(tmpWorkDir, "database") + log.Printf("Restoring database from %s ...", dbPath) + err = models.RestoreDatabaseFixtures(dbPath) + if err != nil { + log.Fatalf("Failed to restore database dir %s: %v", dbPath, err) + } + + return nil +} diff --git a/main.go b/main.go index 976bbdf1f7669..d4ce8371c86ed 100644 --- a/main.go +++ b/main.go @@ -46,6 +46,7 @@ arguments - which can alternatively be run by running the subcommand web.` cmd.CmdServ, cmd.CmdHook, cmd.CmdDump, + cmd.CmdRestore, cmd.CmdCert, cmd.CmdAdmin, cmd.CmdGenerate, diff --git a/models/models.go b/models/models.go index e7ecc67fc5538..8687da0cd67b8 100644 --- a/models/models.go +++ b/models/models.go @@ -9,10 +9,12 @@ import ( "database/sql" "errors" "fmt" + "io/ioutil" "net/url" "os" "path" "path/filepath" + "sort" "strings" "code.gitea.io/gitea/modules/log" @@ -22,6 +24,7 @@ import ( _ "github.com/go-sql-driver/mysql" "github.com/go-xorm/core" "github.com/go-xorm/xorm" + "gopkg.in/yaml.v2" // Needed for the Postgresql driver _ "github.com/lib/pq" @@ -301,6 +304,15 @@ func NewEngine(migrateFunc func(*xorm.Engine) error) (err error) { return nil } +// SyncDBStructs will sync database structs +func SyncDBStructs() error { + if err := x.StoreEngine("InnoDB").Sync2(tables...); err != nil { + return fmt.Errorf("sync database struct error: %v", err) + } + + return nil +} + // Statistic contains the database statistics type Statistic struct { Counter struct { @@ -360,3 +372,113 @@ func DumpDatabase(filePath string, dbType string) error { } return x.DumpTablesToFile(tbs, filePath) } + +// DumpDatabaseFixtures dumps all data from database to fixtures files on dirPath +func DumpDatabaseFixtures(dirPath string) error { + for _, t := range tables { + if err := dumpTableFixtures(t, dirPath); err != nil { + return err + } + } + return nil +} + +func dumpTableFixtures(bean interface{}, dirPath string) error { + table := x.TableInfo(bean) + f, err := os.Create(filepath.Join(dirPath, table.Name+".yml")) + if err != nil { + return err + } + defer f.Close() + + const bufferSize = 100 + var start = 0 + for { + objs, err := x.Table(table.Name).Limit(bufferSize, start).QueryInterface() + if err != nil { + return err + } + if len(objs) == 0 { + break + } + + data, err := yaml.Marshal(objs) + if err != nil { + return err + } + _, err = f.Write(data) + if err != nil { + return err + } + if len(objs) < bufferSize { + break + } + start += len(objs) + } + + return nil +} + +// RestoreDatabaseFixtures restores all data from dir to database +func RestoreDatabaseFixtures(dirPath string) error { + for _, t := range tables { + if err := restoreTableFixtures(t, dirPath); err != nil { + return err + } + } + return nil +} + +func restoreTableFixtures(bean interface{}, dirPath string) error { + table := x.TableInfo(bean) + data, err := ioutil.ReadFile(filepath.Join(dirPath, table.Name+".yml")) + if err != nil { + return err + } + + const bufferSize = 100 + var records = make([]map[string]interface{}, 0, bufferSize*10) + err = yaml.Unmarshal(data, &records) + if err != nil { + return err + } + + if len(records) == 0 { + return nil + } + + var columns = make([]string, 0, len(records[0])) + for k := range records[0] { + columns = append(columns, k) + } + sort.Strings(columns) + + qm := strings.Repeat("?,", len(columns)) + qm = "(" + qm[:len(qm)-1] + ")" + + _, err = x.Exec("DELETE FROM `" + table.Name + "`") + if err != nil { + return err + } + + var sql = "INSERT INTO `" + table.Name + "` (`" + strings.Join(columns, "`,`") + "`) VALUES " + var args = make([]interface{}, 0, bufferSize) + var insertSQLs = make([]string, 0, bufferSize) + for i, vals := range records { + insertSQLs = append(insertSQLs, qm) + for _, colName := range columns { + args = append(args, vals[colName]) + } + + if i+1%100 == 0 || i == len(records)-1 { + _, err = x.Exec(sql+strings.Join(insertSQLs, ","), args...) + if err != nil { + return err + } + insertSQLs = make([]string, 0, bufferSize) + args = make([]interface{}, 0, bufferSize) + } + } + + return err +}