From b7e34e8215092a9fa912af6018ff340cb22cac96 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Mon, 27 Mar 2023 14:39:51 +0800 Subject: [PATCH 01/12] Add backup & restore commands --- assets/go-licenses.json | 5 ++++ cmd/backup.go | 44 ++++++++++++++++++++++++++++ cmd/main.go | 1 + go.mod | 5 +++- go.sum | 5 ++-- models/db/backup.go | 64 +++++++++++++++++++++++++++++++++++++++++ models/db/restore.go | 46 +++++++++++++++++++++++++++++ 7 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 cmd/backup.go create mode 100644 models/db/backup.go create mode 100644 models/db/restore.go diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 8a17148e1bc76..02ad1531279c8 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -989,6 +989,11 @@ "path": "github.com/unknwon/com/LICENSE", "licenseText": "Apache License\nVersion 2.0, January 2004\nhttp://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n\"License\" shall mean the terms and conditions for use, reproduction, and\ndistribution as defined by Sections 1 through 9 of this document.\n\n\"Licensor\" shall mean the copyright owner or entity authorized by the copyright\nowner that is granting the License.\n\n\"Legal Entity\" shall mean the union of the acting entity and all other entities\nthat control, are controlled by, or are under common control with that entity.\nFor the purposes of this definition, \"control\" means (i) the power, direct or\nindirect, to cause the direction or management of such entity, whether by\ncontract or otherwise, or (ii) ownership of fifty percent (50%) or more of the\noutstanding shares, or (iii) beneficial ownership of such entity.\n\n\"You\" (or \"Your\") shall mean an individual or Legal Entity exercising\npermissions granted by this License.\n\n\"Source\" form shall mean the preferred form for making modifications, including\nbut not limited to software source code, documentation source, and configuration\nfiles.\n\n\"Object\" form shall mean any form resulting from mechanical transformation or\ntranslation of a Source form, including but not limited to compiled object code,\ngenerated documentation, and conversions to other media types.\n\n\"Work\" shall mean the work of authorship, whether in Source or Object form, made\navailable under the License, as indicated by a copyright notice that is included\nin or attached to the work (an example is provided in the Appendix below).\n\n\"Derivative Works\" shall mean any work, whether in Source or Object form, that\nis based on (or derived from) the Work and for which the editorial revisions,\nannotations, elaborations, or other modifications represent, as a whole, an\noriginal work of authorship. For the purposes of this License, Derivative Works\nshall not include works that remain separable from, or merely link (or bind by\nname) to the interfaces of, the Work and Derivative Works thereof.\n\n\"Contribution\" shall mean any work of authorship, including the original version\nof the Work and any modifications or additions to that Work or Derivative Works\nthereof, that is intentionally submitted to Licensor for inclusion in the Work\nby the copyright owner or by an individual or Legal Entity authorized to submit\non behalf of the copyright owner. For the purposes of this definition,\n\"submitted\" means any form of electronic, verbal, or written communication sent\nto the Licensor or its representatives, including but not limited to\ncommunication on electronic mailing lists, source code control systems, and\nissue tracking systems that are managed by, or on behalf of, the Licensor for\nthe purpose of discussing and improving the Work, but excluding communication\nthat is conspicuously marked or otherwise designated in writing by the copyright\nowner as \"Not a Contribution.\"\n\n\"Contributor\" shall mean Licensor and any individual or Legal Entity on behalf\nof whom a Contribution has been received by Licensor and subsequently\nincorporated within the Work.\n\n2. Grant of Copyright License.\n\nSubject to the terms and conditions of this License, each Contributor hereby\ngrants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,\nirrevocable copyright license to reproduce, prepare Derivative Works of,\npublicly display, publicly perform, sublicense, and distribute the Work and such\nDerivative Works in Source or Object form.\n\n3. Grant of Patent License.\n\nSubject to the terms and conditions of this License, each Contributor hereby\ngrants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,\nirrevocable (except as stated in this section) patent license to make, have\nmade, use, offer to sell, sell, import, and otherwise transfer the Work, where\nsuch license applies only to those patent claims licensable by such Contributor\nthat are necessarily infringed by their Contribution(s) alone or by combination\nof their Contribution(s) with the Work to which such Contribution(s) was\nsubmitted. If You institute patent litigation against any entity (including a\ncross-claim or counterclaim in a lawsuit) alleging that the Work or a\nContribution incorporated within the Work constitutes direct or contributory\npatent infringement, then any patent licenses granted to You under this License\nfor that Work shall terminate as of the date such litigation is filed.\n\n4. Redistribution.\n\nYou may reproduce and distribute copies of the Work or Derivative Works thereof\nin any medium, with or without modifications, and in Source or Object form,\nprovided that You meet the following conditions:\n\nYou must give any other recipients of the Work or Derivative Works a copy of\nthis License; and\nYou must cause any modified files to carry prominent notices stating that You\nchanged the files; and\nYou must retain, in the Source form of any Derivative Works that You distribute,\nall copyright, patent, trademark, and attribution notices from the Source form\nof the Work, excluding those notices that do not pertain to any part of the\nDerivative Works; and\nIf the Work includes a \"NOTICE\" text file as part of its distribution, then any\nDerivative Works that You distribute must include a readable copy of the\nattribution notices contained within such NOTICE file, excluding those notices\nthat do not pertain to any part of the Derivative Works, in at least one of the\nfollowing places: within a NOTICE text file distributed as part of the\nDerivative Works; within the Source form or documentation, if provided along\nwith the Derivative Works; or, within a display generated by the Derivative\nWorks, if and wherever such third-party notices normally appear. The contents of\nthe NOTICE file are for informational purposes only and do not modify the\nLicense. You may add Your own attribution notices within Derivative Works that\nYou distribute, alongside or as an addendum to the NOTICE text from the Work,\nprovided that such additional attribution notices cannot be construed as\nmodifying the License.\nYou may add Your own copyright statement to Your modifications and may provide\nadditional or different license terms and conditions for use, reproduction, or\ndistribution of Your modifications, or for any such Derivative Works as a whole,\nprovided Your use, reproduction, and distribution of the Work otherwise complies\nwith the conditions stated in this License.\n\n5. Submission of Contributions.\n\nUnless You explicitly state otherwise, any Contribution intentionally submitted\nfor inclusion in the Work by You to the Licensor shall be under the terms and\nconditions of this License, without any additional terms or conditions.\nNotwithstanding the above, nothing herein shall supersede or modify the terms of\nany separate license agreement you may have executed with Licensor regarding\nsuch Contributions.\n\n6. Trademarks.\n\nThis License does not grant permission to use the trade names, trademarks,\nservice marks, or product names of the Licensor, except as required for\nreasonable and customary use in describing the origin of the Work and\nreproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty.\n\nUnless required by applicable law or agreed to in writing, Licensor provides the\nWork (and each Contributor provides its Contributions) on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,\nincluding, without limitation, any warranties or conditions of TITLE,\nNON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are\nsolely responsible for determining the appropriateness of using or\nredistributing the Work and assume any risks associated with Your exercise of\npermissions under this License.\n\n8. Limitation of Liability.\n\nIn no event and under no legal theory, whether in tort (including negligence),\ncontract, or otherwise, unless required by applicable law (such as deliberate\nand grossly negligent acts) or agreed to in writing, shall any Contributor be\nliable to You for damages, including any direct, indirect, special, incidental,\nor consequential damages of any character arising as a result of this License or\nout of the use or inability to use the Work (including but not limited to\ndamages for loss of goodwill, work stoppage, computer failure or malfunction, or\nany and all other commercial damages or losses), even if such Contributor has\nbeen advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability.\n\nWhile redistributing the Work or Derivative Works thereof, You may choose to\noffer, and charge a fee for, acceptance of support, warranty, indemnity, or\nother liability obligations and/or rights consistent with this License. However,\nin accepting such obligations, You may act only on Your own behalf and on Your\nsole responsibility, not on behalf of any other Contributor, and only if You\nagree to indemnify, defend, and hold each Contributor harmless for any liability\nincurred by, or claims asserted against, such Contributor by reason of your\naccepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work\n\nTo apply the Apache License to your work, attach the following boilerplate\nnotice, with the fields enclosed by brackets \"[]\" replaced with your own\nidentifying information. (Don't include the brackets!) The text should be\nenclosed in the appropriate comment syntax for the file format. We also\nrecommend that a file or class name and description of purpose be included on\nthe same \"printed page\" as the copyright notice for easier identification within\nthird-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License." }, + { + "name": "github.com/urfave/cli", + "path": "github.com/urfave/cli/LICENSE", + "licenseText": "MIT License\n\nCopyright (c) 2023 Jeremy Saenz \u0026 Contributors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" + }, { "name": "github.com/urfave/cli/v2", "path": "github.com/urfave/cli/v2/LICENSE", diff --git a/cmd/backup.go b/cmd/backup.go new file mode 100644 index 0000000000000..635de835e763d --- /dev/null +++ b/cmd/backup.go @@ -0,0 +1,44 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package cmd + +import ( + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/setting" + + "github.com/urfave/cli/v2" +) + +// CmdBackup backup all data from database to fixtures files on dirPath +var CmdBackup = &cli.Command{ + Name: "backup", + Usage: "Backup the Gitea database", + Description: "A command to backup all data from database to fixtures files on dirPath", + Action: runBackup, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "dir-path", + Value: "", + Usage: "Directory path to save fixtures files", + }, + }, +} + +func runBackup(ctx *cli.Context) error { + stdCtx, cancel := installSignals() + defer cancel() + + if err := initDB(stdCtx); err != nil { + return err + } + + log.Info("AppPath: %s", setting.AppPath) + log.Info("AppWorkPath: %s", setting.AppWorkPath) + log.Info("Custom path: %s", setting.CustomPath) + log.Info("Log path: %s", setting.Log.RootPath) + log.Info("Configuration file: %s", setting.CustomConf) + + return db.BackupDatabaseAsFixtures(ctx.String("dir-path")) +} diff --git a/cmd/main.go b/cmd/main.go index feda41e68b24a..d1d1ea9c1e5d4 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -142,6 +142,7 @@ func NewMainApp(version, versionExtra string) *cli.App { CmdDumpRepository, CmdRestoreRepository, CmdActions, + CmdBackup, cmdHelp(), // the "help" sub-command was used to show the more information for "work path" and "custom config" } diff --git a/go.mod b/go.mod index f15b3c3234e06..01a1d4a36bc10 100644 --- a/go.mod +++ b/go.mod @@ -99,6 +99,7 @@ require ( github.com/syndtr/goleveldb v1.0.0 github.com/tstranex/u2f v1.0.0 github.com/ulikunitz/xz v0.5.11 + github.com/urfave/cli v1.22.14 github.com/urfave/cli/v2 v2.25.7 github.com/xanzy/go-gitlab v0.91.0 github.com/xeipuuv/gojsonschema v1.2.0 @@ -117,6 +118,7 @@ require ( google.golang.org/protobuf v1.31.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/ini.v1 v1.67.0 + gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 mvdan.cc/xurls/v2 v2.5.0 strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 @@ -296,7 +298,6 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect ) replace github.com/hashicorp/go-version => github.com/6543/go-version v1.3.1 @@ -305,6 +306,8 @@ replace github.com/shurcooL/vfsgen => github.com/lunny/vfsgen v0.0.0-20220105142 replace github.com/nektos/act => gitea.com/gitea/act v0.243.4 +replace xorm.io/xorm => ../../xorm/xorm + exclude github.com/gofrs/uuid v3.2.0+incompatible exclude github.com/gofrs/uuid v4.0.0+incompatible diff --git a/go.sum b/go.sum index 3e2bc9a754c61..e5713c95a6451 100644 --- a/go.sum +++ b/go.sum @@ -82,6 +82,7 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/ch-go v0.58.2 h1:jSm2szHbT9MCAB1rJ3WuCJqmGLi5UTjlNu+f530UTS0= github.com/ClickHouse/ch-go v0.58.2/go.mod h1:Ap/0bEmiLa14gYjCiRkYGbXvbe8vwdrfTYWhsuQ99aw= @@ -1021,6 +1022,8 @@ github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o github.com/unknwon/com v0.0.0-20190804042917-757f69c95f3e/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnlCXM= github.com/unknwon/com v1.0.1 h1:3d1LTxD+Lnf3soQiD4Cp/0BRB+Rsa/+RTvz8GMMzIXs= github.com/unknwon/com v1.0.1/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnlCXM= +github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= +github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= @@ -1637,5 +1640,3 @@ strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251/go.mod h1: xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= xorm.io/builder v0.3.13 h1:a3jmiVVL19psGeXx8GIurTp7p0IIgqeDmwhcR6BAOAo= xorm.io/builder v0.3.13/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= -xorm.io/xorm v1.3.3-0.20230725140238-59b727260d35 h1:qjhRBMFRr3IdT2YR8VByOMfko8vxINiOPeOc267zuPI= -xorm.io/xorm v1.3.3-0.20230725140238-59b727260d35/go.mod h1:qFJGFoVYbbIdnz2vaL5OxSQ2raleMpyRRalnq3n9OJo= diff --git a/models/db/backup.go b/models/db/backup.go new file mode 100644 index 0000000000000..c563ecc514014 --- /dev/null +++ b/models/db/backup.go @@ -0,0 +1,64 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package db + +import ( + "os" + "path/filepath" + + "gopkg.in/yaml.v2" +) + +// BackupDatabaseAsFixtures backup all data from database to fixtures files on dirPath +func BackupDatabaseAsFixtures(dirPath string) error { + if err := os.MkdirAll(dirPath, os.ModePerm); err != nil { + return err + } + + for _, t := range tables { + if err := backupTableFixtures(t, dirPath); err != nil { + return err + } + } + return nil +} + +func backupTableFixtures(bean interface{}, dirPath string) error { + table, err := x.TableInfo(bean) + if err != nil { + return err + } + f, err := os.Create(filepath.Join(dirPath, table.Name+".yml")) + if err != nil { + return err + } + defer f.Close() + + const bufferSize = 100 + start := 0 + for { + objs, err := x.Table(table.Name).Limit(bufferSize, start).QueryInterface() + if err != nil { + return err + } + if len(objs) == 0 { + break + } + + data, err := yaml.Marshal(objs) + if err != nil { + return err + } + _, err = f.Write(data) + if err != nil { + return err + } + if len(objs) < bufferSize { + break + } + start += len(objs) + } + + return nil +} diff --git a/models/db/restore.go b/models/db/restore.go new file mode 100644 index 0000000000000..abf76e2d8c1d5 --- /dev/null +++ b/models/db/restore.go @@ -0,0 +1,46 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package db + +import ( + "fmt" + + "github.com/go-testfixtures/testfixtures/v3" + "xorm.io/xorm/schemas" +) + +func RestoreDatabase(dirPath string) error { + testfiles := testfixtures.Directory(dirPath) + dialect := "unknown" + switch x.Dialect().URI().DBType { + case schemas.POSTGRES: + dialect = "postgres" + case schemas.MYSQL: + dialect = "mysql" + case schemas.MSSQL: + dialect = "mssql" + case schemas.SQLITE: + dialect = "sqlite3" + default: + return fmt.Errorf("Unsupported RDBMS for integration tests") + } + + loaderOptions := []func(loader *testfixtures.Loader) error{ + testfixtures.Database(x.DB().DB), + testfixtures.Dialect(dialect), + testfixtures.DangerousSkipTestDatabaseCheck(), + testfiles, + } + + if x.Dialect().URI().DBType == schemas.POSTGRES { + loaderOptions = append(loaderOptions, testfixtures.SkipResetSequences()) + } + + fixtures, err := testfixtures.New(loaderOptions...) + if err != nil { + return err + } + + return fixtures.Load() +} From d0dfe98901d7f40c8ddfe8377ae5ddae5902ef72 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 6 Sep 2023 19:09:06 +0800 Subject: [PATCH 02/12] Fix go.mod and license file --- assets/go-licenses.json | 5 ----- go.mod | 1 - go.sum | 3 --- 3 files changed, 9 deletions(-) diff --git a/assets/go-licenses.json b/assets/go-licenses.json index 02ad1531279c8..8a17148e1bc76 100644 --- a/assets/go-licenses.json +++ b/assets/go-licenses.json @@ -989,11 +989,6 @@ "path": "github.com/unknwon/com/LICENSE", "licenseText": "Apache License\nVersion 2.0, January 2004\nhttp://www.apache.org/licenses/\n\nTERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n1. Definitions.\n\n\"License\" shall mean the terms and conditions for use, reproduction, and\ndistribution as defined by Sections 1 through 9 of this document.\n\n\"Licensor\" shall mean the copyright owner or entity authorized by the copyright\nowner that is granting the License.\n\n\"Legal Entity\" shall mean the union of the acting entity and all other entities\nthat control, are controlled by, or are under common control with that entity.\nFor the purposes of this definition, \"control\" means (i) the power, direct or\nindirect, to cause the direction or management of such entity, whether by\ncontract or otherwise, or (ii) ownership of fifty percent (50%) or more of the\noutstanding shares, or (iii) beneficial ownership of such entity.\n\n\"You\" (or \"Your\") shall mean an individual or Legal Entity exercising\npermissions granted by this License.\n\n\"Source\" form shall mean the preferred form for making modifications, including\nbut not limited to software source code, documentation source, and configuration\nfiles.\n\n\"Object\" form shall mean any form resulting from mechanical transformation or\ntranslation of a Source form, including but not limited to compiled object code,\ngenerated documentation, and conversions to other media types.\n\n\"Work\" shall mean the work of authorship, whether in Source or Object form, made\navailable under the License, as indicated by a copyright notice that is included\nin or attached to the work (an example is provided in the Appendix below).\n\n\"Derivative Works\" shall mean any work, whether in Source or Object form, that\nis based on (or derived from) the Work and for which the editorial revisions,\nannotations, elaborations, or other modifications represent, as a whole, an\noriginal work of authorship. For the purposes of this License, Derivative Works\nshall not include works that remain separable from, or merely link (or bind by\nname) to the interfaces of, the Work and Derivative Works thereof.\n\n\"Contribution\" shall mean any work of authorship, including the original version\nof the Work and any modifications or additions to that Work or Derivative Works\nthereof, that is intentionally submitted to Licensor for inclusion in the Work\nby the copyright owner or by an individual or Legal Entity authorized to submit\non behalf of the copyright owner. For the purposes of this definition,\n\"submitted\" means any form of electronic, verbal, or written communication sent\nto the Licensor or its representatives, including but not limited to\ncommunication on electronic mailing lists, source code control systems, and\nissue tracking systems that are managed by, or on behalf of, the Licensor for\nthe purpose of discussing and improving the Work, but excluding communication\nthat is conspicuously marked or otherwise designated in writing by the copyright\nowner as \"Not a Contribution.\"\n\n\"Contributor\" shall mean Licensor and any individual or Legal Entity on behalf\nof whom a Contribution has been received by Licensor and subsequently\nincorporated within the Work.\n\n2. Grant of Copyright License.\n\nSubject to the terms and conditions of this License, each Contributor hereby\ngrants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,\nirrevocable copyright license to reproduce, prepare Derivative Works of,\npublicly display, publicly perform, sublicense, and distribute the Work and such\nDerivative Works in Source or Object form.\n\n3. Grant of Patent License.\n\nSubject to the terms and conditions of this License, each Contributor hereby\ngrants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,\nirrevocable (except as stated in this section) patent license to make, have\nmade, use, offer to sell, sell, import, and otherwise transfer the Work, where\nsuch license applies only to those patent claims licensable by such Contributor\nthat are necessarily infringed by their Contribution(s) alone or by combination\nof their Contribution(s) with the Work to which such Contribution(s) was\nsubmitted. If You institute patent litigation against any entity (including a\ncross-claim or counterclaim in a lawsuit) alleging that the Work or a\nContribution incorporated within the Work constitutes direct or contributory\npatent infringement, then any patent licenses granted to You under this License\nfor that Work shall terminate as of the date such litigation is filed.\n\n4. Redistribution.\n\nYou may reproduce and distribute copies of the Work or Derivative Works thereof\nin any medium, with or without modifications, and in Source or Object form,\nprovided that You meet the following conditions:\n\nYou must give any other recipients of the Work or Derivative Works a copy of\nthis License; and\nYou must cause any modified files to carry prominent notices stating that You\nchanged the files; and\nYou must retain, in the Source form of any Derivative Works that You distribute,\nall copyright, patent, trademark, and attribution notices from the Source form\nof the Work, excluding those notices that do not pertain to any part of the\nDerivative Works; and\nIf the Work includes a \"NOTICE\" text file as part of its distribution, then any\nDerivative Works that You distribute must include a readable copy of the\nattribution notices contained within such NOTICE file, excluding those notices\nthat do not pertain to any part of the Derivative Works, in at least one of the\nfollowing places: within a NOTICE text file distributed as part of the\nDerivative Works; within the Source form or documentation, if provided along\nwith the Derivative Works; or, within a display generated by the Derivative\nWorks, if and wherever such third-party notices normally appear. The contents of\nthe NOTICE file are for informational purposes only and do not modify the\nLicense. You may add Your own attribution notices within Derivative Works that\nYou distribute, alongside or as an addendum to the NOTICE text from the Work,\nprovided that such additional attribution notices cannot be construed as\nmodifying the License.\nYou may add Your own copyright statement to Your modifications and may provide\nadditional or different license terms and conditions for use, reproduction, or\ndistribution of Your modifications, or for any such Derivative Works as a whole,\nprovided Your use, reproduction, and distribution of the Work otherwise complies\nwith the conditions stated in this License.\n\n5. Submission of Contributions.\n\nUnless You explicitly state otherwise, any Contribution intentionally submitted\nfor inclusion in the Work by You to the Licensor shall be under the terms and\nconditions of this License, without any additional terms or conditions.\nNotwithstanding the above, nothing herein shall supersede or modify the terms of\nany separate license agreement you may have executed with Licensor regarding\nsuch Contributions.\n\n6. Trademarks.\n\nThis License does not grant permission to use the trade names, trademarks,\nservice marks, or product names of the Licensor, except as required for\nreasonable and customary use in describing the origin of the Work and\nreproducing the content of the NOTICE file.\n\n7. Disclaimer of Warranty.\n\nUnless required by applicable law or agreed to in writing, Licensor provides the\nWork (and each Contributor provides its Contributions) on an \"AS IS\" BASIS,\nWITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,\nincluding, without limitation, any warranties or conditions of TITLE,\nNON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are\nsolely responsible for determining the appropriateness of using or\nredistributing the Work and assume any risks associated with Your exercise of\npermissions under this License.\n\n8. Limitation of Liability.\n\nIn no event and under no legal theory, whether in tort (including negligence),\ncontract, or otherwise, unless required by applicable law (such as deliberate\nand grossly negligent acts) or agreed to in writing, shall any Contributor be\nliable to You for damages, including any direct, indirect, special, incidental,\nor consequential damages of any character arising as a result of this License or\nout of the use or inability to use the Work (including but not limited to\ndamages for loss of goodwill, work stoppage, computer failure or malfunction, or\nany and all other commercial damages or losses), even if such Contributor has\nbeen advised of the possibility of such damages.\n\n9. Accepting Warranty or Additional Liability.\n\nWhile redistributing the Work or Derivative Works thereof, You may choose to\noffer, and charge a fee for, acceptance of support, warranty, indemnity, or\nother liability obligations and/or rights consistent with this License. However,\nin accepting such obligations, You may act only on Your own behalf and on Your\nsole responsibility, not on behalf of any other Contributor, and only if You\nagree to indemnify, defend, and hold each Contributor harmless for any liability\nincurred by, or claims asserted against, such Contributor by reason of your\naccepting any such warranty or additional liability.\n\nEND OF TERMS AND CONDITIONS\n\nAPPENDIX: How to apply the Apache License to your work\n\nTo apply the Apache License to your work, attach the following boilerplate\nnotice, with the fields enclosed by brackets \"[]\" replaced with your own\nidentifying information. (Don't include the brackets!) The text should be\nenclosed in the appropriate comment syntax for the file format. We also\nrecommend that a file or class name and description of purpose be included on\nthe same \"printed page\" as the copyright notice for easier identification within\nthird-party archives.\n\n Copyright [yyyy] [name of copyright owner]\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License." }, - { - "name": "github.com/urfave/cli", - "path": "github.com/urfave/cli/LICENSE", - "licenseText": "MIT License\n\nCopyright (c) 2023 Jeremy Saenz \u0026 Contributors\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE.\n" - }, { "name": "github.com/urfave/cli/v2", "path": "github.com/urfave/cli/v2/LICENSE", diff --git a/go.mod b/go.mod index 01a1d4a36bc10..559c71f5a06f1 100644 --- a/go.mod +++ b/go.mod @@ -99,7 +99,6 @@ require ( github.com/syndtr/goleveldb v1.0.0 github.com/tstranex/u2f v1.0.0 github.com/ulikunitz/xz v0.5.11 - github.com/urfave/cli v1.22.14 github.com/urfave/cli/v2 v2.25.7 github.com/xanzy/go-gitlab v0.91.0 github.com/xeipuuv/gojsonschema v1.2.0 diff --git a/go.sum b/go.sum index e5713c95a6451..9bf6943fe3f3f 100644 --- a/go.sum +++ b/go.sum @@ -82,7 +82,6 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ClickHouse/ch-go v0.58.2 h1:jSm2szHbT9MCAB1rJ3WuCJqmGLi5UTjlNu+f530UTS0= github.com/ClickHouse/ch-go v0.58.2/go.mod h1:Ap/0bEmiLa14gYjCiRkYGbXvbe8vwdrfTYWhsuQ99aw= @@ -1022,8 +1021,6 @@ github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0o github.com/unknwon/com v0.0.0-20190804042917-757f69c95f3e/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnlCXM= github.com/unknwon/com v1.0.1 h1:3d1LTxD+Lnf3soQiD4Cp/0BRB+Rsa/+RTvz8GMMzIXs= github.com/unknwon/com v1.0.1/go.mod h1:tOOxU81rwgoCLoOVVPHb6T/wt8HZygqH5id+GNnlCXM= -github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= -github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= From a90e4b768b7bd6cf2dd9f066b0fcf115c30a882a Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 6 Sep 2023 19:10:28 +0800 Subject: [PATCH 03/12] Fix go.mod --- go.mod | 4 +--- go.sum | 2 ++ models/db/backup.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 559c71f5a06f1..f15b3c3234e06 100644 --- a/go.mod +++ b/go.mod @@ -117,7 +117,6 @@ require ( google.golang.org/protobuf v1.31.0 gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df gopkg.in/ini.v1 v1.67.0 - gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 mvdan.cc/xurls/v2 v2.5.0 strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 @@ -297,6 +296,7 @@ require ( google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect gopkg.in/warnings.v0 v0.1.2 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) replace github.com/hashicorp/go-version => github.com/6543/go-version v1.3.1 @@ -305,8 +305,6 @@ replace github.com/shurcooL/vfsgen => github.com/lunny/vfsgen v0.0.0-20220105142 replace github.com/nektos/act => gitea.com/gitea/act v0.243.4 -replace xorm.io/xorm => ../../xorm/xorm - exclude github.com/gofrs/uuid v3.2.0+incompatible exclude github.com/gofrs/uuid v4.0.0+incompatible diff --git a/go.sum b/go.sum index 9bf6943fe3f3f..3e2bc9a754c61 100644 --- a/go.sum +++ b/go.sum @@ -1637,3 +1637,5 @@ strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251/go.mod h1: xorm.io/builder v0.3.11-0.20220531020008-1bd24a7dc978/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= xorm.io/builder v0.3.13 h1:a3jmiVVL19psGeXx8GIurTp7p0IIgqeDmwhcR6BAOAo= xorm.io/builder v0.3.13/go.mod h1:aUW0S9eb9VCaPohFCH3j7czOx1PMW3i1HrSzbLYGBSE= +xorm.io/xorm v1.3.3-0.20230725140238-59b727260d35 h1:qjhRBMFRr3IdT2YR8VByOMfko8vxINiOPeOc267zuPI= +xorm.io/xorm v1.3.3-0.20230725140238-59b727260d35/go.mod h1:qFJGFoVYbbIdnz2vaL5OxSQ2raleMpyRRalnq3n9OJo= diff --git a/models/db/backup.go b/models/db/backup.go index c563ecc514014..280c9fc471e6e 100644 --- a/models/db/backup.go +++ b/models/db/backup.go @@ -7,7 +7,7 @@ import ( "os" "path/filepath" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // BackupDatabaseAsFixtures backup all data from database to fixtures files on dirPath From c597a29cb536ad9a39375454358ebec80da68a70 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 7 Sep 2023 10:07:28 +0800 Subject: [PATCH 04/12] Fix backup --- models/db/backup.go | 24 +++++++++++++++++------- models/db/restore.go | 3 ++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/models/db/backup.go b/models/db/backup.go index 280c9fc471e6e..6167cf611fc91 100644 --- a/models/db/backup.go +++ b/models/db/backup.go @@ -46,14 +46,24 @@ func backupTableFixtures(bean interface{}, dirPath string) error { break } - data, err := yaml.Marshal(objs) - if err != nil { - return err - } - _, err = f.Write(data) - if err != nil { - return err + for _, obj := range objs { + for k, v := range obj { + if vv, ok := v.([]byte); ok { + obj[k] = string(vv) + } + } + bs, err := yaml.Marshal([]any{obj}) + if err != nil { + return err + } + if _, err := f.Write(bs); err != nil { + return err + } + if _, err := f.Write([]byte{'\n'}); err != nil { + return err + } } + if len(objs) < bufferSize { break } diff --git a/models/db/restore.go b/models/db/restore.go index abf76e2d8c1d5..091158bfe649b 100644 --- a/models/db/restore.go +++ b/models/db/restore.go @@ -7,12 +7,13 @@ import ( "fmt" "github.com/go-testfixtures/testfixtures/v3" + "xorm.io/xorm/schemas" ) func RestoreDatabase(dirPath string) error { testfiles := testfixtures.Directory(dirPath) - dialect := "unknown" + var dialect string switch x.Dialect().URI().DBType { case schemas.POSTGRES: dialect = "postgres" From 7311ed9365553cd0d5451ceadb75bc6e39c2e8bc Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 7 Sep 2023 10:23:25 +0800 Subject: [PATCH 05/12] nit improvement --- models/db/backup.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/models/db/backup.go b/models/db/backup.go index 6167cf611fc91..8aa586c11e3f4 100644 --- a/models/db/backup.go +++ b/models/db/backup.go @@ -8,6 +8,7 @@ import ( "path/filepath" "gopkg.in/yaml.v3" + "xorm.io/xorm" ) // BackupDatabaseAsFixtures backup all data from database to fixtures files on dirPath @@ -17,15 +18,15 @@ func BackupDatabaseAsFixtures(dirPath string) error { } for _, t := range tables { - if err := backupTableFixtures(t, dirPath); err != nil { + if err := backupTableFixtures(x, t, dirPath); err != nil { return err } } return nil } -func backupTableFixtures(bean interface{}, dirPath string) error { - table, err := x.TableInfo(bean) +func backupTableFixtures(e *xorm.Engine, bean interface{}, dirPath string) error { + table, err := e.TableInfo(bean) if err != nil { return err } @@ -38,7 +39,7 @@ func backupTableFixtures(bean interface{}, dirPath string) error { const bufferSize = 100 start := 0 for { - objs, err := x.Table(table.Name).Limit(bufferSize, start).QueryInterface() + objs, err := e.Table(table.Name).Limit(bufferSize, start).QueryInterface() if err != nil { return err } @@ -48,18 +49,19 @@ func backupTableFixtures(bean interface{}, dirPath string) error { for _, obj := range objs { for k, v := range obj { + // convert bytes to string if vv, ok := v.([]byte); ok { obj[k] = string(vv) } } - bs, err := yaml.Marshal([]any{obj}) + bs, err := yaml.Marshal([]any{obj}) // with []any{} to ensure generated a list if err != nil { return err } if _, err := f.Write(bs); err != nil { return err } - if _, err := f.Write([]byte{'\n'}); err != nil { + if _, err := f.Write([]byte{'\n'}); err != nil { // generate a blank line for human readable return err } } From ac42e952ea9533384d925e7a4d3866f5b4ee62c8 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 7 Sep 2023 13:44:36 +0800 Subject: [PATCH 06/12] Finish the implementation of backup/restore database --- cmd/backup.go | 31 ++++++++++++++++++ cmd/main.go | 1 + models/db/backup.go | 78 ++++++++++++++++++++++++++++++++++++++++++-- models/db/restore.go | 5 ++- 4 files changed, 110 insertions(+), 5 deletions(-) diff --git a/cmd/backup.go b/cmd/backup.go index 635de835e763d..fcb9fe6a1e073 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -42,3 +42,34 @@ func runBackup(ctx *cli.Context) error { return db.BackupDatabaseAsFixtures(ctx.String("dir-path")) } + +var CmdRestore = &cli.Command{ + Name: "restore", + Usage: "Restore the Gitea database", + Description: "A command to restore all data from fixtures files on dirPath to database", + Action: runRestore, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "dir-path", + Value: "", + Usage: "Directory path to load fixtures files", + }, + }, +} + +func runRestore(ctx *cli.Context) error { + stdCtx, cancel := installSignals() + defer cancel() + + if err := initDB(stdCtx); err != nil { + return err + } + + log.Info("AppPath: %s", setting.AppPath) + log.Info("AppWorkPath: %s", setting.AppWorkPath) + log.Info("Custom path: %s", setting.CustomPath) + log.Info("Log path: %s", setting.Log.RootPath) + log.Info("Configuration file: %s", setting.CustomConf) + + return db.RestoreDatabase(ctx.String("dir-path")) +} diff --git a/cmd/main.go b/cmd/main.go index d1d1ea9c1e5d4..3f72804022ecf 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -143,6 +143,7 @@ func NewMainApp(version, versionExtra string) *cli.App { CmdRestoreRepository, CmdActions, CmdBackup, + CmdRestore, cmdHelp(), // the "help" sub-command was used to show the more information for "work path" and "custom config" } diff --git a/models/db/backup.go b/models/db/backup.go index 8aa586c11e3f4..0ab233566311f 100644 --- a/models/db/backup.go +++ b/models/db/backup.go @@ -4,6 +4,8 @@ package db import ( + "encoding/base64" + "fmt" "os" "path/filepath" @@ -25,11 +27,62 @@ func BackupDatabaseAsFixtures(dirPath string) error { return nil } +func toNode(tableName, col string, v interface{}) *yaml.Node { + switch vv := v.(type) { + case string: + if tableName == "action_task" && col == "log_indexes" { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!binary", + Value: base64.StdEncoding.EncodeToString([]byte(vv)), + } + } + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: vv, + } + case []byte: + if tableName == "action_task" && col == "log_indexes" { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!binary", + Value: base64.StdEncoding.EncodeToString(vv), + } + } + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: string(vv), + } + case int, int64, int32, int8, int16, uint, uint64: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!int", + Value: fmt.Sprintf("%d", vv), + } + case float64, float32: + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!float", + Value: fmt.Sprintf("%f", vv), + } + default: + panic(fmt.Sprintf("unknow type %#v", v)) + } +} + func backupTableFixtures(e *xorm.Engine, bean interface{}, dirPath string) error { table, err := e.TableInfo(bean) if err != nil { return err } + if isEmpty, err := e.IsTableEmpty(table.Name); err != nil { + return err + } else if isEmpty { + return nil + } + f, err := os.Create(filepath.Join(dirPath, table.Name+".yml")) if err != nil { return err @@ -38,6 +91,7 @@ func backupTableFixtures(e *xorm.Engine, bean interface{}, dirPath string) error const bufferSize = 100 start := 0 + for { objs, err := e.Table(table.Name).Limit(bufferSize, start).QueryInterface() if err != nil { @@ -54,9 +108,29 @@ func backupTableFixtures(e *xorm.Engine, bean interface{}, dirPath string) error obj[k] = string(vv) } } - bs, err := yaml.Marshal([]any{obj}) // with []any{} to ensure generated a list + + node := yaml.Node{ + Kind: yaml.MappingNode, + } + for _, col := range table.ColumnsSeq() { + v, ok := obj[col] + if !ok { + return fmt.Errorf("column %s has no value from database", col) + } + + node.Content = append(node.Content, + &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: col, + }, + toNode(table.Name, col, v), + ) + } + + bs, err := yaml.Marshal([]*yaml.Node{&node}) // with []any{} to ensure generated a list if err != nil { - return err + return fmt.Errorf("marshal table %s record %#v %#v failed: %v", table.Name, obj, node.Content[1], err) } if _, err := f.Write(bs); err != nil { return err diff --git a/models/db/restore.go b/models/db/restore.go index 091158bfe649b..08756a40f4c4c 100644 --- a/models/db/restore.go +++ b/models/db/restore.go @@ -12,7 +12,6 @@ import ( ) func RestoreDatabase(dirPath string) error { - testfiles := testfixtures.Directory(dirPath) var dialect string switch x.Dialect().URI().DBType { case schemas.POSTGRES: @@ -24,14 +23,14 @@ func RestoreDatabase(dirPath string) error { case schemas.SQLITE: dialect = "sqlite3" default: - return fmt.Errorf("Unsupported RDBMS for integration tests") + return fmt.Errorf("unsupported RDBMS for integration tests") } loaderOptions := []func(loader *testfixtures.Loader) error{ testfixtures.Database(x.DB().DB), testfixtures.Dialect(dialect), testfixtures.DangerousSkipTestDatabaseCheck(), - testfiles, + testfixtures.Directory(dirPath), } if x.Dialect().URI().DBType == schemas.POSTGRES { From 8fbcc3e5d45a356f0a487ccede31a4a1d72cdb34 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Thu, 7 Sep 2023 16:04:12 +0800 Subject: [PATCH 07/12] Fix check --- models/db/restore.go | 1 - 1 file changed, 1 deletion(-) diff --git a/models/db/restore.go b/models/db/restore.go index 08756a40f4c4c..cf03faf85a34a 100644 --- a/models/db/restore.go +++ b/models/db/restore.go @@ -7,7 +7,6 @@ import ( "fmt" "github.com/go-testfixtures/testfixtures/v3" - "xorm.io/xorm/schemas" ) From e96abe19d987620eebac2c0b677f755ef64956b9 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 8 Sep 2023 22:06:42 +0800 Subject: [PATCH 08/12] add test --- models/db/backup_test.go | 49 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 models/db/backup_test.go diff --git a/models/db/backup_test.go b/models/db/backup_test.go new file mode 100644 index 0000000000000..0b58db6d17025 --- /dev/null +++ b/models/db/backup_test.go @@ -0,0 +1,49 @@ +// Copyright 2023 The Gitea Authors. All rights reserved. +// SPDX-License-Identifier: MIT + +package db_test + +import ( + "os" + "path/filepath" + "testing" + + "code.gitea.io/gitea/models/db" + "code.gitea.io/gitea/models/unittest" + + "github.com/stretchr/testify/assert" +) + +func TestBackupRestore(t *testing.T) { + assert.NoError(t, unittest.PrepareTestDatabase()) + + d, err := os.MkdirTemp(os.TempDir(), "backup_restore") + assert.NoError(t, err) + + assert.NoError(t, db.BackupDatabaseAsFixtures(d)) + + f, err := os.Open(d) + assert.NoError(t, err) + defer f.Close() + + entries, err := f.ReadDir(0) + assert.NoError(t, err) + for _, entry := range entries { + if entry.IsDir() { + continue + } + + fileEqual(t, filepath.Join("..", "fixtures", entry.Name()), filepath.Join(d, entry.Name())) + } + + // assert.NoError(t, db.RestoreDatabase(d)) +} + +func fileEqual(t *testing.T, a, b string) { + bs1, err := os.ReadFile(a) + assert.NoError(t, err) + + bs2, err := os.ReadFile(b) + assert.NoError(t, err) + assert.EqualValues(t, bs1, bs2) +} From 31b3527516090aca718b59c0b67338eb4baf925f Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Fri, 8 Sep 2023 23:06:09 +0800 Subject: [PATCH 09/12] Fix nil --- models/db/backup.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/models/db/backup.go b/models/db/backup.go index 0ab233566311f..b3775eea86d89 100644 --- a/models/db/backup.go +++ b/models/db/backup.go @@ -28,6 +28,13 @@ func BackupDatabaseAsFixtures(dirPath string) error { } func toNode(tableName, col string, v interface{}) *yaml.Node { + if v == nil { + return &yaml.Node{ + Kind: yaml.ScalarNode, + Tag: "!!str", + Value: "", + } + } switch vv := v.(type) { case string: if tableName == "action_task" && col == "log_indexes" { From 7d158249fdc7ebd49765e98eb63ecafa4b88dbc6 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 9 Sep 2023 21:39:06 +0800 Subject: [PATCH 10/12] Fix test --- models/db/backup_test.go | 69 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 64 insertions(+), 5 deletions(-) diff --git a/models/db/backup_test.go b/models/db/backup_test.go index 0b58db6d17025..65f0e3d1d5976 100644 --- a/models/db/backup_test.go +++ b/models/db/backup_test.go @@ -4,12 +4,16 @@ package db_test import ( + "fmt" "os" "path/filepath" + "sort" + "strconv" "testing" "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/unittest" + "gopkg.in/yaml.v3" "github.com/stretchr/testify/assert" ) @@ -39,11 +43,66 @@ func TestBackupRestore(t *testing.T) { // assert.NoError(t, db.RestoreDatabase(d)) } +func sortTable(tablename string, data []map[string]any) { + sort.Slice(data, func(i, j int) bool { + if tablename == "issue_index" { + return data[i]["group_id"].(int) < data[j]["group_id"].(int) + } + if tablename == "repo_topic" { + return data[i]["repo_id"].(int) < data[j]["repo_id"].(int) + } + return data[i]["id"].(int) < data[j]["id"].(int) + }) +} + func fileEqual(t *testing.T, a, b string) { - bs1, err := os.ReadFile(a) - assert.NoError(t, err) + filename := filepath.Base(a) + tablename := filename[:len(filename)-len(filepath.Ext(filename))] + t.Run(filename, func(t *testing.T) { + bs1, err := os.ReadFile(a) + assert.NoError(t, err) - bs2, err := os.ReadFile(b) - assert.NoError(t, err) - assert.EqualValues(t, bs1, bs2) + var data1 []map[string]any + assert.NoError(t, yaml.Unmarshal(bs1, &data1)) + + sortTable(tablename, data1) + + bs2, err := os.ReadFile(b) + assert.NoError(t, err) + + var data2 []map[string]any + assert.NoError(t, yaml.Unmarshal(bs2, &data2)) + + sortTable(tablename, data2) + + assert.EqualValues(t, len(data1), len(data2), fmt.Sprintf("compare %s with %s", a, b)) + for i := range data1 { + assert.LessOrEqual(t, len(data1[i]), len(data2[i]), fmt.Sprintf("compare %s with %s", a, b)) + for k, v := range data1[i] { + switch vv := v.(type) { + case bool: + var r bool + switch rr := data2[i][k].(type) { + case bool: + r = rr + case int: + r = rr != 0 + default: + r, _ = strconv.ParseBool(data2[i][k].(string)) + } + assert.EqualValues(t, vv, r, fmt.Sprintf("compare %s with %s", a, b)) + case nil: + switch data2[i][k].(type) { + case nil: + case string: + assert.Empty(t, data2[i][k]) + default: + panic(fmt.Sprintf("%#v", data2[i][k])) + } + default: + assert.EqualValues(t, v, data2[i][k], fmt.Sprintf("compare %#v with %#v", v, data2[i][k])) + } + } + } + }) } From fca3ed8b8a226c7872fd5365f46a9794b5b5560b Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 9 Sep 2023 21:45:07 +0800 Subject: [PATCH 11/12] Fix lint --- models/db/backup_test.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/models/db/backup_test.go b/models/db/backup_test.go index 65f0e3d1d5976..6bb8e1e6b7e74 100644 --- a/models/db/backup_test.go +++ b/models/db/backup_test.go @@ -13,9 +13,9 @@ import ( "code.gitea.io/gitea/models/db" "code.gitea.io/gitea/models/unittest" - "gopkg.in/yaml.v3" "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" ) func TestBackupRestore(t *testing.T) { @@ -55,6 +55,18 @@ func sortTable(tablename string, data []map[string]any) { }) } +func convertBool(b any) bool { + switch rr := b.(type) { + case bool: + return rr + case int: + return rr != 0 + default: + r, _ := strconv.ParseBool(b.(string)) + return r + } +} + func fileEqual(t *testing.T, a, b string) { filename := filepath.Base(a) tablename := filename[:len(filename)-len(filepath.Ext(filename))] @@ -81,16 +93,7 @@ func fileEqual(t *testing.T, a, b string) { for k, v := range data1[i] { switch vv := v.(type) { case bool: - var r bool - switch rr := data2[i][k].(type) { - case bool: - r = rr - case int: - r = rr != 0 - default: - r, _ = strconv.ParseBool(data2[i][k].(string)) - } - assert.EqualValues(t, vv, r, fmt.Sprintf("compare %s with %s", a, b)) + assert.EqualValues(t, vv, convertBool(data2[i][k]), fmt.Sprintf("compare %s with %s", a, b)) case nil: switch data2[i][k].(type) { case nil: From 75f2488392279f1f924c8a4910505d36cedd8c6c Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Sat, 9 Sep 2023 21:51:22 +0800 Subject: [PATCH 12/12] Remove unnecessary code --- models/db/backup.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/models/db/backup.go b/models/db/backup.go index b3775eea86d89..9a4073f04555c 100644 --- a/models/db/backup.go +++ b/models/db/backup.go @@ -109,13 +109,6 @@ func backupTableFixtures(e *xorm.Engine, bean interface{}, dirPath string) error } for _, obj := range objs { - for k, v := range obj { - // convert bytes to string - if vv, ok := v.([]byte); ok { - obj[k] = string(vv) - } - } - node := yaml.Node{ Kind: yaml.MappingNode, }