From e423c4efc1031ee50e63f917f31795fa08722908 Mon Sep 17 00:00:00 2001 From: rocket Date: Fri, 11 Jun 2021 19:24:35 +0300 Subject: [PATCH 01/19] Alter heuristic dependencies searching mechanism in order to find dependencies among MetaSql objects more efficient This might fix order of objects during `Sakilla` schema restore process --- .../dbgit/meta/SortedListMetaObject.java | 45 ++++++++++--------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/main/java/ru/fusionsoft/dbgit/meta/SortedListMetaObject.java b/src/main/java/ru/fusionsoft/dbgit/meta/SortedListMetaObject.java index 16fd538..e562b54 100644 --- a/src/main/java/ru/fusionsoft/dbgit/meta/SortedListMetaObject.java +++ b/src/main/java/ru/fusionsoft/dbgit/meta/SortedListMetaObject.java @@ -47,33 +47,34 @@ public List sortFromReferenced() throws ExceptionDBGit { private void calculateImoCrossDependencies(){ - for(DBGitMetaType metaType : Sets.newHashSet(DBGitMetaType.DBGitTable, DBGitMetaType.DbGitFunction)){ + for(DBGitMetaType metaType : Sets.newHashSet( + DBGitMetaType.DBGitTable, DBGitMetaType.DbGitFunction, DBGitMetaType.DbGitProcedure, DBGitMetaType.DbGitTrigger + )){ - List objectsOfType = collection.stream() + final List objectsOfType = collection.stream() .filter( x->x.getType().equals(metaType) ) .collect(Collectors.toList()); - - - Map realNamesToMetaNames = objectsOfType.stream().collect(Collectors.toMap( - x-> x.getUnderlyingDbObject().getSchema() + "." + x.getUnderlyingDbObject().getName(), - IMetaObject::getName - )); - + for(IMetaObject imo : objectsOfType){ - if(imo.getType().equals(DBGitMetaType.DbGitFunction)){ - DBSQLObject dbsql = (DBSQLObject) imo.getUnderlyingDbObject(); - Set deps = realNamesToMetaNames.keySet().stream() - .filter( x -> dbsql.getSql().contains(x) /*&& !(dbsql.getSchema()+"."+dbsql.getName()).equals(x)*/ ) - .map(realNamesToMetaNames::get) - .collect(Collectors.toSet()); - dbsql.setDependencies(deps); - } if(imo.getType().equals(DBGitMetaType.DBGitTable)){ - DBTable dbTable = (DBTable) imo.getUnderlyingDbObject(); - Set deps = realNamesToMetaNames.values().stream() - .filter( x -> dbTable.getDependencies().contains(x) /*&& !x.equals(imo.getName())*/ ) - .collect(Collectors.toSet()); - dbTable.getDependencies().addAll(deps); + final DBTable dbTable = (DBTable) imo.getUnderlyingDbObject(); + dbTable.getDependencies().addAll( + objectsOfType + .stream() + .filter(x -> dbTable.getDependencies().contains(x.getName()) /*&& !x.equals(imo.getName())*/) + .map(IMetaObject::getName) + .collect(Collectors.toSet()) + ); + } + else if (imo instanceof MetaSql) { + final DBSQLObject dbsql = (DBSQLObject) imo.getUnderlyingDbObject(); + dbsql.setDependencies( + objectsOfType + .stream() + .filter(x -> dbsql.getSql().contains(x.getUnderlyingDbObject().getName()) /*&& !(dbsql.getSchema()+"."+dbsql.getName()).equals(x)*/) + .map(IMetaObject::getName) + .collect(Collectors.toSet()) + ); } } From 1b1aff1dd82b7d6856132811b5ba497ec4cb468e Mon Sep 17 00:00:00 2001 From: rocket Date: Sun, 13 Jun 2021 18:49:02 +0300 Subject: [PATCH 02/19] Impl. of fetching aggregate functions on Postgres Fix DBAdapterPostgres errors on single object fetching functions Removed some unnecessary messages parts from console output --- .../dbgit/postgres/DBAdapterPostgres.java | 122 ++++++++++-------- .../specific/test/ReportOfTestGroupRun.java | 2 +- .../chars/specific/test/ReportOfTestRun.java | 2 +- 3 files changed, 69 insertions(+), 57 deletions(-) diff --git a/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java b/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java index 14b7f95..7bd9b83 100644 --- a/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java +++ b/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java @@ -628,9 +628,6 @@ public Map getTriggers(String schema) { String sql = rs.getString("ddl"); StringProperties options = new StringProperties(rs); Set dependencies = Collections.emptySet(); -// rs.getArray("dependencies") == null -// ? Collections.emptySet() -// : new HashSet<>(Arrays.asList((String[])rs.getArray("dependencies").getArray())); DBTrigger trigger = new DBTrigger(name, options, schema, owner, dependencies, sql); listTrigger.put(name, trigger); @@ -656,13 +653,9 @@ public DBTrigger getTrigger(String schema, String name) { if(rs.next()){ String sql = rs.getString("ddl"); String owner = "postgres"; - StringProperties options = new StringProperties(rs); - Set dependencies = rs.getArray("dependencies") == null - ? Collections.emptySet() - : new HashSet<>(Arrays.asList((String[])rs.getArray("dependencies").getArray())); - return new DBTrigger(name, options, schema, owner, dependencies, sql); + return new DBTrigger(name, options, schema, owner, Collections.emptySet(), sql); } else { String msg = lang.getValue("errors", "adapter", "objectNotFoundInDb").toString(); @@ -709,11 +702,8 @@ public Map getProcedures(String schema) { String owner = rs.getString("rolname"); String sql = rs.getString("ddl"); StringProperties options = new StringProperties(rs); - Set dependencies = rs.getArray("dependencies") == null - ? Collections.emptySet() - : new HashSet<>(Arrays.asList((String[])rs.getArray("dependencies").getArray())); - DBProcedure proc = new DBProcedure(name, options, schema, owner, dependencies, sql); + DBProcedure proc = new DBProcedure(name, options, schema, owner, Collections.emptySet(), sql); String nameInMap = mapProcs.containsKey(name) ? name + "_" + proc.getHash() : name; mapProcs.put(nameInMap, proc); @@ -750,11 +740,8 @@ public DBProcedure getProcedure(String schema, String name) { String owner = rs.getString("rolname"); String sql = rs.getString("ddl"); StringProperties options = new StringProperties(rs); - Set dependencies = rs.getArray("dependencies") == null - ? Collections.emptySet() - : new HashSet<>(Arrays.asList((String[])rs.getArray("dependencies").getArray())); - return new DBProcedure(name, options, schema, owner, dependencies, sql); + return new DBProcedure(name, options, schema, owner, Collections.emptySet(), sql); } else { String msg = lang.getValue("errors", "adapter", "objectNotFoundInDb").toString(); @@ -769,19 +756,37 @@ public DBProcedure getProcedure(String schema, String name) { @Override public Map getFunctions(String schema) { Map listFunction = new HashMap(); - String query = - "SELECT n.nspname AS \"schema\", u.rolname, p.proname AS \"name\", \n" + - " pg_catalog.pg_get_function_arguments(p.oid) AS \"arguments\",\n" + - " pg_get_functiondef(p.oid) AS ddl\n" + - "FROM pg_catalog.pg_proc p\n" + - " JOIN pg_catalog.pg_roles u ON u.oid = p.proowner\n" + - " LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace\n" + - ( (getDbVersionNumber() >= 10) - ? "WHERE p.prokind = 'f' \n" - : "WHERE p.proisagg is false " - )+ - "AND n.nspname not in('pg_catalog', 'information_schema')\n" + - "AND n.nspname = '"+schema+"'"; + String query = + "SELECT n.nspname AS schema, u.rolname, p.proname AS name, \n" + + " pg_catalog.pg_get_function_arguments(p.oid) AS arguments, \n" + + " CASE WHEN p.proisagg \n" + + " THEN format(\n" + + " E'CREATE AGGREGATE %s (\\n%s\\n);'\n" + + " , (pg_identify_object('pg_proc'::regclass, aggfnoid, 0)).identity\n" + + " , array_to_string(\n" + + " ARRAY[\n" + + " format(E'\\tSFUNC = %s', aggtransfn::regproc)\n" + + " , format(E'\\tSTYPE = %s', format_type(aggtranstype, NULL))\n" + + " , CASE aggfinalfn WHEN '-'::regproc THEN NULL ELSE format(E'\\tFINALFUNC = %s',aggfinalfn::text) END\n" + + " , CASE aggsortop WHEN 0 THEN NULL ELSE format(E'\\tSORTOP = %s', oprname) END\n" + + " , CASE WHEN agginitval IS NULL THEN NULL ELSE format(E'\\tINITCOND = %s', agginitval) END\n" + + " ]\n" + + " , E',\\n'\n" + + " )\n" + + " ) \n" + + " ELSE pg_get_functiondef(p.oid) \n" + + " END AS ddl \n" + + "FROM pg_catalog.pg_proc p \n" + + " JOIN pg_catalog.pg_roles u ON u.oid = p.proowner \n" + + " LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace\n" + + " LEFT JOIN pg_aggregate a ON a.aggfnoid = p.oid \n" + + " LEFT JOIN pg_operator op ON op.oid = a.aggsortop \n" + +( ( getDbVersionNumber() >= 10 ) + ? "WHERE p.prokind IN ('a', 'f') \n" + : "WHERE 1=1 \n " + ) + + "AND n.nspname not in('pg_catalog', 'information_schema') \n" + + "AND n.nspname = '"+schema+"'"; try (Statement stmt = getConnection().createStatement();ResultSet rs = stmt.executeQuery(query);){ @@ -790,12 +795,8 @@ public Map getFunctions(String schema) { String owner = rs.getString("rolname"); String sql = rs.getString("ddl"); StringProperties options = new StringProperties(rs); - Set dependencies = Collections.emptySet(); -// rs.getArray("dependencies") == null -// ? Collections.emptySet() -// : new HashSet<>(Arrays.asList((String[])rs.getArray("dependencies").getArray())); - DBFunction dbFunction = new DBFunction(name, options, schema, owner, dependencies, sql); + DBFunction dbFunction = new DBFunction(name, options, schema, owner, Collections.emptySet(), sql); String nameInMap = listFunction.containsKey(name) ? name + "_" + dbFunction.getHash() : name; listFunction.put(nameInMap, dbFunction); @@ -811,18 +812,37 @@ public Map getFunctions(String schema) { @Override public DBFunction getFunction(String schema, String name) { String query = - "SELECT n.nspname AS \"schema\", u.rolname, p.proname AS \"name\", \n" + - " pg_catalog.pg_get_function_arguments(p.oid) AS \"arguments\",\n" + - " pg_get_functiondef(p.oid) AS ddl\n" + - "FROM pg_catalog.pg_proc p\n" + - " JOIN pg_catalog.pg_roles u ON u.oid = p.proowner\n" + - " LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace\n" + - ( (getDbVersionNumber() >= 10) - ? "WHERE p.prokind = 'f' \n" - : "WHERE 1=1 \n" - ) + - "AND n.nspname not in('pg_catalog', 'information_schema')\n" + - "AND n.nspname = '"+schema+"' AND p.proname = '"+name+"'"; + "SELECT n.nspname AS schema, u.rolname, p.proname AS name, \n" + + " pg_catalog.pg_get_function_arguments(p.oid) AS arguments, \n" + + " CASE WHEN p.proisagg \n" + + " THEN format(\n" + + " E'CREATE AGGREGATE %s (\\n%s\\n);'\n" + + " , (pg_identify_object('pg_proc'::regclass, aggfnoid, 0)).identity\n" + + " , array_to_string(\n" + + " ARRAY[\n" + + " format(E'\\tSFUNC = %s', aggtransfn::regproc)\n" + + " , format(E'\\tSTYPE = %s', format_type(aggtranstype, NULL))\n" + + " , CASE aggfinalfn WHEN '-'::regproc THEN NULL ELSE format(E'\\tFINALFUNC = %s',aggfinalfn::text) END\n" + + " , CASE aggsortop WHEN 0 THEN NULL ELSE format(E'\\tSORTOP = %s', oprname) END\n" + + " , CASE WHEN agginitval IS NULL THEN NULL ELSE format(E'\\tINITCOND = %s', agginitval) END\n" + + " ]\n" + + " , E',\\n'\n" + + " )\n" + + " ) \n" + + " ELSE pg_get_functiondef(p.oid) \n" + + " END AS ddl \n" + + "FROM pg_catalog.pg_proc p \n" + + " JOIN pg_catalog.pg_roles u ON u.oid = p.proowner \n" + + " LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace\n" + + " LEFT JOIN pg_aggregate a ON a.aggfnoid = p.oid \n" + + " LEFT JOIN pg_operator op ON op.oid = a.aggsortop \n" + + ( + ( getDbVersionNumber() >= 10 ) + ? "WHERE p.prokind IN ('a', 'f') \n" + : "WHERE 1=1 \n " + ) + + "AND n.nspname not in('pg_catalog', 'information_schema') \n" + + "AND n.nspname = '"+schema+"' AND p.proname = '"+name+"'"; try (Statement stmt = getConnection().createStatement();ResultSet rs = stmt.executeQuery(query);){ @@ -831,13 +851,7 @@ public DBFunction getFunction(String schema, String name) { String owner = rs.getString("rolname"); String sql = rs.getString("ddl"); StringProperties options = new StringProperties(rs); - Set dependencies = rs.getArray("dependencies") == null - ? Collections.emptySet() - : new HashSet<>(Arrays.asList((String[])rs.getArray("dependencies").getArray())); - - return new DBFunction(name, options, schema, owner, dependencies, sql); - //String args = rs.getString("arguments"); - //func.setArguments(args); + return new DBFunction(name, options, schema, owner, Collections.emptySet(), sql); } else { String msg = lang.getValue("errors", "adapter", "objectNotFoundInDb").toString(); @@ -1079,7 +1093,6 @@ public Map getUDTs(String schema) { @Override public Map getDomains(String schema) { - System.out.println("getting domains"); final Map objects = new HashMap<>(); final String query = "SELECT \n" @@ -1149,7 +1162,6 @@ public Map getDomains(String schema) { @Override public Map getEnums(String schema) { - System.out.println("getting enums"); final Map objects = new HashMap<>(); final String query = "SELECT t.typname, r.rolname, pg_catalog.obj_description ( t.oid, 'pg_type' ) AS description," diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/test/ReportOfTestGroupRun.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/test/ReportOfTestGroupRun.java index 9601a7c..ab16a5c 100644 --- a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/test/ReportOfTestGroupRun.java +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/test/ReportOfTestGroupRun.java @@ -36,7 +36,7 @@ public ReportOfTestGroupRun(CharSequence description, Subj subject, Test.. new LabelOfTestRunBrokenSubject(), description, e.getMessage(), - ExceptionUtils.readStackTrace(e.getCause().getCause()) + e.getCause().getCause().getMessage() ); } diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/test/ReportOfTestRun.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/test/ReportOfTestRun.java index 53b44c4..b4d0326 100644 --- a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/test/ReportOfTestRun.java +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/test/ReportOfTestRun.java @@ -37,7 +37,7 @@ public ReportOfTestRun(Subj subject, Test test) { new LabelOfTestRunBrokenSubject(), test.description(), e.getMessage(), - ExceptionUtils.readStackTrace(e.getCause().getCause()) + e.getCause().getCause().getMessage() ); } From b6755001f20ebb683e90c0b01deece8172576c6e Mon Sep 17 00:00:00 2001 From: rocket Date: Wed, 16 Jun 2021 17:29:38 +0300 Subject: [PATCH 03/19] Fix SortedListMetaObject alg., extracted search of name in sql code to DbObjectNameInSqlPresenceTest.java with test Add dropping existing in files and database both MetaFunction, MetaProcedure, MetaView, MetaTrigger before actually dropping unneeded and restoring changed and added objects --- .../fusionsoft/dbgit/command/CmdRestore.java | 33 +++++++- .../fusionsoft/dbgit/meta/DBGitMetaType.java | 20 ++--- .../dbgit/meta/DbObjectNameInSqlPresence.java | 20 +++++ .../dbgit/meta/SortedListMetaObject.java | 33 ++++++-- src/main/resources/lang/eng.yaml | 2 + .../meta/DbObjectNameInSqlPresenceTest.java | 82 +++++++++++++++++++ 6 files changed, 170 insertions(+), 20 deletions(-) create mode 100644 src/main/java/ru/fusionsoft/dbgit/meta/DbObjectNameInSqlPresence.java create mode 100644 src/test/java/ru/fusionsoft/dbgit/meta/DbObjectNameInSqlPresenceTest.java diff --git a/src/main/java/ru/fusionsoft/dbgit/command/CmdRestore.java b/src/main/java/ru/fusionsoft/dbgit/command/CmdRestore.java index 07eb480..2eb3b93 100644 --- a/src/main/java/ru/fusionsoft/dbgit/command/CmdRestore.java +++ b/src/main/java/ru/fusionsoft/dbgit/command/CmdRestore.java @@ -4,6 +4,7 @@ import java.io.FileOutputStream; import java.io.IOException; import java.nio.file.Files; +import java.text.MessageFormat; import java.text.SimpleDateFormat; import java.util.Date; import java.util.Map; @@ -140,7 +141,7 @@ public void execute(CommandLine cmdLine) throws Exception { // # steps 1,2 are in GitMetaDataManager::restoreDatabase - ConsoleWriter.println(getLang().getValue("general", "restore", "seekingToRestoreAdditional"),1); + ConsoleWriter.println(getLang().getValue("general", "restore", "seekingToRestoreAdditional"), messageLevel+2); Map affectedTables = new TreeMapMetaObject(); Map foundTables = new TreeMapMetaObject(); do { @@ -157,11 +158,37 @@ public void execute(CommandLine cmdLine) throws Exception { } while (!foundTables.isEmpty()); if(affectedTables.isEmpty()){ - ConsoleWriter.println(getLang().getValue("general", "restore", "nothingToRestoreAdditional"), 2); + ConsoleWriter.println(getLang().getValue("general", "restore", "nothingToRestoreAdditional"), messageLevel+2); } else { - affectedTables.forEach((k,v)->ConsoleWriter.println(k, 2)); + affectedTables.forEach((k,v)->ConsoleWriter.println(k, messageLevel+3)); } + //delete MetaSql (but no UDT's, domains or enums) that are in files and in db to fix errors on table restore + ConsoleWriter.println(getLang().getValue("general", "restore", "droppingSqlObjects"), messageLevel+2); + for (final IMetaObject object : + new SortedListMetaObject( + dbObjs.entrySet().stream() + .filter(x -> fileObjs.containsKey(x.getKey())) + .map(Map.Entry::getValue) + .filter(x -> + x instanceof MetaFunction || + x instanceof MetaProcedure || + x instanceof MetaView || + x instanceof MetaTrigger + ) + .collect(Collectors.toList()) + ).sortFromDependencies() + ) { + ConsoleWriter.println( + getLang().getValue("general", "restore", "droppingObject").withParams(object.getName()), + messageLevel + 3 + ); + adapter + .getFactoryRestore() + .getAdapterRestore(object.getType(), adapter) + .removeMetaObject(object); + updateObjs.put(object); + } if(toMakeBackup && toMakeChanges) { diff --git a/src/main/java/ru/fusionsoft/dbgit/meta/DBGitMetaType.java b/src/main/java/ru/fusionsoft/dbgit/meta/DBGitMetaType.java index 78452b6..7c17e9d 100644 --- a/src/main/java/ru/fusionsoft/dbgit/meta/DBGitMetaType.java +++ b/src/main/java/ru/fusionsoft/dbgit/meta/DBGitMetaType.java @@ -105,16 +105,6 @@ public Integer getPriority() { return 40; } }, - - DbGitTrigger("trg") { - public Class getMetaClass() { - return MetaTrigger.class; - } - - public Integer getPriority() { - return 80; - } - }, DbGitProcedure("prc") { public Class getMetaClass() { @@ -135,6 +125,16 @@ public Integer getPriority() { return 70; } }, + + DbGitTrigger("trg") { + public Class getMetaClass() { + return MetaTrigger.class; + } + + public Integer getPriority() { + return 80; + } + }, DbGitView("vw") { public Class getMetaClass() { diff --git a/src/main/java/ru/fusionsoft/dbgit/meta/DbObjectNameInSqlPresence.java b/src/main/java/ru/fusionsoft/dbgit/meta/DbObjectNameInSqlPresence.java new file mode 100644 index 0000000..623c22c --- /dev/null +++ b/src/main/java/ru/fusionsoft/dbgit/meta/DbObjectNameInSqlPresence.java @@ -0,0 +1,20 @@ +package ru.fusionsoft.dbgit.meta; + +import java.util.regex.Pattern; + +public class DbObjectNameInSqlPresence { + private final CharSequence name; + private final CharSequence sql; + + public DbObjectNameInSqlPresence(CharSequence name, CharSequence sql) { + this.name = name; + this.sql = sql; + } + + public final boolean matches() { + return Pattern + .compile("\\b[a-zA-Z0-9\\.]?" + name) + .matcher(sql) + .find(); + } +} diff --git a/src/main/java/ru/fusionsoft/dbgit/meta/SortedListMetaObject.java b/src/main/java/ru/fusionsoft/dbgit/meta/SortedListMetaObject.java index e562b54..90e5841 100644 --- a/src/main/java/ru/fusionsoft/dbgit/meta/SortedListMetaObject.java +++ b/src/main/java/ru/fusionsoft/dbgit/meta/SortedListMetaObject.java @@ -2,6 +2,7 @@ import com.diogonunes.jcdp.color.api.Ansi; import com.google.common.collect.Sets; +import java.text.MessageFormat; import ru.fusionsoft.dbgit.core.DBGitLang; import ru.fusionsoft.dbgit.core.ExceptionDBGit; import ru.fusionsoft.dbgit.dbobjects.DBSQLObject; @@ -47,12 +48,18 @@ public List sortFromReferenced() throws ExceptionDBGit { private void calculateImoCrossDependencies(){ - for(DBGitMetaType metaType : Sets.newHashSet( - DBGitMetaType.DBGitTable, DBGitMetaType.DbGitFunction, DBGitMetaType.DbGitProcedure, DBGitMetaType.DbGitTrigger + for(Set metaTypeSet : Sets.newHashSet( + Sets.newHashSet(DBGitMetaType.DBGitTable), + Sets.newHashSet( + DBGitMetaType.DbGitFunction, + DBGitMetaType.DbGitProcedure, + DBGitMetaType.DbGitTrigger, + DBGitMetaType.DbGitView + ) )){ final List objectsOfType = collection.stream() - .filter( x->x.getType().equals(metaType) ) + .filter( x->metaTypeSet.contains(x.getType()) ) .collect(Collectors.toList()); for(IMetaObject imo : objectsOfType){ @@ -67,11 +74,19 @@ private void calculateImoCrossDependencies(){ ); } else if (imo instanceof MetaSql) { - final DBSQLObject dbsql = (DBSQLObject) imo.getUnderlyingDbObject(); - dbsql.setDependencies( + final DBSQLObject imoDbSql = (DBSQLObject) imo.getUnderlyingDbObject(); + imoDbSql.setDependencies( objectsOfType .stream() - .filter(x -> dbsql.getSql().contains(x.getUnderlyingDbObject().getName()) /*&& !(dbsql.getSchema()+"."+dbsql.getName()).equals(x)*/) + .filter( + other -> { + return new DbObjectNameInSqlPresence( + other.getUnderlyingDbObject().getName(), + imoDbSql.getSql() + ).matches() && + ! other.getName().equals(imo.getName()); + } + ) .map(IMetaObject::getName) .collect(Collectors.toSet()) ); @@ -123,7 +138,11 @@ public List createSortedList(boolean isSortedFromFree) throws Excep .collect(Collectors.toList()); if (objectsL1.isEmpty()) { warnNotAdded(objectsOfType); - throw new ExceptionDBGit("infinite loop"); + final String details = objectsOfType + .stream() + .map( x-> MessageFormat.format("\n{0} ({1})", x.getName(), x.getUnderlyingDbObject().getDependencies().toString())) + .collect(Collectors.joining()); + throw new ExceptionDBGit("infinite loop\n" + details); } objectsOfType.removeAll(objectsL1); if(isSortedFromFree) { objectsL0.addAll(objectsL1); } diff --git a/src/main/resources/lang/eng.yaml b/src/main/resources/lang/eng.yaml index 1a3e5db..c1e7b59 100644 --- a/src/main/resources/lang/eng.yaml +++ b/src/main/resources/lang/eng.yaml @@ -94,6 +94,8 @@ general: droppingColumns: Dropping columns... droppingTablesConstraints: Dropping constraints for all updating tables... droppingTableConstraints: Dropping constraints for table {0}... + droppingSqlObjects: Dropping sql objects... + droppingObject: Dropping sql object {0}... restoringTablesConstraints: Restoring constraints for all updated tables... addPk: Adding PK... inserting: Inserting... diff --git a/src/test/java/ru/fusionsoft/dbgit/meta/DbObjectNameInSqlPresenceTest.java b/src/test/java/ru/fusionsoft/dbgit/meta/DbObjectNameInSqlPresenceTest.java new file mode 100644 index 0000000..bb56385 --- /dev/null +++ b/src/test/java/ru/fusionsoft/dbgit/meta/DbObjectNameInSqlPresenceTest.java @@ -0,0 +1,82 @@ +package ru.fusionsoft.dbgit.meta; + +import static org.junit.jupiter.api.Assertions.*; +import org.junit.jupiter.api.Test; + +class DbObjectNameInSqlPresenceTest { + + @Test + public final void matchesName() { + final DbObjectNameInSqlPresence presentCase = new DbObjectNameInSqlPresence( + "inventory_in_stock", + "CREATE OR REPLACE FUNCTION public.film_in_stock(p_film_id integer, p_store_id integer, OUT p_film_count integer)\n" + + "RETURNS SETOF integer\n" + + "LANGUAGE sql\n" + + " AS $function$\n" + + " SELECT inventory_id\n" + + " FROM inventory\n" + + " WHERE film_id = $1\n" + + " AND store_id = $2\n" + + " AND inventory_in_stock(inventory_id);\n" + + " $function$" + ); + + assertTrue(presentCase.matches()); + } + @Test + public final void notMatchesSqlWithNameUnderscoped() { + + final DbObjectNameInSqlPresence notPresentCase1 = new DbObjectNameInSqlPresence( + "group_concat", + "... (\n" + + " SFUNC = _group_concat,\n" + + " STYPE = text\n" + + ");" + ); + + assertFalse(notPresentCase1.matches()); + } + + @Test + public final void notMatchesSqlWithSchemaAndUderscopedName() { + final DbObjectNameInSqlPresence notPresentCase2 = new DbObjectNameInSqlPresence( + "group_concat", + "CREATE OR REPLACE FUNCTION public._group_concat(text, text)\n" + + " RETURNS text\n" + + " LANGUAGE sql\n" + + " IMMUTABLE\n" + + "AS $function$\n" + + " SELECT CASE\n" + + " WHEN $2 IS NULL THEN $1\n" + + " WHEN $1 IS NULL THEN $2\n" + + " ELSE $1 || ', ' || $2\n" + + " END\n" + + "$function$" + ); + + assertFalse(notPresentCase2.matches()); + + } + @Test + public final void matchesSqlWithSchemaAndUderscopedName() { + final DbObjectNameInSqlPresence presentCase2 = new DbObjectNameInSqlPresence( + "_group_concat", + "CREATE OR REPLACE FUNCTION public._group_concat(text, text)\n" + + " RETURNS text\n" + + " LANGUAGE sql\n" + + " IMMUTABLE\n" + + "AS $function$\n" + + " SELECT CASE\n" + + " WHEN $2 IS NULL THEN $1\n" + + " WHEN $1 IS NULL THEN $2\n" + + " ELSE $1 || ', ' || $2\n" + + " END\n" + + "$function$" + ); + + assertTrue(presentCase2.matches()); + + } + +} + From ea477d85b9db57227e0d4534ec092b76d2e4a049 Mon Sep 17 00:00:00 2001 From: rocket Date: Wed, 16 Jun 2021 20:36:01 +0300 Subject: [PATCH 04/19] Fix fetching functions/single function in Postgres Fix drop/restore procedure in Postgres Fix drop/restore function/aggregate function in Postgres Fix drop trigger in Postgres --- .../dbgit/postgres/DBAdapterPostgres.java | 22 +++++++++++-------- .../postgres/DBRestoreFunctionPostgres.java | 17 +++++++++++++- .../postgres/DBRestoreProcedurePostgres.java | 13 +++++++++-- .../postgres/DBRestoreTriggerPostgres.java | 7 ++++-- 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java b/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java index 7bd9b83..db3a61e 100644 --- a/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java +++ b/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java @@ -756,10 +756,11 @@ public DBProcedure getProcedure(String schema, String name) { @Override public Map getFunctions(String schema) { Map listFunction = new HashMap(); - String query = + String proisaggQuery = getDbVersionNumber() >= 10 ? "p.prokind IN('a')" : "p.proisagg"; + String query = "SELECT n.nspname AS schema, u.rolname, p.proname AS name, \n" + " pg_catalog.pg_get_function_arguments(p.oid) AS arguments, \n" - + " CASE WHEN p.proisagg \n" + + " CASE WHEN "+proisaggQuery+"\n" + " THEN format(\n" + " E'CREATE AGGREGATE %s (\\n%s\\n);'\n" + " , (pg_identify_object('pg_proc'::regclass, aggfnoid, 0)).identity\n" @@ -775,18 +776,19 @@ public Map getFunctions(String schema) { + " )\n" + " ) \n" + " ELSE pg_get_functiondef(p.oid) \n" - + " END AS ddl \n" + + " END AS ddl, \n" + + " CASE WHEN "+proisaggQuery+" THEN 'true' ELSE 'false' END AS proisagg \n" + "FROM pg_catalog.pg_proc p \n" + " JOIN pg_catalog.pg_roles u ON u.oid = p.proowner \n" + " LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace\n" + " LEFT JOIN pg_aggregate a ON a.aggfnoid = p.oid \n" + " LEFT JOIN pg_operator op ON op.oid = a.aggsortop \n" - +( ( getDbVersionNumber() >= 10 ) + + ( ( getDbVersionNumber() >= 10 ) ? "WHERE p.prokind IN ('a', 'f') \n" : "WHERE 1=1 \n " - ) + ) + "AND n.nspname not in('pg_catalog', 'information_schema') \n" - + "AND n.nspname = '"+schema+"'"; + + "AND n.nspname = '" + schema + "'"; try (Statement stmt = getConnection().createStatement();ResultSet rs = stmt.executeQuery(query);){ @@ -811,10 +813,11 @@ public Map getFunctions(String schema) { @Override public DBFunction getFunction(String schema, String name) { + String proisaggQuery = getDbVersionNumber() >= 10 ? "p.prokind IN('a')" : "p.proisagg"; String query = "SELECT n.nspname AS schema, u.rolname, p.proname AS name, \n" + " pg_catalog.pg_get_function_arguments(p.oid) AS arguments, \n" - + " CASE WHEN p.proisagg \n" + + " CASE WHEN " + proisaggQuery + "\n" + " THEN format(\n" + " E'CREATE AGGREGATE %s (\\n%s\\n);'\n" + " , (pg_identify_object('pg_proc'::regclass, aggfnoid, 0)).identity\n" @@ -830,7 +833,8 @@ public DBFunction getFunction(String schema, String name) { + " )\n" + " ) \n" + " ELSE pg_get_functiondef(p.oid) \n" - + " END AS ddl \n" + + " END AS ddl, \n" + + " CASE WHEN "+proisaggQuery+" THEN 'true' ELSE 'false' END AS proisagg \n" + "FROM pg_catalog.pg_proc p \n" + " JOIN pg_catalog.pg_roles u ON u.oid = p.proowner \n" + " LEFT JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace\n" @@ -842,7 +846,7 @@ public DBFunction getFunction(String schema, String name) { : "WHERE 1=1 \n " ) + "AND n.nspname not in('pg_catalog', 'information_schema') \n" - + "AND n.nspname = '"+schema+"' AND p.proname = '"+name+"'"; + + "AND n.nspname = '" + schema + "' AND p.proname = '" + name + "'"; try (Statement stmt = getConnection().createStatement();ResultSet rs = stmt.executeQuery(query);){ diff --git a/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreFunctionPostgres.java b/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreFunctionPostgres.java index 6d1c3ee..7f512c4 100644 --- a/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreFunctionPostgres.java +++ b/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreFunctionPostgres.java @@ -89,8 +89,23 @@ public void removeMetaObject(IMetaObject obj) throws Exception { DBFunction fnc = (DBFunction) fncMeta.getSqlObject(); if (fnc == null) return; + final StringProperties restoreProcArgs = fnc.getOptions().get("arguments"); + final String objectTypeName = + fnc.getOptions().get("proisagg") == null || + ! Boolean.parseBoolean(fnc.getOptions().get("proisagg").getData()) + ? "FUNCTION" + : "AGGREGATE"; + final String args = restoreProcArgs != null + ? restoreProcArgs.getData().replaceAll("(\\w+ \\w+) (DEFAULT [^\\,\\n]+)(\\,|\\b)", "$1") + : ""; String schema = getPhisicalSchema(fnc.getSchema()); - st.execute("DROP FUNCTION "+adapter.escapeNameIfNeeded(schema)+"."+adapter.escapeNameIfNeeded(fnc.getName())); + st.execute(MessageFormat.format( + "DROP {3} {0}.{1}({2});\n", + adapter.escapeNameIfNeeded(schema), + adapter.escapeNameIfNeeded(fnc.getName()), + args, + objectTypeName + )); } catch (Exception e) { throw new ExceptionDBGitRestore(lang.getValue("errors", "restore", "objectRemoveError").withParams(obj.getName()), e); } finally { diff --git a/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreProcedurePostgres.java b/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreProcedurePostgres.java index a8edf38..c049737 100644 --- a/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreProcedurePostgres.java +++ b/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreProcedurePostgres.java @@ -92,8 +92,17 @@ public void removeMetaObject(IMetaObject obj) throws Exception DBProcedure prc = (DBProcedure) prcMeta.getSqlObject(); if (prc == null) return; - String schema = getPhisicalSchema(prc.getSchema()); - st.execute("DROP PROCEDURE "+schema+"."+adapter.escapeNameIfNeeded(prc.getName())); + final String schema = getPhisicalSchema(prc.getSchema()); + final StringProperties restoreProcArgs = prc.getOptions().get("arguments"); + final String args = restoreProcArgs != null + ? restoreProcArgs.getData().replaceAll("(\\w+ \\w+) (DEFAULT [^\\,\\n]+)(\\,|\\b)", "$1") + : ""; + st.execute(MessageFormat.format( + "DROP PROCEDURE {0}.{1}({2});\n", + adapter.escapeNameIfNeeded(schema), + adapter.escapeNameIfNeeded(prc.getName()), + args + )); } catch (Exception e) { throw new ExceptionDBGitRestore(lang.getValue("errors", "restore", "objectRemoveError").withParams(obj.getName()), e); } finally { diff --git a/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTriggerPostgres.java b/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTriggerPostgres.java index 82bd4e3..f3f7e91 100644 --- a/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTriggerPostgres.java +++ b/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTriggerPostgres.java @@ -1,6 +1,7 @@ package ru.fusionsoft.dbgit.postgres; import java.sql.Connection; +import java.text.MessageFormat; import java.util.Map; import ru.fusionsoft.dbgit.adapters.DBRestoreAdapter; @@ -83,8 +84,10 @@ public void removeMetaObject(IMetaObject obj) throws Exception { DBTrigger trg = (DBTrigger) trgMeta.getSqlObject(); if (trg == null) return; - String schema = getPhisicalSchema(trg.getSchema()); - st.execute("DROP FUNCTION IF EXISTS "+adapter.escapeNameIfNeeded(schema)+"."+adapter.escapeNameIfNeeded(trg.getName())); + st.execute(MessageFormat.format("DROP TRIGGER IF EXISTS {0} ON {1}", + adapter.escapeNameIfNeeded(trg.getName()), + trg.getOptions().get("trigger_table").getData() + )); } catch (Exception e) { throw new ExceptionDBGitRestore(lang.getValue("errors", "restore", "objectRemoveError").withParams(obj.getName()), e); From 97d26c51eca159662c6f51800e2ae1b18999465b Mon Sep 17 00:00:00 2001 From: rocket Date: Sat, 19 Jun 2021 15:13:07 +0300 Subject: [PATCH 05/19] Add IT extension (PathPatchUsingConnectionFromDbLink.java) to operate database related to .dbgit repo Add IT extension CharsDbGitIgnoreWithTypes.java to write .dbignore file with custom type extensions and without .csv (for faster execution of tests when data non needed at the moment) Enhance DbGitIntegrationTestBasic.dbToDbRestoreWorksWithCustomTypes with dropping target schema (for having the same result of each execution everywhere) Add some logging on dropping table constraints to be able to see the actual drop order and actual tables dependencies at the moment --- .../fusionsoft/dbgit/adapters/DBAdapter.java | 4 + .../DbGitIntegrationTestBasic.java | 40 ++- .../dbgit/CharsDbGitIgnoreWithTypes.java | 24 ++ .../connection/ConnectionEnvelope.java | 302 ++++++++++++++++++ .../connection/ConnectionFromDbGitRepo.java | 9 + .../connection/ConnectionFromFileDbLink.java | 20 ++ .../ConnectionPatchExecutingStatement.java | 20 ++ .../PathPatchUsingConnectionFromDbLink.java | 19 ++ 8 files changed, 422 insertions(+), 16 deletions(-) create mode 100644 src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/dbgit/CharsDbGitIgnoreWithTypes.java create mode 100644 src/test/java/ru/fusionsoft/dbgit/integration/primitives/connection/ConnectionEnvelope.java create mode 100644 src/test/java/ru/fusionsoft/dbgit/integration/primitives/connection/ConnectionFromDbGitRepo.java create mode 100644 src/test/java/ru/fusionsoft/dbgit/integration/primitives/connection/ConnectionFromFileDbLink.java create mode 100644 src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/ConnectionPatchExecutingStatement.java create mode 100644 src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/PathPatchUsingConnectionFromDbLink.java diff --git a/src/main/java/ru/fusionsoft/dbgit/adapters/DBAdapter.java b/src/main/java/ru/fusionsoft/dbgit/adapters/DBAdapter.java index ff75722..60a5912 100644 --- a/src/main/java/ru/fusionsoft/dbgit/adapters/DBAdapter.java +++ b/src/main/java/ru/fusionsoft/dbgit/adapters/DBAdapter.java @@ -116,6 +116,10 @@ public void restoreDataBase(IMapMetaObject updateObjs) throws Exception { // remove table indexes and constraints, which is step(-2) of restoreMetaObject(MetaTable) ConsoleWriter.println(lang.getValue("general", "restore", "droppingTablesConstraints"), messageLevel); + tablesExists.sortFromDependencies().forEach( x -> { + ConsoleWriter.println(MessageFormat.format("\n{0} ({1})", x.getName(), x.getUnderlyingDbObject().getDependencies()), + messageLevel + 1); + }); for (IMetaObject table : tablesExists.sortFromDependencies()) { ConsoleWriter.println(lang.getValue("general", "restore", "droppingTableConstraints").withParams(table.getName()), messageLevel+1); getFactoryRestore().getAdapterRestore(DBGitMetaType.DBGitTable, this).restoreMetaObject(table, -2); diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java b/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java index fdffffe..889a82a 100644 --- a/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java +++ b/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java @@ -19,8 +19,11 @@ import ru.fusionsoft.dbgit.integration.primitives.args.specific.ArgsDbGitAddRemoteTestRepo; import ru.fusionsoft.dbgit.integration.primitives.chars.CommitsFromRepo; import ru.fusionsoft.dbgit.integration.primitives.chars.LinesOfUnsafeScalar; +import ru.fusionsoft.dbgit.integration.primitives.chars.specific.dbgit.CharsDbGitIgnoreWithTypes; import ru.fusionsoft.dbgit.integration.primitives.chars.specific.dbgit.CharsDbIgnoreWithDataAndTypes; import ru.fusionsoft.dbgit.integration.primitives.chars.specific.dbgit.CharsDbGitConfigBackupEnabled; +import ru.fusionsoft.dbgit.integration.primitives.patch.ConnectionPatchExecutingStatement; +import ru.fusionsoft.dbgit.integration.primitives.patch.PathPatchUsingConnectionFromDbLink; import ru.fusionsoft.dbgit.integration.primitives.patch.specific.PathPatchDbGitCheckout; import ru.fusionsoft.dbgit.integration.primitives.patch.specific.PathPatchDbGitCheckoutHard; import ru.fusionsoft.dbgit.integration.primitives.patch.specific.PathPatchDbGitClonesRepo; @@ -314,27 +317,32 @@ public final void dbToDbRestoreWorksWithCustomTypes() { //pagilla to local repo new PathAfterDbGitLinkAndAdd( new ArgsDbGitLinkPgAuto("pagilla"), - new CharsDbIgnoreWithDataAndTypes(), + new CharsDbGitIgnoreWithTypes(), //dvdrental to test#databasegit new PathAfterDbGitRun( new ArgsExplicit("restore", "-r", "-v"), - new PathAfterDbGitRun( - new ArgsDbGitLinkPgAuto(new NameOfDefaultTargetTestDatabase()), - - //dvdrental to local repo - new PathAfterDbGitLinkAndAdd( - new ArgsDbGitLinkPgAuto("dvdrental"), - new CharsDbIgnoreWithDataAndTypes(), - - new PathAfterDbGitRun( - new ArgsExplicit("init"), - - new PathWithoutFiles( - "*", - new PathNotProjectRoot( - new ProjectTestResourcesCleanDirectoryPath("05") + new PathPatched( + new PathPatchUsingConnectionFromDbLink(new ConnectionPatchExecutingStatement( + "DROP SCHEMA IF EXISTS public CASCADE;" + )), + new PathAfterDbGitRun( + new ArgsDbGitLinkPgAuto(new NameOfDefaultTargetTestDatabase()), + + //dvdrental to local repo + new PathAfterDbGitLinkAndAdd( + new ArgsDbGitLinkPgAuto("dvdrental"), + new CharsDbGitIgnoreWithTypes(), + + new PathAfterDbGitRun( + new ArgsExplicit("init"), + + new PathWithoutFiles( + "*", + new PathNotProjectRoot( + new ProjectTestResourcesCleanDirectoryPath("05") + ) ) ) ) diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/dbgit/CharsDbGitIgnoreWithTypes.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/dbgit/CharsDbGitIgnoreWithTypes.java new file mode 100644 index 0000000..65e8680 --- /dev/null +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/dbgit/CharsDbGitIgnoreWithTypes.java @@ -0,0 +1,24 @@ +package ru.fusionsoft.dbgit.integration.primitives.chars.specific.dbgit; + +import ru.fusionsoft.dbgit.integration.primitives.chars.CharSequenceEnvelope; + +public class CharsDbGitIgnoreWithTypes extends CharSequenceEnvelope { + public CharsDbGitIgnoreWithTypes() { + super(()->{ + return "*\n" + + "!public/*.ts\n" + + "!public/*.sch\n" + + "!public/*.seq\n" + + "!public/*.tbl\n" + + "!public/*.pkg\n" + + "!public/*.trg\n" + + "!public/*.prc\n" + + "!public/*.fnc\n" + + "!public/*.vw\n" + + "!public/*.blob\n" + + "!public/*.udt\n" + + "!public/*.enum\n" + + "!public/*.domain\n"; + }); + } +} diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/connection/ConnectionEnvelope.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/connection/ConnectionEnvelope.java new file mode 100644 index 0000000..7342a53 --- /dev/null +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/connection/ConnectionEnvelope.java @@ -0,0 +1,302 @@ +package ru.fusionsoft.dbgit.integration.primitives.connection; + +import java.sql.Array; +import java.sql.Blob; +import java.sql.CallableStatement; +import java.sql.Clob; +import java.sql.Connection; +import java.sql.DatabaseMetaData; +import java.sql.NClob; +import java.sql.PreparedStatement; +import java.sql.SQLClientInfoException; +import java.sql.SQLException; +import java.sql.SQLWarning; +import java.sql.SQLXML; +import java.sql.Savepoint; +import java.sql.Statement; +import java.sql.Struct; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.Executor; +import ru.fusionsoft.dbgit.integration.primitives.SafeScalar; +import ru.fusionsoft.dbgit.integration.primitives.SafeScalarOf; +import ru.fusionsoft.dbgit.integration.primitives.Scalar; +import ru.fusionsoft.dbgit.integration.primitives.StickyScalar; + +public class ConnectionEnvelope implements Connection { + private final SafeScalar origin; + + public ConnectionEnvelope(Scalar origin) { + this.origin = new SafeScalarOf<>(new StickyScalar<>(origin)); + } + + @Override + public Statement createStatement() throws SQLException { + return origin.value().createStatement(); + } + + @Override + public PreparedStatement prepareStatement(String sql) throws SQLException { + return origin.value().prepareStatement(sql); + } + + @Override + public CallableStatement prepareCall(String sql) throws SQLException { + return origin.value().prepareCall(sql); + } + + @Override + public String nativeSQL(String sql) throws SQLException { + return origin.value().nativeSQL(sql); + } + + @Override + public void setAutoCommit(boolean autoCommit) throws SQLException { + origin.value().setAutoCommit(autoCommit); + } + + @Override + public boolean getAutoCommit() throws SQLException { + return origin.value().getAutoCommit(); + } + + @Override + public void commit() throws SQLException { + origin.value().commit(); + } + + @Override + public void rollback() throws SQLException { + origin.value().rollback(); + } + + @Override + public void close() throws SQLException { + origin.value().close(); + } + + @Override + public boolean isClosed() throws SQLException { + return origin.value().isClosed(); + } + + @Override + public DatabaseMetaData getMetaData() throws SQLException { + return origin.value().getMetaData(); + } + + @Override + public void setReadOnly(boolean readOnly) throws SQLException { + origin.value().setReadOnly(readOnly); + } + + @Override + public boolean isReadOnly() throws SQLException { + return origin.value().isReadOnly(); + } + + @Override + public void setCatalog(String catalog) throws SQLException { + origin.value().setCatalog(catalog); + } + + @Override + public String getCatalog() throws SQLException { + return origin.value().getCatalog(); + } + + @Override + public void setTransactionIsolation(int level) throws SQLException { + origin.value().setTransactionIsolation(level); + } + + @Override + public int getTransactionIsolation() throws SQLException { + return origin.value().getTransactionIsolation(); + } + + @Override + public SQLWarning getWarnings() throws SQLException { + return origin.value().getWarnings(); + } + + @Override + public void clearWarnings() throws SQLException { + origin.value().clearWarnings(); + } + + @Override + public Statement createStatement(int resultSetType, int resultSetConcurrency) throws SQLException { + return origin.value().createStatement(resultSetType, resultSetConcurrency); + } + + @Override + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + return origin.value().prepareStatement(sql, resultSetType, resultSetConcurrency); + } + + @Override + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency) throws SQLException { + return origin.value().prepareCall(sql, resultSetType, resultSetConcurrency); + } + + @Override + public Map> getTypeMap() throws SQLException { + return origin.value().getTypeMap(); + } + + @Override + public void setTypeMap(Map> map) throws SQLException { + origin.value().setTypeMap(map); + } + + @Override + public void setHoldability(int holdability) throws SQLException { + origin.value().setHoldability(holdability); + } + + @Override + public int getHoldability() throws SQLException { + return origin.value().getHoldability(); + } + + @Override + public Savepoint setSavepoint() throws SQLException { + return origin.value().setSavepoint(); + } + + @Override + public Savepoint setSavepoint(String name) throws SQLException { + return origin.value().setSavepoint(name); + } + + @Override + public void rollback(Savepoint savepoint) throws SQLException { + origin.value().rollback(savepoint); + } + + @Override + public void releaseSavepoint(Savepoint savepoint) throws SQLException { + origin.value().releaseSavepoint(savepoint); + } + + @Override + public Statement createStatement(int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + return origin.value().createStatement(resultSetType, resultSetConcurrency, resultSetHoldability); + } + + @Override + public PreparedStatement prepareStatement(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + return origin.value().prepareStatement(sql, resultSetType, resultSetConcurrency, resultSetHoldability); + } + + @Override + public CallableStatement prepareCall(String sql, int resultSetType, int resultSetConcurrency, int resultSetHoldability) throws SQLException { + return origin.value().prepareCall(sql, resultSetType, resultSetConcurrency, resultSetHoldability); + } + + @Override + public PreparedStatement prepareStatement(String sql, int autoGeneratedKeys) throws SQLException { + return origin.value().prepareStatement(sql, autoGeneratedKeys); + } + + @Override + public PreparedStatement prepareStatement(String sql, int[] columnIndexes) throws SQLException { + return origin.value().prepareStatement(sql, columnIndexes); + } + + @Override + public PreparedStatement prepareStatement(String sql, String[] columnNames) throws SQLException { + return origin.value().prepareStatement(sql, columnNames); + } + + @Override + public Clob createClob() throws SQLException { + return origin.value().createClob(); + } + + @Override + public Blob createBlob() throws SQLException { + return origin.value().createBlob(); + } + + @Override + public NClob createNClob() throws SQLException { + return origin.value().createNClob(); + } + + @Override + public SQLXML createSQLXML() throws SQLException { + return origin.value().createSQLXML(); + } + + @Override + public boolean isValid(int timeout) throws SQLException { + return origin.value().isValid(timeout); + } + + @Override + public void setClientInfo(String name, String value) throws SQLClientInfoException { + origin.value().setClientInfo(name, value); + } + + @Override + public void setClientInfo(Properties properties) throws SQLClientInfoException { + origin.value().setClientInfo(properties); + } + + @Override + public String getClientInfo(String name) throws SQLException { + return origin.value().getClientInfo(name); + } + + @Override + public Properties getClientInfo() throws SQLException { + return origin.value().getClientInfo(); + } + + @Override + public Array createArrayOf(String typeName, Object[] elements) throws SQLException { + return origin.value().createArrayOf(typeName, elements); + } + + @Override + public Struct createStruct(String typeName, Object[] attributes) throws SQLException { + return origin.value().createStruct(typeName, attributes); + } + + @Override + public void setSchema(String schema) throws SQLException { + origin.value().setSchema(schema); + } + + @Override + public String getSchema() throws SQLException { + return origin.value().getSchema(); + } + + @Override + public void abort(Executor executor) throws SQLException { + origin.value().abort(executor); + } + + @Override + public void setNetworkTimeout(Executor executor, int milliseconds) throws SQLException { + origin.value().setNetworkTimeout(executor, milliseconds); + } + + @Override + public int getNetworkTimeout() throws SQLException { + return origin.value().getNetworkTimeout(); + } + + @Override + public T unwrap(Class iface) throws SQLException { + return origin.value().unwrap(iface); + } + + @Override + public boolean isWrapperFor(Class iface) throws SQLException { + return origin.value().isWrapperFor(iface); + } +} diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/connection/ConnectionFromDbGitRepo.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/connection/ConnectionFromDbGitRepo.java new file mode 100644 index 0000000..ad83af4 --- /dev/null +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/connection/ConnectionFromDbGitRepo.java @@ -0,0 +1,9 @@ +package ru.fusionsoft.dbgit.integration.primitives.connection; + +import java.nio.file.Path; + +public class ConnectionFromDbGitRepo extends ConnectionEnvelope { + public ConnectionFromDbGitRepo(Path pathToDbGitRepo) { + super(()->new ConnectionFromFileDbLink(pathToDbGitRepo.resolve(".dbgit/.dblink"))); + } +} diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/connection/ConnectionFromFileDbLink.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/connection/ConnectionFromFileDbLink.java new file mode 100644 index 0000000..79ae4fc --- /dev/null +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/connection/ConnectionFromFileDbLink.java @@ -0,0 +1,20 @@ +package ru.fusionsoft.dbgit.integration.primitives.connection; + +import java.io.FileInputStream; +import java.nio.file.Path; +import java.sql.DriverManager; +import java.util.Properties; + +public class ConnectionFromFileDbLink extends ConnectionEnvelope { + public ConnectionFromFileDbLink(Path pathToFileDbLink) { + super(() -> { + final Properties properties = new Properties(); + properties.load(new FileInputStream(pathToFileDbLink.toFile())); + return DriverManager.getConnection( + properties.getProperty("url"), + properties.getProperty("user"), + properties.getProperty("password") + ); + }); + } +} diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/ConnectionPatchExecutingStatement.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/ConnectionPatchExecutingStatement.java new file mode 100644 index 0000000..53d90e8 --- /dev/null +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/ConnectionPatchExecutingStatement.java @@ -0,0 +1,20 @@ +package ru.fusionsoft.dbgit.integration.primitives.patch; + +import java.sql.Connection; +import java.sql.Statement; +import ru.fusionsoft.dbgit.integration.primitives.Patch; + +public class ConnectionPatchExecutingStatement implements Patch { + private final CharSequence sqlStatementChars; + + public ConnectionPatchExecutingStatement(CharSequence sqlStatementChars) { + this.sqlStatementChars = sqlStatementChars; + } + + @Override + public final void apply(Connection connection) throws Exception { + try (final Statement statement = connection.createStatement()) { + statement.execute(String.valueOf(sqlStatementChars)); + } + } +} diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/PathPatchUsingConnectionFromDbLink.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/PathPatchUsingConnectionFromDbLink.java new file mode 100644 index 0000000..230ee4c --- /dev/null +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/PathPatchUsingConnectionFromDbLink.java @@ -0,0 +1,19 @@ +package ru.fusionsoft.dbgit.integration.primitives.patch; + +import java.nio.file.Path; +import java.sql.Connection; +import ru.fusionsoft.dbgit.integration.primitives.Patch; +import ru.fusionsoft.dbgit.integration.primitives.connection.ConnectionFromDbGitRepo; + +public class PathPatchUsingConnectionFromDbLink implements Patch { + private final Patch connectionPatch; + + public PathPatchUsingConnectionFromDbLink(Patch connectionPatch) { + this.connectionPatch = connectionPatch; + } + + @Override + public final void apply(Path root) throws Exception { + connectionPatch.apply(new ConnectionFromDbGitRepo(root)); + } +} From 44db0fb87d5c070292cb147c088fcfd80bf4ccb7 Mon Sep 17 00:00:00 2001 From: rocket Date: Sat, 19 Jun 2021 18:16:51 +0300 Subject: [PATCH 06/19] Fix drop constraints order error (now uses _source_ schema objects when ordering instead of _target_) Add `Cactoos` lib to project for EO approach learning --- pom.xml | 5 ++++ .../fusionsoft/dbgit/adapters/DBAdapter.java | 18 ----------- .../fusionsoft/dbgit/command/CmdRestore.java | 30 +++++++++++++++++++ 3 files changed, 35 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index d0efc3f..42f43d5 100644 --- a/pom.xml +++ b/pom.xml @@ -147,6 +147,11 @@ + + org.cactoos + cactoos + 0.50 + org.apache.commons diff --git a/src/main/java/ru/fusionsoft/dbgit/adapters/DBAdapter.java b/src/main/java/ru/fusionsoft/dbgit/adapters/DBAdapter.java index 60a5912..24ad5d6 100644 --- a/src/main/java/ru/fusionsoft/dbgit/adapters/DBAdapter.java +++ b/src/main/java/ru/fusionsoft/dbgit/adapters/DBAdapter.java @@ -109,22 +109,10 @@ public void restoreDataBase(IMapMetaObject updateObjs) throws Exception { try { SortedListMetaObject tables = new SortedListMetaObject(updateObjs.values().stream().filter(x->x instanceof MetaTable ).collect(Collectors.toList())); - SortedListMetaObject tablesExists = new SortedListMetaObject(updateObjs.values().stream().filter(x->x instanceof MetaTable && isExists(x)).collect(Collectors.toList())); Set createdSchemas = getSchemes().values().stream().map(DBOptionsObject::getName).collect(Collectors.toSet()); Set createdRoles = getRoles().values().stream().map(DBRole::getName).collect(Collectors.toSet()); - // remove table indexes and constraints, which is step(-2) of restoreMetaObject(MetaTable) - ConsoleWriter.println(lang.getValue("general", "restore", "droppingTablesConstraints"), messageLevel); - tablesExists.sortFromDependencies().forEach( x -> { - ConsoleWriter.println(MessageFormat.format("\n{0} ({1})", x.getName(), x.getUnderlyingDbObject().getDependencies()), - messageLevel + 1); - }); - for (IMetaObject table : tablesExists.sortFromDependencies()) { - ConsoleWriter.println(lang.getValue("general", "restore", "droppingTableConstraints").withParams(table.getName()), messageLevel+1); - getFactoryRestore().getAdapterRestore(DBGitMetaType.DBGitTable, this).restoreMetaObject(table, -2); - } - for (IMetaObject obj : updateObjs.getSortedList().sortFromReferenced()) { Timestamp timestampBefore = new Timestamp(System.currentTimeMillis()); int step = 0; @@ -132,7 +120,6 @@ public void restoreDataBase(IMapMetaObject updateObjs) throws Exception { IDBAdapterRestoreMetaData restoreAdapter = getFactoryRestore().getAdapterRestore(obj.getType(), this) ; if(restoreAdapter == null) throw new Exception("restore adapter is null"); -// ConsoleWriter.printlnGreen(lang.getValue("general", "restore", "restoreType").withParams(obj.getType().toString().substring(5), obj.getName())); obj = tryConvert(obj); createRoleIfNeed(obj, createdRoles); @@ -147,11 +134,6 @@ public void restoreDataBase(IMapMetaObject updateObjs) throws Exception { } Long timeDiff = new Timestamp(System.currentTimeMillis()).getTime() - timestampBefore.getTime(); -// ConsoleWriter.detailsPrintColor(MessageFormat.format(" ({1} {2})" -// , obj.getName() -// , timeDiff -// , lang.getValue("general", "add", "ms")), 0, Ansi.FColor.CYAN -// ); } // restore table constraints, which is step(-1) of restoreMetaObject(MetaTable) diff --git a/src/main/java/ru/fusionsoft/dbgit/command/CmdRestore.java b/src/main/java/ru/fusionsoft/dbgit/command/CmdRestore.java index 2eb3b93..b78abfb 100644 --- a/src/main/java/ru/fusionsoft/dbgit/command/CmdRestore.java +++ b/src/main/java/ru/fusionsoft/dbgit/command/CmdRestore.java @@ -7,13 +7,19 @@ import java.text.MessageFormat; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.List; import java.util.Map; +import java.util.function.Supplier; import java.util.stream.Collectors; import com.diogonunes.jcdp.color.api.Ansi; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Options; +import org.cactoos.Scalar; +import org.cactoos.list.ListEnvelope; +import org.cactoos.scalar.ScalarOf; +import org.cactoos.scalar.Sticky; import ru.fusionsoft.dbgit.adapters.AdapterFactory; import ru.fusionsoft.dbgit.adapters.IDBAdapter; import ru.fusionsoft.dbgit.core.*; @@ -190,6 +196,30 @@ public void execute(CommandLine cmdLine) throws Exception { updateObjs.put(object); } + // remove table indexes and constraints, which is step(-2) of restoreMetaObject(MetaTable) + ConsoleWriter.println(getLang().getValue("general", "restore", "droppingTablesConstraints"), messageLevel + 2); + for (IMetaObject table : new ScalarOf>( + ipt -> { + ipt.forEach(x -> ConsoleWriter.println( + MessageFormat.format("{0} ({1})", x.getName(), x.getUnderlyingDbObject().getDependencies()), + messageLevel + 3 + )); + return ipt; + }, + new SortedListMetaObject( + dbObjs.entrySet().stream() + .filter(x -> updateObjs.containsKey(x.getKey())) + .map(Map.Entry::getValue) + .filter(x -> x instanceof MetaTable) + .collect(Collectors.toList()) + ).sortFromDependencies() + ).value()) { + ConsoleWriter.println( + getLang().getValue("general", "restore", "droppingTableConstraints").withParams(table.getName()), + messageLevel + 3 + ); + adapter.getFactoryRestore().getAdapterRestore(DBGitMetaType.DBGitTable, adapter).restoreMetaObject(table, - 2); + } if(toMakeBackup && toMakeChanges) { IMapMetaObject backupObjs = new TreeMapMetaObject(); From 96a3bf6a2a0cffaecded4c2b11d64e38e44ed21b Mon Sep 17 00:00:00 2001 From: rocket Date: Sat, 19 Jun 2021 19:03:43 +0300 Subject: [PATCH 07/19] Fix restore domain on Postgres --- .../ru/fusionsoft/dbgit/postgres/DBRestoreDomainPostgres.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreDomainPostgres.java b/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreDomainPostgres.java index 56039a1..d170802 100644 --- a/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreDomainPostgres.java +++ b/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreDomainPostgres.java @@ -39,8 +39,8 @@ public final boolean restoreMetaObject(IMetaObject obj, int step) throws Excepti if (domains.containsKey(restoreDomain.getName())) { final DBDomain currentDomain = domains.get(restoreDomain.getName()); if ( - ! restoreDomain.getOptions().get("attributes").equals( - currentDomain.getOptions().get("attributes") + ! restoreDomain.getSql().equals( + currentDomain.getSql() ) ) { st.execute(MessageFormat.format( From 6518147fd2ab66531b193fceaa03e9f6cce61565 Mon Sep 17 00:00:00 2001 From: rocket Date: Sat, 19 Jun 2021 19:51:34 +0300 Subject: [PATCH 08/19] Add url parameter on ArgsDbGitLinkPgRemote Switch to use Postgres v11 on ArgsDbGitLinkPgAuto --- .../primitives/args/specific/ArgsDbGitLinkPgAuto.java | 4 +++- .../args/specific/ArgsDbGitLinkPgRemote.java | 11 ++++++----- .../chars/specific/UrlOfPgTestDatabaseV11.java | 9 +++++++++ .../chars/specific/UrlOfPgTestDatabaseV9.java | 9 +++++++++ 4 files changed, 27 insertions(+), 6 deletions(-) create mode 100644 src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/UrlOfPgTestDatabaseV11.java create mode 100644 src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/UrlOfPgTestDatabaseV9.java diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/args/specific/ArgsDbGitLinkPgAuto.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/args/specific/ArgsDbGitLinkPgAuto.java index afa53be..7e19f0c 100644 --- a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/args/specific/ArgsDbGitLinkPgAuto.java +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/args/specific/ArgsDbGitLinkPgAuto.java @@ -1,7 +1,9 @@ package ru.fusionsoft.dbgit.integration.primitives.args.specific; +import ru.fusionsoft.dbgit.integration.primitives.chars.specific.UrlOfPgTestDatabaseV11; + public class ArgsDbGitLinkPgAuto extends ArgsDbGitLink { public ArgsDbGitLinkPgAuto(CharSequence databaseName) { - super(()->new ArgsDbGitLinkPgRemote(databaseName)); + super(()->new ArgsDbGitLinkPgRemote(new UrlOfPgTestDatabaseV11(), databaseName)); } } diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/args/specific/ArgsDbGitLinkPgRemote.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/args/specific/ArgsDbGitLinkPgRemote.java index 9731bdf..e0ef0de 100644 --- a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/args/specific/ArgsDbGitLinkPgRemote.java +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/args/specific/ArgsDbGitLinkPgRemote.java @@ -4,21 +4,22 @@ import ru.fusionsoft.dbgit.integration.primitives.credentials.specific.CredsOfPgTestDatabase; public class ArgsDbGitLinkPgRemote extends ArgsDbGitLink { - public ArgsDbGitLinkPgRemote(CharSequence database, CharSequence username, CharSequence password) { + public ArgsDbGitLinkPgRemote(CharSequence dbmsUrl, CharSequence database, CharSequence username, CharSequence password) { super( - "jdbc:postgresql://135.181.94.98:31007", + dbmsUrl, database, username, password ); } - public ArgsDbGitLinkPgRemote(CharSequence database, Credentials credentials) { - this(database, credentials.username(), credentials.password()); + public ArgsDbGitLinkPgRemote(CharSequence dbmsUrl, CharSequence database, Credentials credentials) { + this(dbmsUrl, database, credentials.username(), credentials.password()); } - public ArgsDbGitLinkPgRemote(CharSequence database) { + public ArgsDbGitLinkPgRemote(CharSequence dbmsUrl, CharSequence database) { this( + dbmsUrl, database, new CredsOfPgTestDatabase() ); diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/UrlOfPgTestDatabaseV11.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/UrlOfPgTestDatabaseV11.java new file mode 100644 index 0000000..dfd643f --- /dev/null +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/UrlOfPgTestDatabaseV11.java @@ -0,0 +1,9 @@ +package ru.fusionsoft.dbgit.integration.primitives.chars.specific; + +import ru.fusionsoft.dbgit.integration.primitives.chars.CharSequenceEnvelope; + +public class UrlOfPgTestDatabaseV11 extends CharSequenceEnvelope { + public UrlOfPgTestDatabaseV11() { + super(()-> "jdbc:postgresql://135.181.94.98:31107"); + } +} diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/UrlOfPgTestDatabaseV9.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/UrlOfPgTestDatabaseV9.java new file mode 100644 index 0000000..14955bf --- /dev/null +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/UrlOfPgTestDatabaseV9.java @@ -0,0 +1,9 @@ +package ru.fusionsoft.dbgit.integration.primitives.chars.specific; + +import ru.fusionsoft.dbgit.integration.primitives.chars.CharSequenceEnvelope; + +public class UrlOfPgTestDatabaseV9 extends CharSequenceEnvelope { + public UrlOfPgTestDatabaseV9() { + super(()-> "jdbc:postgresql://135.181.94.98:31007"); + } +} From 1898fcc4eda38fd8934583e26dfda6e5ec1b39c9 Mon Sep 17 00:00:00 2001 From: rocket Date: Sat, 19 Jun 2021 20:08:58 +0300 Subject: [PATCH 09/19] Fix restore partition table on Postgres - impl. of dumb table type conversion by drop and create --- .../postgres/DBRestoreTablePostgres.java | 33 +++++++++++-------- src/main/resources/lang/eng.yaml | 1 + 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTablePostgres.java b/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTablePostgres.java index a7ff235..109edf8 100644 --- a/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTablePostgres.java +++ b/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTablePostgres.java @@ -95,9 +95,16 @@ public void restoreTablePostgres(IMetaObject obj) throws Exception { //find existing table and set tablespace or create if (existingTable.loadFromDB()){ - - restoreTableTablespace(st, restoreTable, existingTable); - restoreTableOwner(st, restoreTable, existingTable); + StringProperties existingPKD = existingTable.getTable().getOptions().get("partkeydef"); + StringProperties restorePKD = restoreTable.getTable().getOptions().get("partkeydef"); + if(restorePKD != null && (existingPKD == null || !restorePKD.getData().equals(existingPKD.getData()))) { + removeMetaObject(existingTable); + createTable(st, restoreTable); + existingTable.loadFromDB(); + } else { + restoreTableTablespace(st, restoreTable, existingTable); + restoreTableOwner(st, restoreTable, existingTable); + } ConsoleWriter.detailsPrintGreen(lang.getValue("general", "ok")); } else { @@ -372,21 +379,20 @@ private void createTable(StatementLogging st, MetaTable restoreTable) throws Exc ); - if(!DBGitConfig.getInstance().getToIgnoreOnwer(false)){ - createTableDdl += MessageFormat.format("\n alter table {0}.{1} owner to {2}\n;", - nme.getSchema() ,nme.getName(), + if (restoreTable.getTable().getOptions().getChildren().containsKey("partkeydef")) { + createTableDdl = createTableDdl.replace(") ", ") PARTITION BY " + + restoreTable.getTable().getOptions().getChildren().get("partkeydef") + + " "); + } + if (! DBGitConfig.getInstance().getToIgnoreOnwer(false)) { + createTableDdl += MessageFormat.format("\nalter table {0}.{1} owner to {2}\n;", + nme.getSchema(), nme.getName(), restoreTable.getTable().getOptions().getChildren().containsKey("owner") ? restoreTable.getTable().getOptions().get("owner").getData() : "postgres" ); } - - if (restoreTable.getTable().getOptions().getChildren().containsKey("partkeydef")) { - createTableDdl = createTableDdl.replace(" ) ", ") PARTITION BY " + - restoreTable.getTable().getOptions().getChildren().get("partkeydef") - + " "); - } - + st.execute(createTableDdl); } private void restoreTableOwner(StatementLogging st, MetaTable restoreTable, MetaTable existingTable) throws Exception { @@ -472,6 +478,7 @@ private void restoreTableComment(MetaTable restoreTable, MetaTable existingTable } private void restoreTablePartition(MetaTable restoreTable, Statement st) throws ExceptionDBGit, SQLException { + ConsoleWriter.detailsPrintln(lang.getValue("general", "restore", "restoreTablePartition").withParams(restoreTable.getName()), messageLevel); NameMeta nme = getEscapedNameMeta(restoreTable); StringProperties parent = restoreTable.getTable().getOptions().getChildren().get("parent"); StringProperties pg_get_expr = restoreTable.getTable().getOptions().getChildren().get("pg_get_expr"); diff --git a/src/main/resources/lang/eng.yaml b/src/main/resources/lang/eng.yaml index c1e7b59..593e2d7 100644 --- a/src/main/resources/lang/eng.yaml +++ b/src/main/resources/lang/eng.yaml @@ -85,6 +85,7 @@ general: restoreTableConstraints: Restoring table {0} constraints ... restoreTableData: Restoring table data for {0} ... restoreTablespace: Restoring tablespace {0} ... + restoreTablePartition: Restoring table {0} partitions ... restoreConstr: Restoring constraints for table {0} ... delConstr: Deleting constraints for table {0} ... restoreIndex: Restoring indexes for table {0} ... From f87c77002fdabf6c1a5e1e77ee44834e754dc635 Mon Sep 17 00:00:00 2001 From: rocket Date: Sat, 19 Jun 2021 20:10:56 +0300 Subject: [PATCH 10/19] Change DbGitIntegrationTestBasic.dbToDbRestoreWorksWithCustomTypes back to using table data --- .../dbgit/integration/DbGitIntegrationTestBasic.java | 5 ++--- ...DbGitIgnoreWithTypes.java => CharsDbIgnoreWithTypes.java} | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) rename src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/dbgit/{CharsDbGitIgnoreWithTypes.java => CharsDbIgnoreWithTypes.java} (86%) diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java b/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java index 889a82a..2c2b25b 100644 --- a/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java +++ b/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java @@ -19,7 +19,6 @@ import ru.fusionsoft.dbgit.integration.primitives.args.specific.ArgsDbGitAddRemoteTestRepo; import ru.fusionsoft.dbgit.integration.primitives.chars.CommitsFromRepo; import ru.fusionsoft.dbgit.integration.primitives.chars.LinesOfUnsafeScalar; -import ru.fusionsoft.dbgit.integration.primitives.chars.specific.dbgit.CharsDbGitIgnoreWithTypes; import ru.fusionsoft.dbgit.integration.primitives.chars.specific.dbgit.CharsDbIgnoreWithDataAndTypes; import ru.fusionsoft.dbgit.integration.primitives.chars.specific.dbgit.CharsDbGitConfigBackupEnabled; import ru.fusionsoft.dbgit.integration.primitives.patch.ConnectionPatchExecutingStatement; @@ -317,7 +316,7 @@ public final void dbToDbRestoreWorksWithCustomTypes() { //pagilla to local repo new PathAfterDbGitLinkAndAdd( new ArgsDbGitLinkPgAuto("pagilla"), - new CharsDbGitIgnoreWithTypes(), + new CharsDbIgnoreWithDataAndTypes(), //dvdrental to test#databasegit new PathAfterDbGitRun( @@ -333,7 +332,7 @@ public final void dbToDbRestoreWorksWithCustomTypes() { //dvdrental to local repo new PathAfterDbGitLinkAndAdd( new ArgsDbGitLinkPgAuto("dvdrental"), - new CharsDbGitIgnoreWithTypes(), + new CharsDbIgnoreWithDataAndTypes(), new PathAfterDbGitRun( new ArgsExplicit("init"), diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/dbgit/CharsDbGitIgnoreWithTypes.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/dbgit/CharsDbIgnoreWithTypes.java similarity index 86% rename from src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/dbgit/CharsDbGitIgnoreWithTypes.java rename to src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/dbgit/CharsDbIgnoreWithTypes.java index 65e8680..66ae4d5 100644 --- a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/dbgit/CharsDbGitIgnoreWithTypes.java +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/chars/specific/dbgit/CharsDbIgnoreWithTypes.java @@ -2,8 +2,8 @@ import ru.fusionsoft.dbgit.integration.primitives.chars.CharSequenceEnvelope; -public class CharsDbGitIgnoreWithTypes extends CharSequenceEnvelope { - public CharsDbGitIgnoreWithTypes() { +public class CharsDbIgnoreWithTypes extends CharSequenceEnvelope { + public CharsDbIgnoreWithTypes() { super(()->{ return "*\n" + "!public/*.ts\n" + From cc05f9ecb9fa1b2ec37390c4951ce8fcad13d8e8 Mon Sep 17 00:00:00 2001 From: rocket Date: Sun, 18 Jul 2021 15:16:42 +0300 Subject: [PATCH 11/19] Fix of restore declarative pg table partition --- .../postgres/DBRestoreTablePostgres.java | 43 +++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTablePostgres.java b/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTablePostgres.java index 109edf8..130791a 100644 --- a/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTablePostgres.java +++ b/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTablePostgres.java @@ -14,6 +14,7 @@ import ru.fusionsoft.dbgit.core.db.FieldType; import ru.fusionsoft.dbgit.dbobjects.*; import ru.fusionsoft.dbgit.meta.DBGitMetaType; +import ru.fusionsoft.dbgit.meta.DbObjectNameInSqlPresence; import ru.fusionsoft.dbgit.meta.IMetaObject; import ru.fusionsoft.dbgit.meta.MetaTable; import ru.fusionsoft.dbgit.meta.NameMeta; @@ -97,7 +98,8 @@ public void restoreTablePostgres(IMetaObject obj) throws Exception { if (existingTable.loadFromDB()){ StringProperties existingPKD = existingTable.getTable().getOptions().get("partkeydef"); StringProperties restorePKD = restoreTable.getTable().getOptions().get("partkeydef"); - if(restorePKD != null && (existingPKD == null || !restorePKD.getData().equals(existingPKD.getData()))) { + + if(partitionColsChanged(restoreTable, existingTable)) { removeMetaObject(existingTable); createTable(st, restoreTable); existingTable.loadFromDB(); @@ -111,6 +113,7 @@ public void restoreTablePostgres(IMetaObject obj) throws Exception { ConsoleWriter.detailsPrintln(lang.getValue("general", "restore", "createTable"), messageLevel); createTable(st, restoreTable); + existingTable.loadFromDB(); ConsoleWriter.detailsPrintGreen(lang.getValue("general", "ok")); } @@ -286,6 +289,31 @@ private NameMeta getEscapedNameMeta(MetaTable table) throws ExceptionDBGit { return nm; } + + private String partitionKeyDefinitionOf(MetaTable table){ + final StringProperties opt = table.getTable().getOptions().get("partkeydef"); + if(opt != null){ + return opt.toString(); + } else { + return ""; + } + } + private boolean partitionColsChanged(MetaTable rTbl, MetaTable exTbl) throws Exception{ + final String rPKD = partitionKeyDefinitionOf(rTbl); + final String exPKD = partitionKeyDefinitionOf(exTbl); + if(rPKD.equals(exPKD)){ + final MapDifference difference = Maps.difference(rTbl.getFields(), exTbl.getFields()); + return ! difference.areEqual() && difference.entriesDiffering().entrySet().stream() + .filter( + x -> ! x.getValue().leftValue().getTypeSQL().equals(x.getValue().rightValue().getTypeSQL()) + ) + .anyMatch( + x -> new DbObjectNameInSqlPresence( x.getKey(), rPKD ).matches() + ); + } else { + return true; + } + } private void restoreTableFields(MetaTable restoreTable, MetaTable existingTable, StatementLogging st) throws Exception { String lastField = ""; @@ -370,9 +398,16 @@ private void createTable(StatementLogging st, MetaTable restoreTable) throws Exc NameMeta nme = getEscapedNameMeta(restoreTable); String createTableDdl = MessageFormat.format( - "create table {0}.{1}() {2};" + "create table {0}.{1}({2}) {3};" ,nme.getSchema() ,nme.getName() + ,restoreTable.getFields().values().stream().map( x -> MessageFormat.format( + "\n{0} {1} {2}", + x.getName(), + x.getTypeSQL().replace("NOT NULL", ""), +// x.getIsPrimaryKey() ? "PRIMARY KEY" : "", + (x.getDefaultValue() != null && !x.getDefaultValue().isEmpty()) ? "DEFAULT " + x.getDefaultValue() : "" + )).collect(Collectors.joining(",", "", "\n")) ,restoreTable.getTable().getOptions().getChildren().containsKey("tablespace") ? "tablespace " + restoreTable.getTable().getOptions().get("tablespace").getData() : "" @@ -556,7 +591,9 @@ private void dropColumn(String tblSam, DBTableField tblField, Statement st) thro st.execute("alter table "+ tblSam +" drop column "+ adapter.escapeNameIfNeeded(tblField.getName())); } private boolean isSameTypeSql(DBTableField left, DBTableField right){ - return left.getTypeSQL().equals(right.getTypeSQL()); + final String leftTypeSQL = left.getTypeSQL().replace("NOT NULL", "").trim(); + final String rightTypeSQL = right.getTypeSQL().replace("NOT NULL", "").trim(); + return leftTypeSQL.equals(rightTypeSQL); } private boolean hasNotTypeSql(ValueDifference field, String typeSql){ From af23edf8656ea36f658142ec83fbcb73706f9267 Mon Sep 17 00:00:00 2001 From: rocket Date: Sun, 18 Jul 2021 15:18:21 +0300 Subject: [PATCH 12/19] Fix of DateData truncates time --- src/main/java/ru/fusionsoft/dbgit/data_table/DateData.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/ru/fusionsoft/dbgit/data_table/DateData.java b/src/main/java/ru/fusionsoft/dbgit/data_table/DateData.java index 095d34e..1111459 100644 --- a/src/main/java/ru/fusionsoft/dbgit/data_table/DateData.java +++ b/src/main/java/ru/fusionsoft/dbgit/data_table/DateData.java @@ -21,7 +21,7 @@ public boolean loadFromDB(ResultSet rs, String fieldname) throws Exception { isNull = true; value = 0; } else - value = rs.getDate(fieldname).getTime(); + value = rs.getTimestamp(fieldname).getTime(); return true; } From ac70cea3be8f61eed364b3dd1c8fb36c382a7da3 Mon Sep 17 00:00:00 2001 From: rocket Date: Sun, 18 Jul 2021 18:00:37 +0300 Subject: [PATCH 13/19] Fix of DBAdapterPostgres::getTable(s) on partitioned tables --- .../java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java b/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java index db3a61e..7e58f70 100644 --- a/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java +++ b/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java @@ -233,7 +233,7 @@ public Map getTables(String schema) { final String commentTable = rs.getString("table_comment"); final Set dependencies = rs.getArray("dependencies") != null ? new HashSet<>(Arrays.asList((String[])rs.getArray("dependencies").getArray())) - : Collections.emptySet(); + : new HashSet<>(); if (rs.getString("parent") != null) { dependencies.add(schema + "/" + rs.getString("parent") + ".tbl"); @@ -297,7 +297,7 @@ public DBTable getTable(String schema, String name) { final String commentTable = rs.getString("table_comment"); final Set dependencies = rs.getArray("dependencies") != null ? new HashSet<>(Arrays.asList((String[])rs.getArray("dependencies").getArray())) - : Collections.emptySet(); + : new HashSet<>(); if (rs.getString("parent") != null) { dependencies.add(schema + "/" + rs.getString("parent") + ".tbl"); From 411757a7c677d7fb88998e170e5b9970230d6568 Mon Sep 17 00:00:00 2001 From: rocket Date: Sun, 18 Jul 2021 18:07:55 +0300 Subject: [PATCH 14/19] Fix/refactor of MetaTableData::loadFromDB for retry loading data only when network or database is temporary unavailable --- .../ru/fusionsoft/dbgit/config/TryCount.java | 19 ++++++ .../dbgit/config/TryDelaySeconds.java | 18 ++++++ .../fusionsoft/dbgit/core/SleepSeconds.java | 15 +++++ .../dbgit/data_table/BooleanData.java | 3 +- .../fusionsoft/dbgit/data_table/DateData.java | 3 +- .../dbgit/data_table/ICellData.java | 4 +- .../fusionsoft/dbgit/data_table/LongData.java | 3 +- .../dbgit/data_table/MapFileData.java | 6 +- .../fusionsoft/dbgit/data_table/RowData.java | 26 ++++---- .../dbgit/dbobjects/DBTableData.java | 4 +- .../fusionsoft/dbgit/meta/MetaTableData.java | 59 +++++++++---------- .../dbgit/postgres/LargeBlobPg.java | 4 +- 12 files changed, 115 insertions(+), 49 deletions(-) create mode 100644 src/main/java/ru/fusionsoft/dbgit/config/TryCount.java create mode 100644 src/main/java/ru/fusionsoft/dbgit/config/TryDelaySeconds.java create mode 100644 src/main/java/ru/fusionsoft/dbgit/core/SleepSeconds.java diff --git a/src/main/java/ru/fusionsoft/dbgit/config/TryCount.java b/src/main/java/ru/fusionsoft/dbgit/config/TryCount.java new file mode 100644 index 0000000..c97cd1e --- /dev/null +++ b/src/main/java/ru/fusionsoft/dbgit/config/TryCount.java @@ -0,0 +1,19 @@ +package ru.fusionsoft.dbgit.config; + +import java.util.function.Supplier; +import org.cactoos.scalar.Unchecked; +import ru.fusionsoft.dbgit.core.DBGitConfig; + +public class TryCount implements Supplier { + + @Override + public final Integer get() { + return new Unchecked(() -> { + return DBGitConfig.getInstance().getInteger( + "core", + "TRY_COUNT", + DBGitConfig.getInstance().getIntegerGlobal("core", "TRY_COUNT", 1000) + ); + }).value(); + } +} diff --git a/src/main/java/ru/fusionsoft/dbgit/config/TryDelaySeconds.java b/src/main/java/ru/fusionsoft/dbgit/config/TryDelaySeconds.java new file mode 100644 index 0000000..4797fb5 --- /dev/null +++ b/src/main/java/ru/fusionsoft/dbgit/config/TryDelaySeconds.java @@ -0,0 +1,18 @@ +package ru.fusionsoft.dbgit.config; + +import java.util.function.Supplier; +import org.cactoos.scalar.Unchecked; +import ru.fusionsoft.dbgit.core.DBGitConfig; + +public class TryDelaySeconds implements Supplier { + @Override + public final Integer get() { + return new Unchecked<>( + () -> DBGitConfig.getInstance().getInteger( + "core", + "TRY_DELAY", + DBGitConfig.getInstance().getIntegerGlobal("core", "TRY_DELAY", 1000) + ) + ).value(); + } +} diff --git a/src/main/java/ru/fusionsoft/dbgit/core/SleepSeconds.java b/src/main/java/ru/fusionsoft/dbgit/core/SleepSeconds.java new file mode 100644 index 0000000..106aaf2 --- /dev/null +++ b/src/main/java/ru/fusionsoft/dbgit/core/SleepSeconds.java @@ -0,0 +1,15 @@ +package ru.fusionsoft.dbgit.core; + +import java.util.concurrent.TimeUnit; +import java.util.function.Consumer; + +public class SleepSeconds implements Consumer { + @Override + public final void accept(Integer seconds) { + try { + TimeUnit.SECONDS.sleep(seconds); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/ru/fusionsoft/dbgit/data_table/BooleanData.java b/src/main/java/ru/fusionsoft/dbgit/data_table/BooleanData.java index f07d46c..9243cd3 100644 --- a/src/main/java/ru/fusionsoft/dbgit/data_table/BooleanData.java +++ b/src/main/java/ru/fusionsoft/dbgit/data_table/BooleanData.java @@ -2,6 +2,7 @@ import java.sql.ResultSet; +import java.sql.SQLException; import ru.fusionsoft.dbgit.core.ExceptionDBGit; import ru.fusionsoft.dbgit.dbobjects.DBTable; @@ -11,7 +12,7 @@ public class BooleanData implements ICellData { private boolean isNull = false; @Override - public boolean loadFromDB(ResultSet rs, String fieldName) throws Exception { + public boolean loadFromDB(ResultSet rs, String fieldName) throws SQLException { value = rs.getBoolean(fieldName); if (rs.wasNull()) { isNull = true; diff --git a/src/main/java/ru/fusionsoft/dbgit/data_table/DateData.java b/src/main/java/ru/fusionsoft/dbgit/data_table/DateData.java index 1111459..fb46032 100644 --- a/src/main/java/ru/fusionsoft/dbgit/data_table/DateData.java +++ b/src/main/java/ru/fusionsoft/dbgit/data_table/DateData.java @@ -2,6 +2,7 @@ import java.sql.Date; import java.sql.ResultSet; +import java.sql.SQLException; import java.text.DateFormat; import java.text.SimpleDateFormat; @@ -16,7 +17,7 @@ public class DateData implements ICellData { public static SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHHmmss"); @Override - public boolean loadFromDB(ResultSet rs, String fieldname) throws Exception { + public boolean loadFromDB(ResultSet rs, String fieldname) throws SQLException { if (rs.getDate(fieldname) == null) { isNull = true; value = 0; diff --git a/src/main/java/ru/fusionsoft/dbgit/data_table/ICellData.java b/src/main/java/ru/fusionsoft/dbgit/data_table/ICellData.java index 40601cf..815d9bd 100644 --- a/src/main/java/ru/fusionsoft/dbgit/data_table/ICellData.java +++ b/src/main/java/ru/fusionsoft/dbgit/data_table/ICellData.java @@ -1,13 +1,15 @@ package ru.fusionsoft.dbgit.data_table; +import java.io.IOException; import java.sql.ResultSet; +import java.sql.SQLException; import ru.fusionsoft.dbgit.core.ExceptionDBGit; import ru.fusionsoft.dbgit.dbobjects.DBTable; public interface ICellData { - public boolean loadFromDB(ResultSet rs, String fieldname) throws Exception; + public boolean loadFromDB(ResultSet rs, String fieldname) throws SQLException, ExceptionDBGit, IOException; public String serialize(DBTable tbl) throws Exception; diff --git a/src/main/java/ru/fusionsoft/dbgit/data_table/LongData.java b/src/main/java/ru/fusionsoft/dbgit/data_table/LongData.java index 0e04dd0..b76601e 100644 --- a/src/main/java/ru/fusionsoft/dbgit/data_table/LongData.java +++ b/src/main/java/ru/fusionsoft/dbgit/data_table/LongData.java @@ -2,6 +2,7 @@ import java.sql.ResultSet; +import java.sql.SQLException; import ru.fusionsoft.dbgit.dbobjects.DBTable; @@ -10,7 +11,7 @@ public class LongData implements ICellData { private boolean isNull = false; @Override - public boolean loadFromDB(ResultSet rs, String fieldname) throws Exception { + public boolean loadFromDB(ResultSet rs, String fieldname) throws SQLException { value = rs.getDouble(fieldname); if (rs.wasNull()) { isNull = true; diff --git a/src/main/java/ru/fusionsoft/dbgit/data_table/MapFileData.java b/src/main/java/ru/fusionsoft/dbgit/data_table/MapFileData.java index d127076..0bb8870 100644 --- a/src/main/java/ru/fusionsoft/dbgit/data_table/MapFileData.java +++ b/src/main/java/ru/fusionsoft/dbgit/data_table/MapFileData.java @@ -1,10 +1,12 @@ package ru.fusionsoft.dbgit.data_table; import java.io.File; +import java.io.IOException; import java.io.InputStream; import java.nio.file.Files; import java.nio.file.StandardCopyOption; import java.sql.ResultSet; +import java.sql.SQLException; import java.util.HashSet; import java.util.Set; @@ -25,12 +27,12 @@ public class MapFileData implements ICellData { //private String hash = null; - public InputStream getBlobData(ResultSet rs, String fieldname) throws Exception { + public InputStream getBlobData(ResultSet rs, String fieldname) throws SQLException, ExceptionDBGit { return rs.getBinaryStream(fieldname); } @Override - public boolean loadFromDB(ResultSet rs, String fieldname) throws Exception { + public boolean loadFromDB(ResultSet rs, String fieldname) throws SQLException, ExceptionDBGit, IOException { InputStream stream = getBlobData(rs, fieldname); if (stream != null) { diff --git a/src/main/java/ru/fusionsoft/dbgit/data_table/RowData.java b/src/main/java/ru/fusionsoft/dbgit/data_table/RowData.java index a3e58f7..f3476ae 100644 --- a/src/main/java/ru/fusionsoft/dbgit/data_table/RowData.java +++ b/src/main/java/ru/fusionsoft/dbgit/data_table/RowData.java @@ -1,6 +1,8 @@ package ru.fusionsoft.dbgit.data_table; +import java.io.IOException; import java.sql.ResultSet; +import java.sql.SQLException; import java.text.MessageFormat; import java.util.*; @@ -25,7 +27,7 @@ public class RowData { //protected String key; //protected MetaTable metaTable; - public RowData(ResultSet rs, MetaTable metaTable) throws Exception { + public RowData(ResultSet rs, MetaTable metaTable) throws SQLException, ExceptionDBGit, IOException { //this.metaTable = metaTable; loadDataFromRS(rs, metaTable); } @@ -41,7 +43,7 @@ public RowData(CSVRecord record, MetaTable metaTable, CSVRecord titleColumns) th loadDataFromCSVRecord(record, titleColumns, metaTable); } - private void loadDataFromRS(ResultSet rs, MetaTable metaTable) throws Exception { + private void loadDataFromRS(ResultSet rs, MetaTable metaTable) throws SQLException, ExceptionDBGit, IOException { for (int i = 0; i < rs.getMetaData().getColumnCount(); i++) { String columnName = rs.getMetaData().getColumnName(i+1); @@ -142,15 +144,19 @@ public String calcRowKey(List idColumns) throws Exception { } - public String calcRowHash() throws Exception { - CalcHash ch = new CalcHash(); - //for (ICellData cd : data.values()) { - for (ICellData cd : rowList) { - String str = cd.convertToString(); - if ( str != null) - ch.addData(str); + public String calcRowHash() throws ExceptionDBGit { + try { + CalcHash ch = new CalcHash(); + //for (ICellData cd : data.values()) { + for (ICellData cd : rowList) { + String str = cd.convertToString(); + if (str != null) + ch.addData(str); + } + return ch.calcHashStr(); + } catch (Exception ex){ + throw new ExceptionDBGit("Error calculate row hash", ex); } - return ch.calcHashStr(); } public Map getData(List fields) { diff --git a/src/main/java/ru/fusionsoft/dbgit/dbobjects/DBTableData.java b/src/main/java/ru/fusionsoft/dbgit/dbobjects/DBTableData.java index affce1d..d2338b8 100644 --- a/src/main/java/ru/fusionsoft/dbgit/dbobjects/DBTableData.java +++ b/src/main/java/ru/fusionsoft/dbgit/dbobjects/DBTableData.java @@ -30,7 +30,7 @@ public DBTableData(int errorFlag) { public int errorFlag() { return errorFlag; } - public ResultSet resultSet() throws ExceptionDBGitTableData { + public ResultSet resultSet() { return resultSet; // TODO find usages and adapt to recover from the ExceptionDBGitTableData @@ -55,4 +55,4 @@ public void close() { throw new ExceptionDBGitRunTime(ex); } } -} \ No newline at end of file +} diff --git a/src/main/java/ru/fusionsoft/dbgit/meta/MetaTableData.java b/src/main/java/ru/fusionsoft/dbgit/meta/MetaTableData.java index fa3042d..cd5f2fb 100644 --- a/src/main/java/ru/fusionsoft/dbgit/meta/MetaTableData.java +++ b/src/main/java/ru/fusionsoft/dbgit/meta/MetaTableData.java @@ -3,6 +3,7 @@ import java.io.*; import java.nio.charset.StandardCharsets; import java.sql.ResultSet; +import java.sql.SQLException; import java.util.ArrayList; import java.util.List; import java.util.Set; @@ -20,15 +21,15 @@ import com.diogonunes.jcdp.color.api.Ansi.FColor; -import org.apache.commons.lang3.exception.ExceptionUtils; import org.slf4j.Logger; import ru.fusionsoft.dbgit.adapters.AdapterFactory; import ru.fusionsoft.dbgit.adapters.IDBAdapter; -import ru.fusionsoft.dbgit.core.DBGitConfig; +import ru.fusionsoft.dbgit.config.TryCount; +import ru.fusionsoft.dbgit.config.TryDelaySeconds; import ru.fusionsoft.dbgit.core.DBGitLang; import ru.fusionsoft.dbgit.core.ExceptionDBGit; -import ru.fusionsoft.dbgit.core.ExceptionDBGitRunTime; import ru.fusionsoft.dbgit.core.GitMetaDataManager; +import ru.fusionsoft.dbgit.core.SleepSeconds; import ru.fusionsoft.dbgit.data_table.ICellData; import ru.fusionsoft.dbgit.data_table.MapFileData; import ru.fusionsoft.dbgit.data_table.RowData; @@ -285,34 +286,32 @@ public boolean loadPortionFromDB(int currentPortionIndex, int tryNumber) throws return true; - } catch (Exception e) { - - ConsoleWriter.println(e.getLocalizedMessage(), messageLevel); - ConsoleWriter.detailsPrintln(ExceptionUtils.getStackTrace(e), messageLevel); - logger.error(DBGitLang.getInstance().getValue("errors", "adapter", "tableData").toString(), e); - - try { - if (tryNumber <= DBGitConfig.getInstance().getInteger("core", "TRY_COUNT", DBGitConfig.getInstance().getIntegerGlobal("core", "TRY_COUNT", 1000))) { - try { - TimeUnit.SECONDS.sleep(DBGitConfig.getInstance().getInteger("core", "TRY_DELAY", DBGitConfig.getInstance().getIntegerGlobal("core", "TRY_DELAY", 1000))); - } catch (InterruptedException e1) { - throw new ExceptionDBGitRunTime(e1.getMessage()); - } - ConsoleWriter.println(DBGitLang.getInstance() - .getValue("errors", "dataTable", "tryAgain") - .withParams(String.valueOf(tryNumber)) - , messageLevel - ); - return loadPortionFromDB(currentPortionIndex, tryNumber++); - } - } catch (Exception e1) { - throw new ExceptionDBGitRunTime(e1); - // TODO Auto-generated catch block -// e1.printStackTrace(); + } catch (SQLException sqlEx){ + if( + new TryCount().get() < tryNumber && ( + sqlEx.getSQLState().equals("ORA-17008"/*Connection closed code*/) || + sqlEx.getSQLState().equals("ORA-17016"/*Statement timed out*/) || + sqlEx.getSQLState().equals("ORA-17126"/*Fixed Wait timeout elapsed*/) + ) + ){ + ConsoleWriter.println(sqlEx.getMessage(), messageLevel); + ConsoleWriter.println(DBGitLang.getInstance() + .getValue("errors", "dataTable", "tryAgain") + .withParams(String.valueOf(tryNumber)) + , messageLevel + ); + new SleepSeconds().accept(new TryDelaySeconds().get()); + return loadPortionFromDB(currentPortionIndex, tryNumber+1); + } else { + throw new ExceptionDBGit(DBGitLang.getInstance().getValue("errors", "adapter", "tableData").toString(), sqlEx); } - - if (e instanceof ExceptionDBGit) throw (ExceptionDBGit)e; - throw new ExceptionDBGit(e); + + } + catch (Exception ex){ + throw new ExceptionDBGit( + DBGitLang.getInstance().getValue("errors", "adapter", "tableData").toString(), + ex + ); } } diff --git a/src/main/java/ru/fusionsoft/dbgit/postgres/LargeBlobPg.java b/src/main/java/ru/fusionsoft/dbgit/postgres/LargeBlobPg.java index dd1f36d..94ff810 100644 --- a/src/main/java/ru/fusionsoft/dbgit/postgres/LargeBlobPg.java +++ b/src/main/java/ru/fusionsoft/dbgit/postgres/LargeBlobPg.java @@ -4,17 +4,19 @@ import java.sql.Connection; import java.sql.ResultSet; +import java.sql.SQLException; import org.postgresql.largeobject.LargeObject; import org.postgresql.largeobject.LargeObjectManager; import ru.fusionsoft.dbgit.adapters.AdapterFactory; import ru.fusionsoft.dbgit.adapters.IDBAdapter; +import ru.fusionsoft.dbgit.core.ExceptionDBGit; import ru.fusionsoft.dbgit.data_table.MapFileData; public class LargeBlobPg extends MapFileData { @Override - public InputStream getBlobData(ResultSet rs, String fieldname) throws Exception { + public InputStream getBlobData(ResultSet rs, String fieldname) throws SQLException, ExceptionDBGit { IDBAdapter adapter = AdapterFactory.createAdapter(); Connection connect = adapter.getConnection(); System.out.println(connect.getClass().getName()); From 428a3ac2e36dd47b9941574a3b76417a26d6b4cb Mon Sep 17 00:00:00 2001 From: rocket Date: Wed, 21 Jul 2021 13:36:51 +0300 Subject: [PATCH 15/19] Fix PK detection of column in DBAdapterPostgres::getTableFields --- .../dbgit/postgres/DBAdapterPostgres.java | 69 +++++++++++-------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java b/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java index 7e58f70..803c8e7 100644 --- a/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java +++ b/src/main/java/ru/fusionsoft/dbgit/postgres/DBAdapterPostgres.java @@ -320,26 +320,43 @@ public DBTable getTable(String schema, String name) { public Map getTableFields(String schema, String nameTable) { Map listField = new HashMap(); String query = - "SELECT distinct col.column_name,col.is_nullable,col.data_type, col.udt_name::regtype dtype,col.character_maximum_length, col.column_default, tc.constraint_name, " + - "case\r\n" + - " when lower(data_type) in ('integer', 'numeric', 'smallint', 'double precision', 'bigint') then 'number' \r\n" + - " when lower(data_type) in ('character varying', 'char', 'character', 'varchar') then 'string'\r\n" + - " when lower(data_type) in ('timestamp without time zone', 'timestamp with time zone', 'date') then 'date'\r\n" + - " when lower(data_type) in ('boolean') then 'boolean'\r\n" + - " when lower(data_type) in ('text') then 'text'\r\n" + - " when lower(data_type) in ('bytea') then 'binary'" + - " else 'native'\r\n" + - " end tp, " + - " case when lower(data_type) in ('char', 'character') then true else false end fixed, " + - " pgd.description," + - "col.* FROM " + - "information_schema.columns col " + - "left join information_schema.key_column_usage kc on col.table_schema = kc.table_schema and col.table_name = kc.table_name and col.column_name=kc.column_name " + - "left join information_schema.table_constraints tc on col.table_schema = kc.table_schema and col.table_name = kc.table_name and kc.constraint_name = tc.constraint_name and tc.constraint_type = 'PRIMARY KEY' " + - "left join pg_catalog.pg_statio_all_tables st on st.schemaname = col.table_schema and st.relname = col.table_name " + - "left join pg_catalog.pg_description pgd on (pgd.objoid=st.relid and pgd.objsubid=col.ordinal_position) " + - "where upper(col.table_schema) = upper(:schema) and col.table_name = :table " + - "order by col.column_name "; + "SELECT \n" + + " col.column_name,\n" + + " pgd.description,\n" + + " col.column_default,\n" + + " col.is_nullable,\n" + + " col.udt_name::regtype dtype,\n" + + " col.character_maximum_length,\n" + + " col.numeric_precision,\n" + + " col.numeric_scale,\n" + + " col.ordinal_position,\n" + + " case when pkeys.ispk is null then false else pkeys.ispk end ispk,\n" + + " case\n" + + " when lower(data_type) in ('integer', 'numeric', 'smallint', 'double precision', 'bigint') then 'number' \n" + + " when lower(data_type) in ('character varying', 'char', 'character', 'varchar') then 'string'\n" + + " when lower(data_type) in ('timestamp without time zone', 'timestamp with time zone', 'date') then 'date'\n" + + " when lower(data_type) in ('boolean') then 'boolean'\n" + + " when lower(data_type) in ('text') then 'text'\n" + + " when lower(data_type) in ('bytea') then 'binary'\n" + + " else 'native'\n" + + " end tp, \n" + + " case when lower(data_type) in ('char', 'character') then true else false \n" + + " end fixed\n" + + "FROM information_schema.columns col \n" + + "left join (\n" + + " select \n" + + " kc.table_schema,\n" + + " kc.table_name,\n" + + " kc.column_name,\n" + + " bool_or(case when tc.constraint_type = 'PRIMARY KEY' then true else false end) ispk\n" + + " from information_schema.table_constraints tc\n" + + " inner join information_schema.key_column_usage kc on kc.constraint_name = tc.constraint_name\n" + + " group by kc.table_schema, kc.table_name, kc.column_name\n" + + ") pkeys on pkeys.table_schema = col.table_schema and pkeys.table_name = col.table_name and pkeys.column_name = col.column_name\n" + + "left join pg_catalog.pg_statio_all_tables st on st.schemaname = col.table_schema and st.relname = col.table_name \n" + + "left join pg_catalog.pg_description pgd on (pgd.objoid=st.relid and pgd.objsubid=col.ordinal_position) \n" + + "where upper(col.table_schema) = upper(:schema) and col.table_name = :table\n" + + "order by col.column_name "; try ( PreparedStatement stmt = preparedStatement(getConnection(), query, ImmutableMap.of("schema", schema, "table", nameTable)); @@ -353,17 +370,9 @@ public Map getTableFields(String schema, String nameTable final String descField = rs.getString("description"); final String columnDefault = rs.getString("column_default"); final boolean isFixed = rs.getBoolean("fixed"); - final boolean isPrimaryKey = rs.getString("constraint_name") != null; - final boolean isNullable = !typeSQL.toLowerCase().contains("not null"); + final boolean isPrimaryKey = rs.getBoolean("ispk"); + final boolean isNullable = rs.getString("is_nullable").equals("YES"); final FieldType typeUniversal = FieldType.fromString(rs.getString("tp")); -// System.out.println(( -// nameField -// + " > " -// + typeSQL -// + " (" -// + typeUniversal.toString() -// + ")" -// )); final FieldType actualTypeUniversal = typeUniversal.equals(FieldType.TEXT) ? FieldType.STRING_NATIVE : typeUniversal; final int length = rs.getInt("character_maximum_length"); final int precision = rs.getInt("numeric_precision"); From 06e6594c12a22500b6bdfd5e5fa0739815b7c0ea Mon Sep 17 00:00:00 2001 From: rocket Date: Wed, 21 Jul 2021 13:40:02 +0300 Subject: [PATCH 16/19] Refactor IT case 'DbGitIntegrationTestBasic::dbToDbRestoreWorksWithCustomTypes' Change scenario to cover 'empty_db -> pagilla <-> dvdrental' sequences of restore --- .../DbGitIntegrationTestBasic.java | 69 ++++++++++--------- .../specific/PathPatchDbGitLinkAndAdd.java | 21 ++++++ .../PathPatchDbGitRestoreFromDbToDb.java | 29 ++++++++ .../scenarios/PathAfterDbGitLinkAndAdd.java | 18 ++--- .../PathAfterDbGitRestoreFromDbToDb.java | 61 ++++++++++++++++ 5 files changed, 155 insertions(+), 43 deletions(-) create mode 100644 src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/specific/PathPatchDbGitLinkAndAdd.java create mode 100644 src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/specific/PathPatchDbGitRestoreFromDbToDb.java create mode 100644 src/test/java/ru/fusionsoft/dbgit/integration/primitives/path/specific/dbgit/scenarios/PathAfterDbGitRestoreFromDbToDb.java diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java b/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java index 2c2b25b..aa4fb32 100644 --- a/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java +++ b/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java @@ -17,10 +17,12 @@ import ru.fusionsoft.dbgit.integration.primitives.args.ArgsExplicit; import ru.fusionsoft.dbgit.integration.primitives.args.specific.ArgsDbGitLinkPgAuto; import ru.fusionsoft.dbgit.integration.primitives.args.specific.ArgsDbGitAddRemoteTestRepo; +import ru.fusionsoft.dbgit.integration.primitives.args.specific.ArgsDbGitRestore; import ru.fusionsoft.dbgit.integration.primitives.chars.CommitsFromRepo; import ru.fusionsoft.dbgit.integration.primitives.chars.LinesOfUnsafeScalar; import ru.fusionsoft.dbgit.integration.primitives.chars.specific.dbgit.CharsDbIgnoreWithDataAndTypes; import ru.fusionsoft.dbgit.integration.primitives.chars.specific.dbgit.CharsDbGitConfigBackupEnabled; +import ru.fusionsoft.dbgit.integration.primitives.chars.specific.dbgit.CharsDbIgnoreWithTypes; import ru.fusionsoft.dbgit.integration.primitives.patch.ConnectionPatchExecutingStatement; import ru.fusionsoft.dbgit.integration.primitives.patch.PathPatchUsingConnectionFromDbLink; import ru.fusionsoft.dbgit.integration.primitives.patch.specific.PathPatchDbGitCheckout; @@ -43,6 +45,7 @@ import ru.fusionsoft.dbgit.integration.primitives.path.specific.dbgit.PathWithDbGitRepoCloned; import ru.fusionsoft.dbgit.integration.primitives.path.PathWithoutFiles; import ru.fusionsoft.dbgit.integration.primitives.path.specific.ProjectTestResourcesCleanDirectoryPath; +import ru.fusionsoft.dbgit.integration.primitives.path.specific.dbgit.scenarios.PathAfterDbGitRestoreFromDbToDb; @Tag("integration") public class DbGitIntegrationTestBasic { @@ -302,46 +305,50 @@ public final void gitToDbRestoreWorks() throws Exception { @Test public final void dbToDbRestoreWorksWithCustomTypes() { - final String description = "Hardest sakilla_database sequential add and restore with table data and custom types (mpaa_rating) works"; + final String description = "Hardest sakilla_database sequential add and restore with table data, custom types and partitions works"; final TestResult result = new DescribedTestResult( description, new SimpleTestResult<>( - new PathAfterDbGitRun( - //pagilla to test#databasegit over dvdrental - new ArgsExplicit("restore", "-r", "-v"), - - new PathAfterDbGitRun( + new PathAfterDbGitRestoreFromDbToDb( + new ArgsDbGitLinkPgAuto("dvdrental"), + new ArgsDbGitLinkPgAuto(new NameOfDefaultTargetTestDatabase()), + new CharsDbIgnoreWithDataAndTypes(), + new ArgsDbGitRestore("-r"), + + new PathAfterDbGitRestoreFromDbToDb( + new ArgsDbGitLinkPgAuto("pagilla"), new ArgsDbGitLinkPgAuto(new NameOfDefaultTargetTestDatabase()), + new CharsDbIgnoreWithDataAndTypes(), + new ArgsDbGitRestore("-r"), - //pagilla to local repo - new PathAfterDbGitLinkAndAdd( - new ArgsDbGitLinkPgAuto("pagilla"), + new PathAfterDbGitRestoreFromDbToDb( + new ArgsDbGitLinkPgAuto("dvdrental"), + new ArgsDbGitLinkPgAuto(new NameOfDefaultTargetTestDatabase()), new CharsDbIgnoreWithDataAndTypes(), + new ArgsDbGitRestore("-r"), - //dvdrental to test#databasegit - new PathAfterDbGitRun( - new ArgsExplicit("restore", "-r", "-v"), - + new PathAfterDbGitRestoreFromDbToDb( + new ArgsDbGitLinkPgAuto("pagilla"), + new ArgsDbGitLinkPgAuto(new NameOfDefaultTargetTestDatabase()), + new CharsDbIgnoreWithDataAndTypes(), + new ArgsDbGitRestore("-r"), new PathPatched( - new PathPatchUsingConnectionFromDbLink(new ConnectionPatchExecutingStatement( - "DROP SCHEMA IF EXISTS public CASCADE;" - )), + new PathPatchUsingConnectionFromDbLink( + new ConnectionPatchExecutingStatement("DROP SCHEMA IF EXISTS public CASCADE;") + ), + new PathAfterDbGitRun( - new ArgsDbGitLinkPgAuto(new NameOfDefaultTargetTestDatabase()), - - //dvdrental to local repo - new PathAfterDbGitLinkAndAdd( - new ArgsDbGitLinkPgAuto("dvdrental"), - new CharsDbIgnoreWithDataAndTypes(), - - new PathAfterDbGitRun( - new ArgsExplicit("init"), - - new PathWithoutFiles( - "*", - new PathNotProjectRoot( - new ProjectTestResourcesCleanDirectoryPath("05") - ) + new ArgsDbGitLinkPgAuto( + new NameOfDefaultTargetTestDatabase() + ), + + new PathAfterDbGitRun( + new ArgsExplicit("init"), + + new PathWithoutFiles( + "*", + new PathNotProjectRoot( + new ProjectTestResourcesCleanDirectoryPath("05") ) ) ) diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/specific/PathPatchDbGitLinkAndAdd.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/specific/PathPatchDbGitLinkAndAdd.java new file mode 100644 index 0000000..477f7ac --- /dev/null +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/specific/PathPatchDbGitLinkAndAdd.java @@ -0,0 +1,21 @@ +package ru.fusionsoft.dbgit.integration.primitives.patch.specific; + +import java.io.PrintStream; +import java.nio.file.Path; +import ru.fusionsoft.dbgit.integration.primitives.PatchSequential; +import ru.fusionsoft.dbgit.integration.primitives.args.specific.ArgsDbGitLink; +import ru.fusionsoft.dbgit.integration.primitives.patch.PathPatchCreatingFile; + +public class PathPatchDbGitLinkAndAdd extends PatchSequential { + public PathPatchDbGitLinkAndAdd( + ArgsDbGitLink argsDbGitLink, + CharSequence ignoreChars, + PrintStream printStream + ) { + super( + new PathPatchDbGitLink(argsDbGitLink, printStream), + new PathPatchCreatingFile(".dbgit/.dbignore", ignoreChars, printStream), + new PathPatchDbGitAdd(printStream) + ); + } +} diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/specific/PathPatchDbGitRestoreFromDbToDb.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/specific/PathPatchDbGitRestoreFromDbToDb.java new file mode 100644 index 0000000..21f0305 --- /dev/null +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/specific/PathPatchDbGitRestoreFromDbToDb.java @@ -0,0 +1,29 @@ +package ru.fusionsoft.dbgit.integration.primitives.patch.specific; + +import java.io.PrintStream; +import java.nio.file.Path; +import ru.fusionsoft.dbgit.integration.primitives.PatchSequential; +import ru.fusionsoft.dbgit.integration.primitives.args.ArgsExplicit; +import ru.fusionsoft.dbgit.integration.primitives.args.specific.ArgsDbGitLink; +import ru.fusionsoft.dbgit.integration.primitives.args.specific.ArgsDbGitRestore; +import ru.fusionsoft.dbgit.integration.primitives.patch.PathPatchCreatingFile; +import ru.fusionsoft.dbgit.integration.primitives.patch.PathPatchRunningDbGit; + +public class PathPatchDbGitRestoreFromDbToDb extends PatchSequential{ + public PathPatchDbGitRestoreFromDbToDb( + ArgsDbGitLink sourceDbLinkArgs, + ArgsDbGitLink targetDbLinkArgs, + CharSequence dbIgnoreChars, + ArgsDbGitRestore restoreArgs, + PrintStream printStream + ) { + super( + new PathPatchRunningDbGit(new ArgsExplicit("rm", "\"*\"", "-idx"), printStream), + new PathPatchDbGitLink(sourceDbLinkArgs, printStream), + new PathPatchCreatingFile(".dbgit/.dbignore", dbIgnoreChars), + new PathPatchDbGitAdd(printStream), + new PathPatchDbGitLink(targetDbLinkArgs, printStream), + new PathPatchDbGitRestore(restoreArgs, printStream) + ); + } +} diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/path/specific/dbgit/scenarios/PathAfterDbGitLinkAndAdd.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/path/specific/dbgit/scenarios/PathAfterDbGitLinkAndAdd.java index 36632f2..4c5f6e5 100644 --- a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/path/specific/dbgit/scenarios/PathAfterDbGitLinkAndAdd.java +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/path/specific/dbgit/scenarios/PathAfterDbGitLinkAndAdd.java @@ -2,22 +2,16 @@ import java.io.PrintStream; import java.nio.file.Path; -import ru.fusionsoft.dbgit.integration.primitives.patch.PathPatchCreatingFile; -import ru.fusionsoft.dbgit.integration.primitives.path.PathWithFiles; +import ru.fusionsoft.dbgit.integration.primitives.patch.specific.PathPatchDbGitLinkAndAdd; +import ru.fusionsoft.dbgit.integration.primitives.path.PathPatched; import ru.fusionsoft.dbgit.integration.primitives.printstream.DefaultPrintStream; -import ru.fusionsoft.dbgit.integration.primitives.args.ArgsExplicit; import ru.fusionsoft.dbgit.integration.primitives.args.specific.ArgsDbGitLink; -import ru.fusionsoft.dbgit.integration.primitives.path.PathAfterDbGitRun; -public class PathAfterDbGitLinkAndAdd extends PathAfterDbGitRun { - public PathAfterDbGitLinkAndAdd(ArgsDbGitLink argsDbGitLink, CharSequence ignoreChars, PrintStream printStream, Path origin) { +public class PathAfterDbGitLinkAndAdd extends PathPatched { + public PathAfterDbGitLinkAndAdd(ArgsDbGitLink argsDbGitLink, CharSequence ignoreChars, PrintStream printStream, Path workingDirectory) { super( - new ArgsExplicit("add", "\"*\"", "-v"), - printStream, - new PathWithFiles( - new PathPatchCreatingFile(".dbgit/.dbignore", ignoreChars), - new PathAfterDbGitRun(argsDbGitLink, printStream, origin) - ) + new PathPatchDbGitLinkAndAdd(argsDbGitLink, ignoreChars, printStream), + workingDirectory ); } diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/path/specific/dbgit/scenarios/PathAfterDbGitRestoreFromDbToDb.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/path/specific/dbgit/scenarios/PathAfterDbGitRestoreFromDbToDb.java new file mode 100644 index 0000000..57a37cc --- /dev/null +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/path/specific/dbgit/scenarios/PathAfterDbGitRestoreFromDbToDb.java @@ -0,0 +1,61 @@ +package ru.fusionsoft.dbgit.integration.primitives.path.specific.dbgit.scenarios; + +import java.io.PrintStream; +import java.nio.file.Path; +import ru.fusionsoft.dbgit.integration.primitives.patch.specific.PathPatchDbGitRestoreFromDbToDb; +import ru.fusionsoft.dbgit.integration.primitives.path.PathPatched; +import ru.fusionsoft.dbgit.integration.primitives.printstream.DefaultPrintStream; +import ru.fusionsoft.dbgit.integration.primitives.args.specific.ArgsDbGitLink; +import ru.fusionsoft.dbgit.integration.primitives.args.specific.ArgsDbGitRestore; + +public class PathAfterDbGitRestoreFromDbToDb extends PathPatched { + public PathAfterDbGitRestoreFromDbToDb( + ArgsDbGitLink sourceDbLinkArgs, + ArgsDbGitLink targetDbLinkArgs, + CharSequence dbIgnoreChars, + ArgsDbGitRestore restoreArgs, + PrintStream printStream, + Path workingDirectory + ) { + super( + new PathPatchDbGitRestoreFromDbToDb( + sourceDbLinkArgs, targetDbLinkArgs, dbIgnoreChars, restoreArgs, printStream + ), + workingDirectory + ); + } + + public PathAfterDbGitRestoreFromDbToDb( + ArgsDbGitLink sourceDbLinkArgs, + ArgsDbGitLink targetDbLinkArgs, + CharSequence dbIgnoreChars, + ArgsDbGitRestore restoreArgs, + Path workingDirectory + ) { + this( + sourceDbLinkArgs, + targetDbLinkArgs, + dbIgnoreChars, + restoreArgs, + new DefaultPrintStream(), + workingDirectory + ); + + } + + public PathAfterDbGitRestoreFromDbToDb( + ArgsDbGitLink sourceDbLinkArgs, + ArgsDbGitLink targetDbLinkArgs, + CharSequence dbIgnoreChars, + Path workingDirectory + ) { + this( + sourceDbLinkArgs, + targetDbLinkArgs, + dbIgnoreChars, + new ArgsDbGitRestore("-r", "-v"), + new DefaultPrintStream(), + workingDirectory + ); + } +} From 31fc5b961257d5a95de8855b44d7082f3aa0fb7c Mon Sep 17 00:00:00 2001 From: rocket Date: Fri, 23 Jul 2021 22:27:40 +0300 Subject: [PATCH 17/19] IT small refactor DBRestoreTableDataPostgres messages cleanup --- .../dbgit/core/GitMetaDataManager.java | 2 +- .../postgres/DBRestoreTableDataPostgres.java | 64 +++++++++++++------ src/main/resources/lang/eng.yaml | 3 + .../DbGitIntegrationTestBasic.java | 10 +-- .../dbgit/integration/SelfTest.java | 5 +- .../primitives/files/AutoDeletingPath.java | 18 ++++++ .../files/AutoDeletingTempFilePath.java | 33 ---------- .../primitives/files/TempFilePath.java | 27 ++++++++ .../ConnectionPatchExecutingStatement.java | 18 ++++++ .../patch/PathPatchRunningProcessFrom.java | 7 +- 10 files changed, 123 insertions(+), 64 deletions(-) create mode 100644 src/test/java/ru/fusionsoft/dbgit/integration/primitives/files/AutoDeletingPath.java delete mode 100644 src/test/java/ru/fusionsoft/dbgit/integration/primitives/files/AutoDeletingTempFilePath.java create mode 100644 src/test/java/ru/fusionsoft/dbgit/integration/primitives/files/TempFilePath.java diff --git a/src/main/java/ru/fusionsoft/dbgit/core/GitMetaDataManager.java b/src/main/java/ru/fusionsoft/dbgit/core/GitMetaDataManager.java index bf50d04..bed3510 100644 --- a/src/main/java/ru/fusionsoft/dbgit/core/GitMetaDataManager.java +++ b/src/main/java/ru/fusionsoft/dbgit/core/GitMetaDataManager.java @@ -308,7 +308,7 @@ public IMapMetaObject loadFileMetaData(boolean force) throws ExceptionDBGit { String filename = files.get(i); if (DBGitPath.isServiceFile(filename)) continue; - ConsoleWriter.println(DBGitLang.getInstance() + ConsoleWriter.detailsPrintln(DBGitLang.getInstance() .getValue("general", "meta", "loadFile") .withParams(filename) , messageLevel+1 diff --git a/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTableDataPostgres.java b/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTableDataPostgres.java index 4b5da5c..7514a81 100644 --- a/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTableDataPostgres.java +++ b/src/main/java/ru/fusionsoft/dbgit/postgres/DBRestoreTableDataPostgres.java @@ -163,7 +163,7 @@ public void restoreTableDataPostgres(MetaTableData restoreTableData, MetaTableDa if (!diffTableData.entriesOnlyOnRight().isEmpty()) { ConsoleWriter.detailsPrintln(lang.getValue("general", "restore", "deleting"), messageLevel); StringBuilder deleteQuery = new StringBuilder(); - + long insertedRowsNumber = 0; for (RowData rowData : diffTableData.entriesOnlyOnRight().values()) { StringJoiner fieldJoiner = new StringJoiner(","); StringJoiner valuejoiner = new StringJoiner(","); @@ -183,32 +183,36 @@ public void restoreTableDataPostgres(MetaTableData restoreTableData, MetaTableDa String delFields = "(" + fieldJoiner.toString() + ")"; String delValues = "(" + valuejoiner.toString() + ")"; - if (delValues.length() > 2){ + if (delValues.length() > 1){ final String ddl = MessageFormat.format( "DELETE FROM {0} WHERE {1} = {2};", tblNameEscaped, delFields, delValues ); deleteQuery.append(ddl).append("\n"); - ConsoleWriter.detailsPrintln(ddl, messageLevel + 1); + insertedRowsNumber++; +// ConsoleWriter.detailsPrintln(ddl, messageLevel + 1); } - if (deleteQuery.length() > 50000) { + if (deleteQuery.length() > 16384) { st.execute(deleteQuery.toString()); - deleteQuery = new StringBuilder(); + deleteQuery.setLength(0); } } if (deleteQuery.length() > 1) { st.execute(deleteQuery.toString()); } ConsoleWriter.detailsPrintGreen(lang.getValue("general", "ok")); + ConsoleWriter.detailsPrintGreen(lang.getValue("general", "restore", "deletedRows").withParams(String.valueOf(insertedRowsNumber))); } //UPDATE if (!diffTableData.entriesDiffering().isEmpty()) { ConsoleWriter.detailsPrintln(lang.getValue("general", "restore", "updating"), messageLevel); - String updateQuery = ""; + StringBuilder updateQuery = new StringBuilder(); Map primarykeys = new HashMap(); + long updatedRowsNumber = 0; for (ValueDifference diffRowData : diffTableData.entriesDiffering().values()) { + //TODO wow, wow, wow... what kind of key is used in that maps? if (!diffRowData.leftValue().getHashRow().equals(diffRowData.rightValue().getHashRow())) { Map tempCols = diffRowData.leftValue().getData(restoreTableData.getFields()); @@ -237,41 +241,61 @@ public void restoreTableDataPostgres(MetaTableData restoreTableData, MetaTableDa }); - updateQuery = "UPDATE " + tblNameEscaped + " SET (" + updFieldJoiner.toString() + ") = " + updateQuery.append("UPDATE " + tblNameEscaped + " SET (" + updFieldJoiner.toString() + ") = " + valuesToString(tempCols.values(), colTypes, restoreTableData.getFields()) + " " - + "WHERE (" + keyFieldsJoiner.toString() + ") = (" + keyValuesJoiner.toString() + ");\n"; + + "WHERE (" + keyFieldsJoiner.toString() + ") = (" + keyValuesJoiner.toString() + ");\n"); - ConsoleWriter.detailsPrintln(updateQuery, messageLevel); - st.execute(updateQuery); - updateQuery = ""; +// ConsoleWriter.detailsPrintln(updateQuery, messageLevel); + if (updateQuery.length() > 16384) { + st.execute(updateQuery.toString()); + updateQuery.setLength(0); + } } } } - - ConsoleWriter.detailsPrintGreen(lang.getValue("general", "ok")); if (updateQuery.length() > 1) { - ConsoleWriter.println(updateQuery, messageLevel); - st.execute(updateQuery); +// ConsoleWriter.println(updateQuery, messageLevel); + st.execute(updateQuery.toString()); + updatedRowsNumber++; } + + ConsoleWriter.detailsPrintGreen(lang.getValue("general", "ok")); + ConsoleWriter.detailsPrintGreen(lang.getValue("general", "restore", "updatedRows").withParams(String.valueOf(updatedRowsNumber))); } //INSERT if (!diffTableData.entriesOnlyOnLeft().isEmpty()) { ConsoleWriter.detailsPrintln(lang.getValue("general", "restore", "inserting"), messageLevel); + long insertedRowsCount = 0; + StringBuilder insertStatements = new StringBuilder(); + for (RowData rowData : diffTableData.entriesOnlyOnLeft().values()) { - String insertQuery = MessageFormat.format("INSERT INTO {0}{1}{2};" + insertStatements.append( MessageFormat.format("INSERT INTO {0}{1}{2};" , tblNameEscaped, fields , valuesToString(rowData.getData(restoreTableData.getFields()).values(), colTypes, restoreTableData.getFields()) - ); + )); + insertedRowsCount++; - ConsoleWriter.detailsPrintln(insertQuery, messageLevel+1); - st.execute(insertQuery); +// ConsoleWriter.detailsPrintln(insertQuery, messageLevel+1); + if (insertStatements.length() > 16384) { + st.execute(insertStatements.toString()); + insertStatements.setLength(0); + } + } + + if (insertStatements.length() > 1) { +// ConsoleWriter.println(updateQuery, messageLevel); + st.execute(insertStatements.toString()); + insertedRowsCount++; } + ConsoleWriter.detailsPrintln(lang.getValue("general", "ok"), messageLevel); - } + ConsoleWriter.detailsPrintGreen(lang.getValue("general", "restore", "insertedRows").withParams(String.valueOf(insertedRowsCount))); + + } } catch (Exception e) { throw new ExceptionDBGitRestore(lang.getValue("errors", "restore", "objectRestoreError").withParams(restoreTableData.getTable().getSchema() + "." + restoreTableData.getTable().getName()) diff --git a/src/main/resources/lang/eng.yaml b/src/main/resources/lang/eng.yaml index 593e2d7..83c735c 100644 --- a/src/main/resources/lang/eng.yaml +++ b/src/main/resources/lang/eng.yaml @@ -100,8 +100,11 @@ general: restoringTablesConstraints: Restoring constraints for all updated tables... addPk: Adding PK... inserting: Inserting... + insertedRows: Inserted {0} rows. deleting: Deleting... + deletedRows: Deleted {0} rows. updating: Updating... + updatedRows: Updated {0} rows. unsupportedTypes: Table {0} contains unsupported types, it will be skipped willMakeBackup: Backup option enabled, all objects will be backed up before making changes wontMakeBackup: Backup option DISABLED, no objects will be backed up before making changes diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java b/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java index aa4fb32..4c8c79d 100644 --- a/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java +++ b/src/test/java/ru/fusionsoft/dbgit/integration/DbGitIntegrationTestBasic.java @@ -313,25 +313,25 @@ public final void dbToDbRestoreWorksWithCustomTypes() { new ArgsDbGitLinkPgAuto("dvdrental"), new ArgsDbGitLinkPgAuto(new NameOfDefaultTargetTestDatabase()), new CharsDbIgnoreWithDataAndTypes(), - new ArgsDbGitRestore("-r"), - + new ArgsDbGitRestore("-r", "-v"), + new PathAfterDbGitRestoreFromDbToDb( new ArgsDbGitLinkPgAuto("pagilla"), new ArgsDbGitLinkPgAuto(new NameOfDefaultTargetTestDatabase()), new CharsDbIgnoreWithDataAndTypes(), - new ArgsDbGitRestore("-r"), + new ArgsDbGitRestore("-r", "-v"), new PathAfterDbGitRestoreFromDbToDb( new ArgsDbGitLinkPgAuto("dvdrental"), new ArgsDbGitLinkPgAuto(new NameOfDefaultTargetTestDatabase()), new CharsDbIgnoreWithDataAndTypes(), - new ArgsDbGitRestore("-r"), + new ArgsDbGitRestore("-r", "-v"), new PathAfterDbGitRestoreFromDbToDb( new ArgsDbGitLinkPgAuto("pagilla"), new ArgsDbGitLinkPgAuto(new NameOfDefaultTargetTestDatabase()), new CharsDbIgnoreWithDataAndTypes(), - new ArgsDbGitRestore("-r"), + new ArgsDbGitRestore("-r", "-v"), new PathPatched( new PathPatchUsingConnectionFromDbLink( new ConnectionPatchExecutingStatement("DROP SCHEMA IF EXISTS public CASCADE;") diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/SelfTest.java b/src/test/java/ru/fusionsoft/dbgit/integration/SelfTest.java index 73eacc6..b5bb0ad 100644 --- a/src/test/java/ru/fusionsoft/dbgit/integration/SelfTest.java +++ b/src/test/java/ru/fusionsoft/dbgit/integration/SelfTest.java @@ -7,7 +7,8 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import ru.fusionsoft.dbgit.integration.primitives.TestResult; -import ru.fusionsoft.dbgit.integration.primitives.files.AutoDeletingTempFilePath; +import ru.fusionsoft.dbgit.integration.primitives.files.AutoDeletingPath; +import ru.fusionsoft.dbgit.integration.primitives.files.TempFilePath; import ru.fusionsoft.dbgit.integration.primitives.patch.PathPatchCreatingFile; import ru.fusionsoft.dbgit.integration.primitives.path.PathNotProjectRoot; import ru.fusionsoft.dbgit.integration.primitives.path.PathPatched; @@ -195,7 +196,7 @@ public final void patchSequentalWorks() { @Test public final void tempFileWorks() throws IOException { Path path = new CurrentWorkingDirectory(); - try (AutoDeletingTempFilePath tempFilePath = new AutoDeletingTempFilePath(new CurrentWorkingDirectory(), "some")) { + try (AutoDeletingPath tempFilePath = new AutoDeletingPath(new TempFilePath(new CurrentWorkingDirectory(), "some"))) { path = tempFilePath; final String data = "123"; FileUtils.writeStringToFile(tempFilePath.toFile(), data); diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/files/AutoDeletingPath.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/files/AutoDeletingPath.java new file mode 100644 index 0000000..735867f --- /dev/null +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/files/AutoDeletingPath.java @@ -0,0 +1,18 @@ +package ru.fusionsoft.dbgit.integration.primitives.files; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import ru.fusionsoft.dbgit.integration.primitives.path.PathEnvelope; + +public class AutoDeletingPath extends PathEnvelope implements AutoCloseable { + + public AutoDeletingPath(Path origin) { + super( () -> origin ); + } + + @Override + public final void close() throws IOException { + Files.deleteIfExists(this.toFile().toPath()); + } +} diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/files/AutoDeletingTempFilePath.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/files/AutoDeletingTempFilePath.java deleted file mode 100644 index 9fabb1d..0000000 --- a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/files/AutoDeletingTempFilePath.java +++ /dev/null @@ -1,33 +0,0 @@ -package ru.fusionsoft.dbgit.integration.primitives.files; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Random; -import ru.fusionsoft.dbgit.integration.primitives.chars.CharsOf; -import ru.fusionsoft.dbgit.integration.primitives.path.PathEnvelope; - -public class AutoDeletingTempFilePath extends PathEnvelope implements AutoCloseable { - - public AutoDeletingTempFilePath(CharSequence fileName) { - super(()-> Paths.get(String.valueOf(fileName))); - } - public AutoDeletingTempFilePath(Path directory, CharSequence prefix) { - this( - new CharsOf<>( ()-> { - return directory.resolve( - prefix + - String.format("#%06x", new Random().nextInt(256 * 256 * 256)) - ) - .toAbsolutePath() - .toString(); - }) - ); - } - - @Override - public final void close() throws IOException { - Files.deleteIfExists(this.toFile().toPath()); - } -} diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/files/TempFilePath.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/files/TempFilePath.java new file mode 100644 index 0000000..7740f34 --- /dev/null +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/files/TempFilePath.java @@ -0,0 +1,27 @@ +package ru.fusionsoft.dbgit.integration.primitives.files; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Random; +import ru.fusionsoft.dbgit.integration.primitives.chars.CharsOf; +import ru.fusionsoft.dbgit.integration.primitives.path.PathEnvelope; + +public class TempFilePath extends PathEnvelope { + + public TempFilePath(CharSequence fileName) { + super(() -> Paths.get(String.valueOf(fileName))); + } + + public TempFilePath(Path directory, CharSequence prefix) { + this( + new CharsOf<>(() -> { + return directory.resolve( + prefix + + String.format("#%06x", new Random().nextInt(256 * 256 * 256)) + ) + .toAbsolutePath() + .toString(); + }) + ); + } +} diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/ConnectionPatchExecutingStatement.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/ConnectionPatchExecutingStatement.java index 53d90e8..6799e3c 100644 --- a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/ConnectionPatchExecutingStatement.java +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/ConnectionPatchExecutingStatement.java @@ -1,18 +1,36 @@ package ru.fusionsoft.dbgit.integration.primitives.patch; +import java.io.PrintStream; import java.sql.Connection; import java.sql.Statement; +import java.text.MessageFormat; import ru.fusionsoft.dbgit.integration.primitives.Patch; +import ru.fusionsoft.dbgit.integration.primitives.chars.LinesOfUnsafeScalar; +import ru.fusionsoft.dbgit.integration.primitives.printstream.DefaultPrintStream; public class ConnectionPatchExecutingStatement implements Patch { private final CharSequence sqlStatementChars; + private final PrintStream printStream; + public ConnectionPatchExecutingStatement(CharSequence sqlStatementChars, PrintStream printStream) { + this.sqlStatementChars = sqlStatementChars; + this.printStream = printStream; + } + public ConnectionPatchExecutingStatement(CharSequence sqlStatementChars) { this.sqlStatementChars = sqlStatementChars; + this.printStream = new DefaultPrintStream(); } @Override public final void apply(Connection connection) throws Exception { + printStream.println(MessageFormat.format( + "{0} # {1}", + connection.getMetaData().getURL(), + new LinesOfUnsafeScalar(sqlStatementChars).list().size() > 1 + ? "\n" + sqlStatementChars + : sqlStatementChars + )); try (final Statement statement = connection.createStatement()) { statement.execute(String.valueOf(sqlStatementChars)); } diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/PathPatchRunningProcessFrom.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/PathPatchRunningProcessFrom.java index 7dc91fa..4c52247 100644 --- a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/PathPatchRunningProcessFrom.java +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/PathPatchRunningProcessFrom.java @@ -12,7 +12,8 @@ import ru.fusionsoft.dbgit.integration.primitives.Patch; import ru.fusionsoft.dbgit.integration.primitives.chars.CharsOf; import ru.fusionsoft.dbgit.integration.primitives.chars.CharsOfLines; -import ru.fusionsoft.dbgit.integration.primitives.files.AutoDeletingTempFilePath; +import ru.fusionsoft.dbgit.integration.primitives.files.AutoDeletingPath; +import ru.fusionsoft.dbgit.integration.primitives.files.TempFilePath; public class PathPatchRunningProcessFrom implements Patch { @@ -34,8 +35,8 @@ public final void apply(Path workingDirectory) throws Exception { )); try ( - final AutoDeletingTempFilePath tempOutPath = new AutoDeletingTempFilePath(workingDirectory.resolve("../"), "out"); - final AutoDeletingTempFilePath tempErrPath = new AutoDeletingTempFilePath(workingDirectory.resolve("../"), "err"); + final AutoDeletingPath tempOutPath = new AutoDeletingPath(new TempFilePath(workingDirectory.resolve("../"), "out")); + final AutoDeletingPath tempErrPath = new AutoDeletingPath(new TempFilePath(workingDirectory.resolve("../"), "err")); ) { final Process process = new ProcessBuilder() From fa21cf138cb0bdfd323faa6250dbe6deab2a1d8d Mon Sep 17 00:00:00 2001 From: rocket Date: Fri, 23 Jul 2021 22:42:02 +0300 Subject: [PATCH 18/19] Fix UB when affected tables for backups come to update objects --- .../java/ru/fusionsoft/dbgit/command/CmdRestore.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/ru/fusionsoft/dbgit/command/CmdRestore.java b/src/main/java/ru/fusionsoft/dbgit/command/CmdRestore.java index b78abfb..bdcbc86 100644 --- a/src/main/java/ru/fusionsoft/dbgit/command/CmdRestore.java +++ b/src/main/java/ru/fusionsoft/dbgit/command/CmdRestore.java @@ -79,6 +79,7 @@ public void execute(CommandLine cmdLine) throws Exception { IMapMetaObject fileObjs = gmdm.loadFileMetaData(); IMapMetaObject updateObjs = new TreeMapMetaObject(); IMapMetaObject deleteObjs = new TreeMapMetaObject(); + IMapMetaObject backupObjs = new TreeMapMetaObject(); if (toMakeBackup) { ConsoleWriter.printlnColor(getLang().getValue("general", "restore", "willMakeBackup").toString(), Ansi.FColor.GREEN, messageLevel+1); } else { ConsoleWriter.printlnColor(getLang().getValue("general", "restore", "wontMakeBackup").toString(), Ansi.FColor.GREEN, messageLevel+1); } @@ -148,6 +149,7 @@ public void execute(CommandLine cmdLine) throws Exception { // # steps 1,2 are in GitMetaDataManager::restoreDatabase ConsoleWriter.println(getLang().getValue("general", "restore", "seekingToRestoreAdditional"), messageLevel+2); + Map updateObjectsCopy = new TreeMapMetaObject(updateObjs.values()); Map affectedTables = new TreeMapMetaObject(); Map foundTables = new TreeMapMetaObject(); do { @@ -155,12 +157,14 @@ public void execute(CommandLine cmdLine) throws Exception { dbObjs.values().stream() .filter(excluded -> { return excluded instanceof MetaTable - && ! updateObjs.containsKey(excluded.getName()) - && updateObjs.values().stream().anyMatch(excluded::dependsOn); + && ! updateObjectsCopy.containsKey(excluded.getName()) + && updateObjectsCopy.values().stream().anyMatch(excluded::dependsOn); }) .collect(Collectors.toMap(IMetaObject::getName, val -> val)); affectedTables.putAll(foundTables); - updateObjs.putAll(foundTables); + updateObjectsCopy.putAll(foundTables); + deleteObjs.putAll(foundTables); + backupObjs.putAll(foundTables); } while (!foundTables.isEmpty()); if(affectedTables.isEmpty()){ @@ -222,7 +226,6 @@ public void execute(CommandLine cmdLine) throws Exception { } if(toMakeBackup && toMakeChanges) { - IMapMetaObject backupObjs = new TreeMapMetaObject(); backupObjs.putAll(deleteObjs); backupObjs.putAll(updateObjs); adapter.getBackupAdapterFactory().getBackupAdapter(adapter).backupDatabase(backupObjs); From ae70f550b196c5d07657470080006fd87810d06e Mon Sep 17 00:00:00 2001 From: rocket Date: Fri, 23 Jul 2021 23:26:03 +0300 Subject: [PATCH 19/19] Add backups enabled in IT --- .../PathPatchDbGitRestoreFromDbToDb.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/specific/PathPatchDbGitRestoreFromDbToDb.java b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/specific/PathPatchDbGitRestoreFromDbToDb.java index 21f0305..952757b 100644 --- a/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/specific/PathPatchDbGitRestoreFromDbToDb.java +++ b/src/test/java/ru/fusionsoft/dbgit/integration/primitives/patch/specific/PathPatchDbGitRestoreFromDbToDb.java @@ -6,10 +6,11 @@ import ru.fusionsoft.dbgit.integration.primitives.args.ArgsExplicit; import ru.fusionsoft.dbgit.integration.primitives.args.specific.ArgsDbGitLink; import ru.fusionsoft.dbgit.integration.primitives.args.specific.ArgsDbGitRestore; +import ru.fusionsoft.dbgit.integration.primitives.chars.specific.dbgit.CharsDbGitConfigBackupEnabled; import ru.fusionsoft.dbgit.integration.primitives.patch.PathPatchCreatingFile; import ru.fusionsoft.dbgit.integration.primitives.patch.PathPatchRunningDbGit; -public class PathPatchDbGitRestoreFromDbToDb extends PatchSequential{ +public class PathPatchDbGitRestoreFromDbToDb extends PatchSequential { public PathPatchDbGitRestoreFromDbToDb( ArgsDbGitLink sourceDbLinkArgs, ArgsDbGitLink targetDbLinkArgs, @@ -18,12 +19,13 @@ public PathPatchDbGitRestoreFromDbToDb( PrintStream printStream ) { super( - new PathPatchRunningDbGit(new ArgsExplicit("rm", "\"*\"", "-idx"), printStream), - new PathPatchDbGitLink(sourceDbLinkArgs, printStream), - new PathPatchCreatingFile(".dbgit/.dbignore", dbIgnoreChars), - new PathPatchDbGitAdd(printStream), - new PathPatchDbGitLink(targetDbLinkArgs, printStream), - new PathPatchDbGitRestore(restoreArgs, printStream) + new PathPatchRunningDbGit(new ArgsExplicit("rm", "\"*\"", "-idx", "-v"), printStream), + new PathPatchDbGitLink(sourceDbLinkArgs, printStream), + new PathPatchCreatingFile(".dbgit/.dbignore", dbIgnoreChars), + new PathPatchDbGitAdd(printStream), + new PathPatchDbGitLink(targetDbLinkArgs, printStream), + new PathPatchCreatingFile(".dbgit/dbgitconfig", new CharsDbGitConfigBackupEnabled()), + new PathPatchDbGitRestore(restoreArgs, printStream) ); } }