diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml new file mode 100644 index 0000000..9e5bf12 --- /dev/null +++ b/.github/workflows/maven.yml @@ -0,0 +1,31 @@ +# This workflow will build a Java project with Maven, and cache/restore any dependencies to improve the workflow execution time +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-maven + +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Java CI with Maven + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up JDK 17 + uses: actions/setup-java@v3 + with: + java-version: '17' + distribution: 'temurin' + cache: maven + - name: Build with Maven + run: mvn -B package --file pom.xml diff --git a/README.md b/README.md index fc56243..794a7fb 100644 --- a/README.md +++ b/README.md @@ -138,6 +138,10 @@ Supplying `false` to these functions will disable their respective actions. CHANGELOG ========= +V 1.2.5 + - Handling of BOOLEAN +V 1.2.4 + - Handling of BIGINT, SMALLINT, FLOAT, REAL, DOUBLE, NUMERIC, DECIMAL V 1.2.1 - Raises a new runtime exception `MysqlBackup4JException` if the required properties are not configured diff --git a/install-into-local-repo.bat b/install-into-local-repo.bat new file mode 100644 index 0000000..bfb7432 --- /dev/null +++ b/install-into-local-repo.bat @@ -0,0 +1,3 @@ +call mvn install:install-file -Dfile="target/mysql-backup4j-1.2.2.jar" -DgroupId="com.smattme" -DartifactId="mysql-backup4j" -Dversion="1.2.2" -Dpackaging=jar -DlocalRepositoryPath="../bartleby/.m2" +certutil -hashfile "../bartleby/.m2/com/smattme/mysql-backup4j/1.2.2/mysql-backup4j-1.2.2.jar" MD5 > "../bartleby/.m2/com/smattme/mysql-backup4j/1.2.2/mysql-backup4j-1.2.2.jar.MD5" +certutil -hashfile "../bartleby/.m2/com/smattme/mysql-backup4j/1.2.2/mysql-backup4j-1.2.2.pom" MD5 > "../bartleby/.m2/com/smattme/mysql-backup4j/1.2.2/mysql-backup4j-1.2.2.pom.MD5" \ No newline at end of file diff --git a/pom.xml b/pom.xml index b098d14..3dfcff0 100644 --- a/pom.xml +++ b/pom.xml @@ -1,12 +1,11 @@ + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - com.smattme + fr.neolegal mysql-backup4j - 1.2.1 + 1.2.7 jar ${project.groupId}:${project.artifactId} @@ -14,7 +13,7 @@ This is a simple library for backing up mysql databases and sending to emails, cloud storage and so on. It also provide a method for programmatically, importing SQL queries generated during the export process, - https://github.com/SeunMatt/mysql-backup4j + https://github.com/nicolasriousset/mysql-backup4j @@ -30,46 +29,70 @@ SmattMe https://smattme.com + + Nicolas Riousset + nicolas@neolegal.fr + NeoLegal + https://neolegal.fr + - scm:git:git://github.com/SeunMatt/mysql-backup4j.git - scm:git:ssh://github.com:SeunMatt/mysql-backup4j.git - https://github.com/SeunMatt/mysql-backup4j/tree/master + scm:git:git://github.com/nicolasriousset/mysql-backup4j.git + scm:git:ssh://github.com:nicolasriousset/mysql-backup4j.git + https://github.com/nicolasriousset/mysql-backup4j/tree/master + + + 3.11.0 + 17 + + mysql mysql-connector-java - 8.0.21 + 8.0.33 org.zeroturnaround zt-zip - 1.12 + 1.16 jar - javax.mail - mail - 1.5.0-b01 + com.sun.mail + jakarta.mail + 2.0.1 org.slf4j slf4j-api - 1.7.25 + 2.0.9 org.slf4j slf4j-simple - 1.7.25 + 2.0.9 test org.junit.jupiter junit-jupiter-api - 5.7.0 + 5.10.0 + test + + + org.testcontainers + junit-jupiter + 1.17.6 + test + + + org.testcontainers + mysql + 1.17.6 test @@ -77,11 +100,11 @@ ossrh - https://oss.sonatype.org/content/repositories/snapshots + https://s01.oss.sonatype.org/content/repositories/snapshots/ ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ + https://s01.oss.sonatype.org/content/repositories/releases/ @@ -90,26 +113,24 @@ org.apache.maven.plugins maven-compiler-plugin - 3.1 + ${maven.version} - 1.8 - 1.8 + ${java.version} + ${java.version} org.apache.maven.plugins maven-surefire-plugin - 3.0.0-M5 + 3.1.2 - - com.smattme.MysqlBackup4JIntegrationTest - + Integration org.apache.maven.plugins maven-javadoc-plugin - 3.2.0 + 3.5.0 attach-javadocs @@ -122,7 +143,7 @@ org.apache.maven.plugins maven-source-plugin - 3.0.1 + 3.3.0 attach-sources @@ -135,7 +156,7 @@ org.apache.maven.plugins maven-gpg-plugin - 1.6 + 3.1.0 sign-artifacts @@ -149,11 +170,11 @@ org.sonatype.plugins nexus-staging-maven-plugin - 1.6.8 + 1.6.13 true ossrh - https://oss.sonatype.org/ + https://s01.oss.sonatype.org/ false diff --git a/src/main/java/com/smattme/EmailService.java b/src/main/java/com/smattme/EmailService.java index b256ded..ccbb30c 100644 --- a/src/main/java/com/smattme/EmailService.java +++ b/src/main/java/com/smattme/EmailService.java @@ -3,11 +3,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import javax.mail.*; -import javax.mail.internet.InternetAddress; -import javax.mail.internet.MimeBodyPart; -import javax.mail.internet.MimeMessage; -import javax.mail.internet.MimeMultipart; +import jakarta.mail.*; +import jakarta.mail.internet.InternetAddress; +import jakarta.mail.internet.MimeBodyPart; +import jakarta.mail.internet.MimeMessage; +import jakarta.mail.internet.MimeMultipart; import java.io.File; import java.util.Properties; diff --git a/src/main/java/com/smattme/MysqlExportService.java b/src/main/java/com/smattme/MysqlExportService.java index 005c46c..23e4cb4 100644 --- a/src/main/java/com/smattme/MysqlExportService.java +++ b/src/main/java/com/smattme/MysqlExportService.java @@ -68,6 +68,7 @@ public class MysqlExportService { public static final String JDBC_CONNECTION_STRING = "JDBC_CONNECTION_STRING"; public static final String JDBC_DRIVER_NAME = "JDBC_DRIVER_NAME"; public static final String SQL_FILE_NAME = "SQL_FILE_NAME"; + public static final String MAX_INSERT_SIZE = "MAX_INSERT_SIZE"; public MysqlExportService(Properties properties) { @@ -136,9 +137,7 @@ private String getTableInsertStatement(String table) throws SQLException { while ( rs.next() ) { String qtbl = rs.getString(1); String query = rs.getString(2); - sql.append("\n\n--"); - sql.append("\n").append(MysqlBaseService.SQL_START_PATTERN).append(" table dump : ").append(qtbl); - sql.append("\n--\n\n"); + sql.append(buildStartPattern(" table dump : " + qtbl)); if(addIfNotExists) { query = query.trim().replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS"); @@ -147,9 +146,7 @@ private String getTableInsertStatement(String table) throws SQLException { sql.append(query).append(";\n\n"); } - sql.append("\n\n--"); - sql.append("\n").append(MysqlBaseService.SQL_END_PATTERN).append(" table dump : ").append(table); - sql.append("\n--\n\n"); + sql.append(buildEndPattern(" table dump : " + table)); } return sql.toString(); @@ -171,18 +168,14 @@ private String getCreateViewStatement(String view) throws SQLException { rs = stmt.executeQuery("SHOW CREATE VIEW " + "`" + view + "`;"); while ( rs.next() ) { String viewName = rs.getString(1); - String viewQuery = rs.getString(2); - sql.append("\n\n--"); - sql.append("\n").append(MysqlBaseService.SQL_START_PATTERN).append(" view dump : ").append(view); - sql.append("\n--\n\n"); + String viewQuery = rs.getString(2); + sql.append(buildStartPattern(" view dump : " + view)); String finalQuery = "CREATE OR REPLACE VIEW `" + viewName + "` " + (viewQuery.substring(viewQuery.indexOf("AS")).trim()); sql.append(finalQuery).append(";\n\n"); } - sql.append("\n\n--"); - sql.append("\n").append(MysqlBaseService.SQL_END_PATTERN).append(" view dump : ").append(view); - sql.append("\n--\n\n"); + sql.append(buildEndPattern(" view dump : " + view)); } return sql.toString(); @@ -216,74 +209,37 @@ private String getDataInsertStatement(String table) throws SQLException { //temporarily disable foreign key constraint sql.append("\n/*!40000 ALTER TABLE `").append(table).append("` DISABLE KEYS */;\n"); - sql.append("\n--\n") - .append(MysqlBaseService.SQL_START_PATTERN).append(" table insert : ").append(table) - .append("\n--\n"); + final String queryStart = buildInsertQueryStart(table, rs); + Long maxInsertSize = getMaxInsertSize(); + int currentQueryStartPosition = sql.length(); - sql.append("INSERT INTO `").append(table).append("`("); - - ResultSetMetaData metaData = rs.getMetaData(); - int columnCount = metaData.getColumnCount(); - - //generate the column names that are present - //in the returned result set - //at this point the insert is INSERT INTO (`col1`, `col2`, ...) - for(int i = 0; i < columnCount; i++) { - sql.append("`") - .append(metaData.getColumnName( i + 1)) - .append("`, "); - } - - //remove the last whitespace and comma - sql.deleteCharAt(sql.length() - 1).deleteCharAt(sql.length() - 1).append(") VALUES \n"); //now we're going to build the values for data insertion rs.beforeFirst(); while(rs.next()) { - sql.append("("); - for(int i = 0; i < columnCount; i++) { - - int columnType = metaData.getColumnType(i + 1); - int columnIndex = i + 1; - - //this is the part where the values are processed based on their type - if(Objects.isNull(rs.getObject(columnIndex))) { - sql.append("").append(rs.getObject(columnIndex)).append(", "); - } - else if( columnType == Types.INTEGER || columnType == Types.TINYINT || columnType == Types.BIT) { - sql.append(rs.getInt(columnIndex)).append(", "); - } - else { - - String val = rs.getString(columnIndex); - //escape the single quotes that might be in the value - val = val.replace("'", "\\'"); - - sql.append("'").append(val).append("', "); - } - } - - //now that we're done with a row - //let's remove the last whitespace and comma - sql.deleteCharAt(sql.length() - 1).deleteCharAt(sql.length() - 1); - - //if this is the last row, just append a closing - //parenthesis otherwise append a closing parenthesis and a comma - //for the next set of values - if(rs.isLast()) { - sql.append(")"); + String insertQueryValues = buildInsertQueryValues(rs); + + int currentInsertQueryLength = sql.length() - currentQueryStartPosition; + boolean newInsertQueryRequired = currentInsertQueryLength == 0 || (maxInsertSize > 0 && currentInsertQueryLength >= maxInsertSize); + if (newInsertQueryRequired) { + if (currentInsertQueryLength > 0) { + sql.append(";\n"); + sql.append(buildEndPattern(" table insert : " + table)); + } + sql.append(buildStartPattern(" table insert : " + table)); + currentQueryStartPosition = sql.length(); + sql.append(queryStart); } else { - sql.append("),\n"); + sql.append(",\n"); } + sql.append(insertQueryValues); } //now that we are done processing the entire row //let's add the terminator sql.append(";"); - sql.append("\n--\n") - .append(MysqlBaseService.SQL_END_PATTERN).append(" table insert : ").append(table) - .append("\n--\n"); + sql.append(buildEndPattern(" table insert : " + table)); //enable FK constraint sql.append("\n/*!40000 ALTER TABLE `").append(table).append("` ENABLE KEYS */;\n"); @@ -291,6 +247,80 @@ else if( columnType == Types.INTEGER || columnType == Types.TINYINT || columnTyp return sql.toString(); } + String buildStartPattern(String comment) { + return "\n--\n" + MysqlBaseService.SQL_START_PATTERN + comment + "\n--\n"; + } + + String buildEndPattern(String comment) { + return "\n--\n" + MysqlBaseService.SQL_END_PATTERN + comment + "\n--\n"; + } + + private String buildInsertQueryValues(ResultSet rs) throws SQLException { + StringBuilder sql = new StringBuilder(); + ResultSetMetaData metaData = rs.getMetaData(); + int columnCount = metaData.getColumnCount(); + sql.append("("); + for(int i = 0; i < columnCount; i++) { + + int columnIndex = i + 1; + int columnType = metaData.getColumnType(columnIndex); + + //this is the part where the values are processed based on their type + if(Objects.isNull(rs.getObject(columnIndex))) { + sql.append("").append(rs.getObject(columnIndex)).append(", "); + } + else if( columnType == Types.BIGINT || columnType == Types.INTEGER || columnType == Types.SMALLINT || columnType == Types.TINYINT || columnType == Types.BIT || columnType == Types.FLOAT || columnType == Types.REAL || columnType == Types.DOUBLE || columnType == Types.NUMERIC || columnType == Types.DECIMAL || columnType == Types.BOOLEAN) { + sql.append(rs.getInt(columnIndex)).append(", "); + } + else { + + String val = rs.getString(columnIndex); + //escape the single quotes that might be in the value + val = val.replace("'", "\\'"); + + sql.append("'").append(val).append("', "); + } + } + + //now that we're done with a row + //let's remove the last whitespace and comma + sql.deleteCharAt(sql.length() - 1).deleteCharAt(sql.length() - 1); + + sql.append(")"); + + return sql.toString(); + } + + + private Long getMaxInsertSize() { + try { + String prop = properties.getProperty(MAX_INSERT_SIZE); + return prop != null ? Long.parseLong(prop) : 0L; + } catch (Exception e) { + return 0L; + } + } + + private String buildInsertQueryStart(String table, ResultSet rs) throws SQLException { + StringBuilder sql = new StringBuilder(); + sql.append("INSERT INTO `").append(table).append("`("); + + ResultSetMetaData metaData = rs.getMetaData(); + int columnCount = metaData.getColumnCount(); + + //generate the column names that are present + //in the returned result set + //at this point the insert is INSERT INTO (`col1`, `col2`, ...) + for(int i = 0; i < columnCount; i++) { + sql.append("`") + .append(metaData.getColumnName( i + 1)) + .append("`, "); + } + + //remove the last whitespace and comma + sql.deleteCharAt(sql.length() - 1).deleteCharAt(sql.length() - 1).append(") VALUES \n"); + return sql.toString(); + } /** * This is the entry function that'll diff --git a/src/test/java/com/smattme/MysqlBackup4JIntegrationTest.java b/src/test/java/com/smattme/MysqlBackup4JIntegrationTest.java index 9e07454..ce0834e 100644 --- a/src/test/java/com/smattme/MysqlBackup4JIntegrationTest.java +++ b/src/test/java/com/smattme/MysqlBackup4JIntegrationTest.java @@ -2,10 +2,13 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.testcontainers.containers.MySQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; import java.io.File; import java.nio.file.Files; @@ -21,15 +24,32 @@ * Created by seun_ on 10-Oct-20. * */ +@Tag("Integration") +@Testcontainers class MysqlBackup4JIntegrationTest { private Logger logger = LoggerFactory.getLogger(getClass()); private static final String TEST_DB = "mysqlbackup4j_test"; private static final String RESTORED_DB = "mysqlbackup4j_restored"; private static final String DB_USERNAME = "travis"; - private static final String DB_PASSWORD = ""; + private static final String DB_PASSWORD = "test"; private static final String DRIVER_CLASS_NAME = "com.mysql.cj.jdbc.Driver"; + @Container + private static final MySQLContainer mySQLContainer = new MySQLContainer<>("mysql:8.0.30") + .withDatabaseName(TEST_DB) + .withUsername(DB_USERNAME) + .withPassword(DB_PASSWORD) + .withExposedPorts(3306) + .withInitScript("sample_database.sql"); + + @Container + private static final MySQLContainer mySQLRestoredContainer = new MySQLContainer<>("mysql:8.0.30") + .withDatabaseName(RESTORED_DB) + .withUsername(DB_USERNAME) + .withPassword(DB_PASSWORD) + .withExposedPorts(3306); + @BeforeAll static void setUp() { System.setProperty("org.slf4j.simpleLogger.defaultLogLevel", "debug"); @@ -49,10 +69,12 @@ void givenDBCredentials_whenExportDatabaseAndImportDatabase_thenBackUpAndRestore properties.setProperty(MysqlExportService.JDBC_DRIVER_NAME, DRIVER_CLASS_NAME); properties.setProperty(MysqlExportService.ADD_IF_NOT_EXISTS, "true"); - properties.setProperty(MysqlExportService.TEMP_DIR, new File("external").getPath()); properties.setProperty(MysqlExportService.SQL_FILE_NAME, "test_output_file_name"); + properties.setProperty(MysqlExportService.JDBC_CONNECTION_STRING, mySQLContainer.getJdbcUrl() + "?useUnicode=true&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=UTC&useSSL=false"); + + MysqlExportService mysqlExportService = new MysqlExportService(properties); mysqlExportService.export(); @@ -70,7 +92,7 @@ void givenDBCredentials_whenExportDatabaseAndImportDatabase_thenBackUpAndRestore String sql = new String(Files.readAllBytes(sqlFile.toPath())); MysqlImportService res = MysqlImportService.builder() .setJdbcDriver("com.mysql.cj.jdbc.Driver") - .setDatabase(RESTORED_DB) + .setJdbcConnString(mySQLRestoredContainer.getJdbcUrl()) .setSqlString(sql) .setUsername(DB_USERNAME) .setPassword(DB_PASSWORD) @@ -91,7 +113,7 @@ void givenJDBCConString_whenExportDatabaseAndImportDatabase_thenBackUpAndRestore properties.setProperty(MysqlExportService.DB_USERNAME, DB_USERNAME); properties.setProperty(MysqlExportService.DB_PASSWORD, DB_PASSWORD); properties.setProperty(MysqlExportService.DB_NAME, TEST_DB); - properties.setProperty(MysqlExportService.JDBC_CONNECTION_STRING, "jdbc:mysql://localhost:3306/" + TEST_DB + "?useUnicode=true&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=UTC&useSSL=false"); + properties.setProperty(MysqlExportService.JDBC_CONNECTION_STRING, mySQLContainer.getJdbcUrl() + "?useUnicode=true&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=UTC&useSSL=false"); properties.setProperty(MysqlExportService.PRESERVE_GENERATED_ZIP, "true"); properties.setProperty(MysqlExportService.PRESERVE_GENERATED_SQL_FILE, "true"); @@ -119,7 +141,7 @@ void givenJDBCConString_whenExportDatabaseAndImportDatabase_thenBackUpAndRestore String sql = new String(Files.readAllBytes(sqlFile.toPath())); boolean res = MysqlImportService.builder() .setSqlString(sql) - .setJdbcConnString("jdbc:mysql://localhost:3306/" + RESTORED_DB + "?useUnicode=true&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=UTC&useSSL=false") + .setJdbcConnString(mySQLRestoredContainer.getJdbcUrl() + "?useUnicode=true&useJDBCCompliantTimezoneShift=true&useLegacyDatetimeCode=false&serverTimezone=UTC&useSSL=false") .setUsername(DB_USERNAME) .setPassword(DB_PASSWORD) .setDatabase(RESTORED_DB) @@ -134,7 +156,7 @@ void givenJDBCConString_whenExportDatabaseAndImportDatabase_thenBackUpAndRestore private void assertDatabaseBackedUp() throws Exception { - Connection connection = MysqlBaseService.connect(DB_USERNAME, DB_PASSWORD, RESTORED_DB, DRIVER_CLASS_NAME); + Connection connection = MysqlBaseService.connectWithURL(DB_USERNAME, DB_PASSWORD, mySQLRestoredContainer.getJdbcUrl(), DRIVER_CLASS_NAME); Statement statement = connection.createStatement(ResultSet.TYPE_SCROLL_INSENSITIVE, ResultSet.CONCUR_READ_ONLY); statement.execute("SELECT COUNT(1) as total FROM users"); ResultSet resultSet = statement.getResultSet(); @@ -142,4 +164,4 @@ private void assertDatabaseBackedUp() throws Exception { assertTrue(resultSet.getLong("total") > 0); } -} \ No newline at end of file +}