From 3eca110dfcdbc62ad5b192d984847593a0139575 Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Mon, 17 Aug 2020 21:53:06 +0200 Subject: [PATCH 1/5] Fix cursor in Django 3.1 --- .github/workflows/main.yml | 12 ++++++++++++ .travis.yml | 6 ++++++ sql_server/pyodbc/creation.py | 13 ++++++++++--- tox.ini | 2 ++ 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 03205a8b..4cdf48d8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -41,11 +41,14 @@ jobs: tox_env: - "py36-django22" - "py36-django30" + - "py36-django31" - "py37-django22" - "py37-django30" + - "py37-django31" - "py38-django30" + - "py38-django31" include: - python: "3.6" @@ -54,15 +57,24 @@ jobs: - python: "3.6" tox_env: "py36-django30" + - python: "3.6" + tox_env: "py36-django31" + - python: "3.7" tox_env: "py37-django22" - python: "3.7" tox_env: "py37-django30" + - python: "3.7" + tox_env: "py37-django31" + - python: "3.8" tox_env: "py38-django30" + - python: "3.8" + tox_env: "py38-django31" + steps: - uses: actions/checkout@v2 diff --git a/.travis.yml b/.travis.yml index a938aad3..6b8f8425 100644 --- a/.travis.yml +++ b/.travis.yml @@ -41,19 +41,25 @@ matrix: - { before_install: *linux_before_install, python: "3.6", os: linux, env: TOX_ENV=py36-django22 } - { before_install: *linux_before_install, python: "3.6", os: linux, env: TOX_ENV=py36-django30 } + - { before_install: *linux_before_install, python: "3.6", os: linux, env: TOX_ENV=py36-django31 } - { before_install: *linux_before_install, python: "3.7", os: linux, env: TOX_ENV=py37-django22 } - { before_install: *linux_before_install, python: "3.7", os: linux, env: TOX_ENV=py37-django30 } + - { before_install: *linux_before_install, python: "3.7", os: linux, env: TOX_ENV=py37-django31 } - { before_install: *linux_before_install, python: "3.8", os: linux, env: TOX_ENV=py38-django30 } + - { before_install: *linux_before_install, python: "3.8", os: linux, env: TOX_ENV=py38-django31 } - { before_install: *win_before_install, language: sh, python: "3.6", os: windows, env: TOX_ENV=py36-django22 } - { before_install: *win_before_install, language: sh, python: "3.6", os: windows, env: TOX_ENV=py36-django30 } + - { before_install: *win_before_install, language: sh, python: "3.6", os: windows, env: TOX_ENV=py36-django31 } - { before_install: *win_before_install, language: sh, python: "3.7", os: windows, env: TOX_ENV=py37-django22 } - { before_install: *win_before_install, language: sh, python: "3.7", os: windows, env: TOX_ENV=py37-django30 } + - { before_install: *win_before_install, language: sh, python: "3.7", os: windows, env: TOX_ENV=py37-django31 } - { before_install: *win_before_install, language: sh, python: "3.8", os: windows, env: TOX_ENV=py38-django30 } + - { before_install: *win_before_install, language: sh, python: "3.8", os: windows, env: TOX_ENV=py38-django31 } diff --git a/sql_server/pyodbc/creation.py b/sql_server/pyodbc/creation.py index 61745b57..eb0cc890 100644 --- a/sql_server/pyodbc/creation.py +++ b/sql_server/pyodbc/creation.py @@ -1,10 +1,17 @@ import binascii import os +import django from django.db.backends.base.creation import BaseDatabaseCreation class DatabaseCreation(BaseDatabaseCreation): + @property + def cursor(self): + if django.VERSION >= (3, 1): + return self.connection._nodb_cursor + + return self.connection._nodb_connection.cursor def _destroy_test_db(self, test_database_name, verbosity): """ @@ -14,7 +21,7 @@ def _destroy_test_db(self, test_database_name, verbosity): # ourselves. Connect to the previous database (not the test database) # to do so, because it's not allowed to delete a database while being # connected to it. - with self.connection._nodb_connection.cursor() as cursor: + with self.cursor() as cursor: to_azure_sql_db = self.connection.to_azure_sql_db if not to_azure_sql_db: cursor.execute("ALTER DATABASE %s SET SINGLE_USER WITH ROLLBACK IMMEDIATE" @@ -36,7 +43,7 @@ def enable_clr(self): This function will not fail if current user doesn't have permissions to enable clr, and clr is already enabled """ - with self._nodb_connection.cursor() as cursor: + with self.cursor() as cursor: # check whether clr is enabled cursor.execute(''' SELECT value FROM sys.configurations @@ -86,7 +93,7 @@ def install_regex_clr(self, database_name): self.enable_clr() - with self._nodb_connection.cursor() as cursor: + with self.cursor() as cursor: for s in sql: cursor.execute(s) diff --git a/tox.ini b/tox.ini index 1b56e027..6bc0e3ae 100644 --- a/tox.ini +++ b/tox.ini @@ -2,6 +2,7 @@ envlist = {py36,py37}-django22, {py36,py37,py38}-django30, + {py36,py37,py38}-django31, [testenv] passenv = @@ -19,4 +20,5 @@ commands = deps = django22: django==2.2.* django30: django>=3.0a1,<3.1 + django31: django>=3.1,<3.2 dj-database-url==0.5.0 From d5440e09d77fb7ec3bb0967b138a29ca398ba9ca Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Mon, 17 Aug 2020 22:27:13 +0200 Subject: [PATCH 2/5] Fix deferrable error --- sql_server/pyodbc/features.py | 1 + sql_server/pyodbc/schema.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/sql_server/pyodbc/features.py b/sql_server/pyodbc/features.py index 6563b9d9..1e184217 100644 --- a/sql_server/pyodbc/features.py +++ b/sql_server/pyodbc/features.py @@ -22,6 +22,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): requires_literal_defaults = True requires_sqlparse_for_splitting = False supports_boolean_expr_in_select_clause = False + supports_deferrable_unique_constraints = False supports_ignore_conflicts = False supports_index_on_text_field = False supports_paramstyle_pyformat = False diff --git a/sql_server/pyodbc/schema.py b/sql_server/pyodbc/schema.py index 9abcbd04..38bc80aa 100644 --- a/sql_server/pyodbc/schema.py +++ b/sql_server/pyodbc/schema.py @@ -694,6 +694,7 @@ def create_unique_name(*args, **kwargs): name=name, columns=columns, condition=' WHERE ' + condition, + deferrable='' ) if self.connection.features.supports_partial_indexes else None else: return Statement( @@ -701,6 +702,7 @@ def create_unique_name(*args, **kwargs): table=table, name=name, columns=columns, + deferrable='' ) def _create_index_sql(self, model, fields, *, name=None, suffix='', using='', From 41d6123ecf43ea5aae1c8d955fb4be9c25350397 Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Mon, 17 Aug 2020 22:29:15 +0200 Subject: [PATCH 3/5] Fix conditional expression --- sql_server/pyodbc/operations.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/sql_server/pyodbc/operations.py b/sql_server/pyodbc/operations.py index 74b1c009..6af85bab 100644 --- a/sql_server/pyodbc/operations.py +++ b/sql_server/pyodbc/operations.py @@ -1,9 +1,13 @@ import datetime import uuid import warnings +import django from django.conf import settings from django.db.backends.base.operations import BaseDatabaseOperations +from django.db.models import Exists, ExpressionWrapper +from django.db.models.expressions import RawSQL +from django.db.models.sql.where import WhereNode from django.utils import timezone from django.utils.encoding import force_str @@ -440,3 +444,18 @@ def time_trunc_sql(self, lookup_type, field_name): elif lookup_type == 'second': sql = "CONVERT(time, SUBSTRING(CONVERT(varchar, %s, 114), 0, 9))" % field_name return sql + + def conditional_expression_supported_in_where_clause(self, expression): + """ + Following "Moved conditional expression wrapping to the Exact lookup" in django 3.1 + https://github.com/django/django/commit/37e6c5b79bd0529a3c85b8c478e4002fd33a2a1d + """ + if django.VERSION >= (3, 1): + if isinstance(expression, (Exists, WhereNode)): + return True + if isinstance(expression, ExpressionWrapper) and expression.conditional: + return self.conditional_expression_supported_in_where_clause(expression.expression) + if isinstance(expression, RawSQL) and expression.conditional: + return True + return False + return True From 05da7a2821f85885fd79a167b2fe246419d861a2 Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Tue, 18 Aug 2020 19:39:38 +0200 Subject: [PATCH 4/5] Fix sql_flush --- sql_server/pyodbc/operations.py | 117 +++++++++++++++++++------------- 1 file changed, 69 insertions(+), 48 deletions(-) diff --git a/sql_server/pyodbc/operations.py b/sql_server/pyodbc/operations.py index 6af85bab..12a0c8d6 100644 --- a/sql_server/pyodbc/operations.py +++ b/sql_server/pyodbc/operations.py @@ -314,7 +314,7 @@ def savepoint_rollback_sql(self, sid): """ return "ROLLBACK TRANSACTION %s" % sid - def sql_flush(self, style, tables, sequences, allow_cascade=False): + def sql_flush(self, style, tables, *args, allow_cascade=False, **kwargs): """ Returns a list of SQL statements required to remove all data from the given database tables (without actually removing the tables @@ -329,55 +329,76 @@ def sql_flush(self, style, tables, sequences, allow_cascade=False): The `allow_cascade` argument determines whether truncation may cascade to tables with foreign keys pointing the tables being truncated. """ - if tables: - # Cannot use TRUNCATE on tables that are referenced by a FOREIGN KEY - # So must use the much slower DELETE - from django.db import connections - cursor = connections[self.connection.alias].cursor() - # Try to minimize the risks of the braindeaded inconsistency in - # DBCC CHEKIDENT(table, RESEED, n) behavior. - seqs = [] - for seq in sequences: - cursor.execute("SELECT COUNT(*) FROM %s" % self.quote_name(seq["table"])) - rowcnt = cursor.fetchone()[0] - elem = {} - if rowcnt: - elem['start_id'] = 0 - else: - elem['start_id'] = 1 - elem.update(seq) - seqs.append(elem) - COLUMNS = "TABLE_NAME, CONSTRAINT_NAME" - WHERE = "CONSTRAINT_TYPE not in ('PRIMARY KEY','UNIQUE')" - cursor.execute( - "SELECT {} FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE {}".format(COLUMNS, WHERE)) - fks = cursor.fetchall() - sql_list = ['ALTER TABLE %s NOCHECK CONSTRAINT %s;' % - (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks] - sql_list.extend(['%s %s %s;' % (style.SQL_KEYWORD('DELETE'), style.SQL_KEYWORD('FROM'), - style.SQL_FIELD(self.quote_name(table))) for table in tables]) - - if self.connection.to_azure_sql_db and self.connection.sql_server_version < 2014: - warnings.warn("Resetting identity columns is not supported " - "on this versios of Azure SQL Database.", - RuntimeWarning) + + if not tables: + return [] + + if django.VERSION >= (3, 1): + truncated_tables = {table.upper() for table in tables} + constraints = set() + for table in tables: + for foreign_table, constraint in self._foreign_key_constraints(table, recursive=allow_cascade): + if allow_cascade: + truncated_tables.add(foreign_table) + constraints.add((foreign_table, constraint)) + + if kwargs['reset_sequences']: + sequences = [ + sequence + for sequence in self.connection.introspection.sequence_list() + if sequence['table'].upper() in truncated_tables + ] else: - # Then reset the counters on each table. - sql_list.extend(['%s %s (%s, %s, %s) %s %s;' % ( - style.SQL_KEYWORD('DBCC'), - style.SQL_KEYWORD('CHECKIDENT'), - style.SQL_FIELD(self.quote_name(seq["table"])), - style.SQL_KEYWORD('RESEED'), - style.SQL_FIELD('%d' % seq['start_id']), - style.SQL_KEYWORD('WITH'), - style.SQL_KEYWORD('NO_INFOMSGS'), - ) for seq in seqs]) - - sql_list.extend(['ALTER TABLE %s CHECK CONSTRAINT %s;' % - (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks]) - return sql_list + sequences = [] else: - return [] + sequences = args[0] + + # Cannot use TRUNCATE on tables that are referenced by a FOREIGN KEY + # So must use the much slower DELETE + from django.db import connections + cursor = connections[self.connection.alias].cursor() + # Try to minimize the risks of the braindeaded inconsistency in + # DBCC CHEKIDENT(table, RESEED, n) behavior. + seqs = [] + for seq in sequences: + cursor.execute("SELECT COUNT(*) FROM %s" % self.quote_name(seq["table"])) + rowcnt = cursor.fetchone()[0] + elem = {} + if rowcnt: + elem['start_id'] = 0 + else: + elem['start_id'] = 1 + elem.update(seq) + seqs.append(elem) + COLUMNS = "TABLE_NAME, CONSTRAINT_NAME" + WHERE = "CONSTRAINT_TYPE not in ('PRIMARY KEY','UNIQUE')" + cursor.execute( + "SELECT {} FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE {}".format(COLUMNS, WHERE)) + fks = cursor.fetchall() + sql_list = ['ALTER TABLE %s NOCHECK CONSTRAINT %s;' % + (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks] + sql_list.extend(['%s %s %s;' % (style.SQL_KEYWORD('DELETE'), style.SQL_KEYWORD('FROM'), + style.SQL_FIELD(self.quote_name(table))) for table in tables]) + + if self.connection.to_azure_sql_db and self.connection.sql_server_version < 2014: + warnings.warn("Resetting identity columns is not supported " + "on this versios of Azure SQL Database.", + RuntimeWarning) + else: + # Then reset the counters on each table. + sql_list.extend(['%s %s (%s, %s, %s) %s %s;' % ( + style.SQL_KEYWORD('DBCC'), + style.SQL_KEYWORD('CHECKIDENT'), + style.SQL_FIELD(self.quote_name(seq["table"])), + style.SQL_KEYWORD('RESEED'), + style.SQL_FIELD('%d' % seq['start_id']), + style.SQL_KEYWORD('WITH'), + style.SQL_KEYWORD('NO_INFOMSGS'), + ) for seq in seqs]) + + sql_list.extend(['ALTER TABLE %s CHECK CONSTRAINT %s;' % + (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks]) + return sql_list def start_transaction_sql(self): """ From 0f19fc984088a3462def971986a79649a80156ef Mon Sep 17 00:00:00 2001 From: Oskar Persson Date: Tue, 18 Aug 2020 20:46:39 +0200 Subject: [PATCH 5/5] Revert "Fix sql_flush" This reverts commit 05da7a2821f85885fd79a167b2fe246419d861a2. --- sql_server/pyodbc/operations.py | 117 +++++++++++++------------------- 1 file changed, 48 insertions(+), 69 deletions(-) diff --git a/sql_server/pyodbc/operations.py b/sql_server/pyodbc/operations.py index 12a0c8d6..6af85bab 100644 --- a/sql_server/pyodbc/operations.py +++ b/sql_server/pyodbc/operations.py @@ -314,7 +314,7 @@ def savepoint_rollback_sql(self, sid): """ return "ROLLBACK TRANSACTION %s" % sid - def sql_flush(self, style, tables, *args, allow_cascade=False, **kwargs): + def sql_flush(self, style, tables, sequences, allow_cascade=False): """ Returns a list of SQL statements required to remove all data from the given database tables (without actually removing the tables @@ -329,76 +329,55 @@ def sql_flush(self, style, tables, *args, allow_cascade=False, **kwargs): The `allow_cascade` argument determines whether truncation may cascade to tables with foreign keys pointing the tables being truncated. """ - - if not tables: - return [] - - if django.VERSION >= (3, 1): - truncated_tables = {table.upper() for table in tables} - constraints = set() - for table in tables: - for foreign_table, constraint in self._foreign_key_constraints(table, recursive=allow_cascade): - if allow_cascade: - truncated_tables.add(foreign_table) - constraints.add((foreign_table, constraint)) - - if kwargs['reset_sequences']: - sequences = [ - sequence - for sequence in self.connection.introspection.sequence_list() - if sequence['table'].upper() in truncated_tables - ] + if tables: + # Cannot use TRUNCATE on tables that are referenced by a FOREIGN KEY + # So must use the much slower DELETE + from django.db import connections + cursor = connections[self.connection.alias].cursor() + # Try to minimize the risks of the braindeaded inconsistency in + # DBCC CHEKIDENT(table, RESEED, n) behavior. + seqs = [] + for seq in sequences: + cursor.execute("SELECT COUNT(*) FROM %s" % self.quote_name(seq["table"])) + rowcnt = cursor.fetchone()[0] + elem = {} + if rowcnt: + elem['start_id'] = 0 + else: + elem['start_id'] = 1 + elem.update(seq) + seqs.append(elem) + COLUMNS = "TABLE_NAME, CONSTRAINT_NAME" + WHERE = "CONSTRAINT_TYPE not in ('PRIMARY KEY','UNIQUE')" + cursor.execute( + "SELECT {} FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE {}".format(COLUMNS, WHERE)) + fks = cursor.fetchall() + sql_list = ['ALTER TABLE %s NOCHECK CONSTRAINT %s;' % + (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks] + sql_list.extend(['%s %s %s;' % (style.SQL_KEYWORD('DELETE'), style.SQL_KEYWORD('FROM'), + style.SQL_FIELD(self.quote_name(table))) for table in tables]) + + if self.connection.to_azure_sql_db and self.connection.sql_server_version < 2014: + warnings.warn("Resetting identity columns is not supported " + "on this versios of Azure SQL Database.", + RuntimeWarning) else: - sequences = [] + # Then reset the counters on each table. + sql_list.extend(['%s %s (%s, %s, %s) %s %s;' % ( + style.SQL_KEYWORD('DBCC'), + style.SQL_KEYWORD('CHECKIDENT'), + style.SQL_FIELD(self.quote_name(seq["table"])), + style.SQL_KEYWORD('RESEED'), + style.SQL_FIELD('%d' % seq['start_id']), + style.SQL_KEYWORD('WITH'), + style.SQL_KEYWORD('NO_INFOMSGS'), + ) for seq in seqs]) + + sql_list.extend(['ALTER TABLE %s CHECK CONSTRAINT %s;' % + (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks]) + return sql_list else: - sequences = args[0] - - # Cannot use TRUNCATE on tables that are referenced by a FOREIGN KEY - # So must use the much slower DELETE - from django.db import connections - cursor = connections[self.connection.alias].cursor() - # Try to minimize the risks of the braindeaded inconsistency in - # DBCC CHEKIDENT(table, RESEED, n) behavior. - seqs = [] - for seq in sequences: - cursor.execute("SELECT COUNT(*) FROM %s" % self.quote_name(seq["table"])) - rowcnt = cursor.fetchone()[0] - elem = {} - if rowcnt: - elem['start_id'] = 0 - else: - elem['start_id'] = 1 - elem.update(seq) - seqs.append(elem) - COLUMNS = "TABLE_NAME, CONSTRAINT_NAME" - WHERE = "CONSTRAINT_TYPE not in ('PRIMARY KEY','UNIQUE')" - cursor.execute( - "SELECT {} FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE {}".format(COLUMNS, WHERE)) - fks = cursor.fetchall() - sql_list = ['ALTER TABLE %s NOCHECK CONSTRAINT %s;' % - (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks] - sql_list.extend(['%s %s %s;' % (style.SQL_KEYWORD('DELETE'), style.SQL_KEYWORD('FROM'), - style.SQL_FIELD(self.quote_name(table))) for table in tables]) - - if self.connection.to_azure_sql_db and self.connection.sql_server_version < 2014: - warnings.warn("Resetting identity columns is not supported " - "on this versios of Azure SQL Database.", - RuntimeWarning) - else: - # Then reset the counters on each table. - sql_list.extend(['%s %s (%s, %s, %s) %s %s;' % ( - style.SQL_KEYWORD('DBCC'), - style.SQL_KEYWORD('CHECKIDENT'), - style.SQL_FIELD(self.quote_name(seq["table"])), - style.SQL_KEYWORD('RESEED'), - style.SQL_FIELD('%d' % seq['start_id']), - style.SQL_KEYWORD('WITH'), - style.SQL_KEYWORD('NO_INFOMSGS'), - ) for seq in seqs]) - - sql_list.extend(['ALTER TABLE %s CHECK CONSTRAINT %s;' % - (self.quote_name(fk[0]), self.quote_name(fk[1])) for fk in fks]) - return sql_list + return [] def start_transaction_sql(self): """