From 4cf9988cc6f3dc08f5dd78edce75fa3e5e8819f1 Mon Sep 17 00:00:00 2001 From: Dmitriy Trochshenko Date: Fri, 20 May 2016 14:03:20 +0600 Subject: [PATCH 1/9] added option to use one database per suite when xdist plugin is used --- pytest_django/fixtures.py | 10 ++++++++++ pytest_django/plugin.py | 35 +++++++++++++++++++++++++++++++++-- tests/test_db_setup.py | 26 ++++++++++++++++++++++++++ 3 files changed, 69 insertions(+), 2 deletions(-) diff --git a/pytest_django/fixtures.py b/pytest_django/fixtures.py index cae7d4763..fb6fe1333 100644 --- a/pytest_django/fixtures.py +++ b/pytest_django/fixtures.py @@ -28,6 +28,9 @@ def _django_db_setup(request, """Session-wide database setup, internal to pytest-django""" skip_if_no_django() + if is_xdist_one_db_enabled(request.config): + return + from .compat import setup_databases, teardown_databases # xdist @@ -64,6 +67,13 @@ def teardown_database(): request.addfinalizer(teardown_database) +def is_xdist_one_db_enabled(config): + from django.conf import settings + is_sqlite = (settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3') + # can't use one sqlite3 db for distributed database because of lock + return config.getvalue('xdist_one_db') and not is_sqlite + + def _django_db_fixture_helper(transactional, request, _django_cursor_wrapper): if is_django_unittest(request): return diff --git a/pytest_django/plugin.py b/pytest_django/plugin.py index d499e6f50..deabba7f4 100644 --- a/pytest_django/plugin.py +++ b/pytest_django/plugin.py @@ -13,13 +13,15 @@ import py import pytest +from django.test.runner import setup_databases +from .db_reuse import monkey_patch_creation_for_db_reuse from .django_compat import is_django_unittest from .fixtures import (_django_db_setup, _live_server_helper, admin_client, admin_user, client, db, django_user_model, django_username_field, live_server, rf, settings, - transactional_db) -from .lazy_django import django_settings_is_configured, skip_if_no_django + transactional_db, _handle_south, _disable_native_migrations) +from .lazy_django import django_settings_is_configured, skip_if_no_django, get_django_version # Silence linters for imported fixtures. (_django_db_setup, _live_server_helper, admin_client, admin_user, client, db, @@ -44,6 +46,10 @@ def pytest_addoption(parser): action='store_true', dest='create_db', default=False, help='Re-create the database, even if it exists. This ' 'option will be ignored if not --reuse-db is given.') + group._addoption('--xdist-one-db', + dest='xdist_one_db', default=False, action='store_true', + help="Use only one database with xdist plugin. " + "Doesn't work with sqlite3 backend due to db lock") group._addoption('--ds', action='store', type='string', dest='ds', default=None, help='Set DJANGO_SETTINGS_MODULE.') @@ -246,6 +252,31 @@ def pytest_report_header(config): return [config._dsm_report_header] +def pytest_xdist_setupnodes(config): + """called once before any remote node is set up. """ + if not config.getvalue('xdist_one_db'): + return + _setup_django() + + from django.conf import settings + if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': + return + + _handle_south() + + if config.getvalue('nomigrations'): + _disable_native_migrations() + + db_args = {} + if get_django_version() >= (1, 8): + db_args['keepdb'] = True + else: + monkey_patch_creation_for_db_reuse() + + # Create the database + setup_databases(verbosity=config.option.verbose, interactive=False, **db_args) + + @pytest.mark.trylast def pytest_configure(): # Allow Django settings to be configured in a user pytest_configure call, diff --git a/tests/test_db_setup.py b/tests/test_db_setup.py index 05022db27..fbd212c2e 100644 --- a/tests/test_db_setup.py +++ b/tests/test_db_setup.py @@ -190,6 +190,32 @@ def test_d(settings): result.stdout.fnmatch_lines(['*PASSED*test_d*']) +def test_xdist_with_one_db_does_not_work_with_sqlite(django_testdir): + django_testdir.create_test_module(''' + import pytest + + from .app.models import Item + + def _check(settings): + # Make sure that the database name looks correct + db_name = settings.DATABASES['default']['NAME'] + assert db_name.endswith('_gw0') + + assert Item.objects.count() == 0 + Item.objects.create(name='foo') + assert Item.objects.count() == 1 + + + @pytest.mark.django_db + def test_a(settings): + _check(settings) + ''') + + result = django_testdir.runpytest_subprocess('-vv', '-n1', '-s', '--xdist-one-db') + assert result.ret == 0 + result.stdout.fnmatch_lines(['*PASSED*test_a*']) + + class TestSqliteWithXdist: pytestmark = skip_on_python32 From 515bd71ab564f4b2a5401877b188a9c6a516ad6f Mon Sep 17 00:00:00 2001 From: Dmitriy Trochshenko Date: Fri, 20 May 2016 14:22:21 +0600 Subject: [PATCH 2/9] fix typo --- pytest_django/fixtures.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_django/fixtures.py b/pytest_django/fixtures.py index fb6fe1333..f44a809cf 100644 --- a/pytest_django/fixtures.py +++ b/pytest_django/fixtures.py @@ -70,7 +70,7 @@ def teardown_database(): def is_xdist_one_db_enabled(config): from django.conf import settings is_sqlite = (settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3') - # can't use one sqlite3 db for distributed database because of lock + # can't use one sqlite3 db for distributed test run because of lock return config.getvalue('xdist_one_db') and not is_sqlite From 8010b32693aea69f7c5f6357aff57db20c307916 Mon Sep 17 00:00:00 2001 From: Dmitriy Trochshenko Date: Fri, 20 May 2016 14:34:17 +0600 Subject: [PATCH 3/9] make function private --- pytest_django/fixtures.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pytest_django/fixtures.py b/pytest_django/fixtures.py index f44a809cf..534f444eb 100644 --- a/pytest_django/fixtures.py +++ b/pytest_django/fixtures.py @@ -28,7 +28,7 @@ def _django_db_setup(request, """Session-wide database setup, internal to pytest-django""" skip_if_no_django() - if is_xdist_one_db_enabled(request.config): + if _is_xdist_one_db_enabled(request.config): return from .compat import setup_databases, teardown_databases @@ -67,7 +67,7 @@ def teardown_database(): request.addfinalizer(teardown_database) -def is_xdist_one_db_enabled(config): +def _is_xdist_one_db_enabled(config): from django.conf import settings is_sqlite = (settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3') # can't use one sqlite3 db for distributed test run because of lock From 9119320646e417ec93fc8e3b90b631855da0d0e5 Mon Sep 17 00:00:00 2001 From: Dmitriy Trochshenko Date: Mon, 23 May 2016 11:08:35 +0600 Subject: [PATCH 4/9] tests fix --- pytest_django/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_django/plugin.py b/pytest_django/plugin.py index deabba7f4..724de69dd 100644 --- a/pytest_django/plugin.py +++ b/pytest_django/plugin.py @@ -13,7 +13,6 @@ import py import pytest -from django.test.runner import setup_databases from .db_reuse import monkey_patch_creation_for_db_reuse from .django_compat import is_django_unittest @@ -273,6 +272,7 @@ def pytest_xdist_setupnodes(config): else: monkey_patch_creation_for_db_reuse() + from django.test.runner import setup_databases # Create the database setup_databases(verbosity=config.option.verbose, interactive=False, **db_args) From 207254b6eb67eea1c625bdce47e62f574ad5d2fb Mon Sep 17 00:00:00 2001 From: Dmitriy Trochshenko Date: Mon, 23 May 2016 11:17:32 +0600 Subject: [PATCH 5/9] raise ValueError in case of usage sqlite together with --xdist-one-db --- pytest_django/fixtures.py | 7 +++++-- tests/test_db_setup.py | 19 ++++--------------- 2 files changed, 9 insertions(+), 17 deletions(-) diff --git a/pytest_django/fixtures.py b/pytest_django/fixtures.py index 534f444eb..50a81e7d1 100644 --- a/pytest_django/fixtures.py +++ b/pytest_django/fixtures.py @@ -70,9 +70,12 @@ def teardown_database(): def _is_xdist_one_db_enabled(config): from django.conf import settings is_sqlite = (settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3') - # can't use one sqlite3 db for distributed test run because of lock - return config.getvalue('xdist_one_db') and not is_sqlite + one_db = config.getvalue('xdist_one_db') + if one_db and is_sqlite: + raise ValueError("xdist-one-db option can not be used together with sqlite3 backend") + + return one_db def _django_db_fixture_helper(transactional, request, _django_cursor_wrapper): if is_django_unittest(request): diff --git a/tests/test_db_setup.py b/tests/test_db_setup.py index fbd212c2e..fe42a0a5e 100644 --- a/tests/test_db_setup.py +++ b/tests/test_db_setup.py @@ -194,26 +194,15 @@ def test_xdist_with_one_db_does_not_work_with_sqlite(django_testdir): django_testdir.create_test_module(''' import pytest - from .app.models import Item - - def _check(settings): - # Make sure that the database name looks correct - db_name = settings.DATABASES['default']['NAME'] - assert db_name.endswith('_gw0') - - assert Item.objects.count() == 0 - Item.objects.create(name='foo') - assert Item.objects.count() == 1 - - @pytest.mark.django_db def test_a(settings): - _check(settings) + pass ''') result = django_testdir.runpytest_subprocess('-vv', '-n1', '-s', '--xdist-one-db') - assert result.ret == 0 - result.stdout.fnmatch_lines(['*PASSED*test_a*']) + assert result.ret == 1 + result.stdout.fnmatch_lines(['*= 1 error*']) + result.stdout.fnmatch_lines(['*ValueError*']) class TestSqliteWithXdist: From c7e1f259364786a74942e3c70a8e965bc4b78508 Mon Sep 17 00:00:00 2001 From: Dmitriy Trochshenko Date: Mon, 23 May 2016 12:10:58 +0600 Subject: [PATCH 6/9] tests to be backend-specific --- pytest_django/fixtures.py | 1 + pytest_django_test/db_helpers.py | 7 +++++ tests/test_db_setup.py | 49 +++++++++++++++++++++++++++----- 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/pytest_django/fixtures.py b/pytest_django/fixtures.py index 50a81e7d1..4fe274d0d 100644 --- a/pytest_django/fixtures.py +++ b/pytest_django/fixtures.py @@ -77,6 +77,7 @@ def _is_xdist_one_db_enabled(config): return one_db + def _django_db_fixture_helper(transactional, request, _django_cursor_wrapper): if is_django_unittest(request): return diff --git a/pytest_django_test/db_helpers.py b/pytest_django_test/db_helpers.py index f7460fce5..29afec6c3 100644 --- a/pytest_django_test/db_helpers.py +++ b/pytest_django_test/db_helpers.py @@ -48,6 +48,13 @@ def run_mysql(*args): return run_cmd(*args) +def skip_if_sqlite(): + from django.conf import settings + + if settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3': + pytest.skip('Skip if sqlite3 backend') + + def skip_if_sqlite_in_memory(): from django.conf import settings diff --git a/tests/test_db_setup.py b/tests/test_db_setup.py index fe42a0a5e..d5c149e88 100644 --- a/tests/test_db_setup.py +++ b/tests/test_db_setup.py @@ -5,7 +5,8 @@ from pytest_django.lazy_django import get_django_version from pytest_django_test.db_helpers import (db_exists, drop_database, mark_database, mark_exists, - skip_if_sqlite_in_memory) + skip_if_sqlite_in_memory, + skip_if_sqlite) skip_on_python32 = pytest.mark.skipif(sys.version_info[:2] == (3, 2), reason='xdist is flaky with Python 3.2') @@ -190,19 +191,39 @@ def test_d(settings): result.stdout.fnmatch_lines(['*PASSED*test_d*']) -def test_xdist_with_one_db_does_not_work_with_sqlite(django_testdir): +@skip_on_python32 +def test_xdist_one_db(django_testdir): + skip_if_sqlite() + + drop_database('gw0') + drop_database('gw1') + django_testdir.create_test_module(''' import pytest + from .app.models import Item + + def _check(settings): + # Make sure that the database name looks correct + db_name = settings.DATABASES['default']['NAME'] + assert 'gw' not in db_name + @pytest.mark.django_db def test_a(settings): - pass + _check(settings) + + @pytest.mark.django_db + def test_b(settings): + _check(settings) + ''') - result = django_testdir.runpytest_subprocess('-vv', '-n1', '-s', '--xdist-one-db') - assert result.ret == 1 - result.stdout.fnmatch_lines(['*= 1 error*']) - result.stdout.fnmatch_lines(['*ValueError*']) + result = django_testdir.runpytest_subprocess('-vv', '-n2', '-s', '--xdist-one-db') + assert result.ret == 0 + result.stdout.fnmatch_lines(['*PASSED*test_a*']) + result.stdout.fnmatch_lines(['*PASSED*test_b*']) + + assert db_exists() class TestSqliteWithXdist: @@ -232,6 +253,20 @@ def test_a(): assert result.ret == 0 result.stdout.fnmatch_lines(['*PASSED*test_a*']) + def test_xdist_with_one_db_does_not_work_with_sqlite(self, django_testdir): + django_testdir.create_test_module(''' + import pytest + + @pytest.mark.django_db + def test_a(settings): + pass + ''') + + result = django_testdir.runpytest_subprocess('-vv', '-n1', '-s', '--xdist-one-db') + assert result.ret == 1 + result.stdout.fnmatch_lines(['*= 1 error*']) + result.stdout.fnmatch_lines(['*ValueError*']) + @pytest.mark.skipif(get_django_version() >= (1, 9), reason=('Django 1.9 requires migration and has no concept ' From 83ad206f872b15dcf06b0dc377da8d0aff806597 Mon Sep 17 00:00:00 2001 From: Dmitriy Trochshenko Date: Fri, 17 Jun 2016 17:11:22 +0600 Subject: [PATCH 7/9] make sure test database is used (not production one) --- pytest_django/fixtures.py | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/pytest_django/fixtures.py b/pytest_django/fixtures.py index 4fe274d0d..38d050c7c 100644 --- a/pytest_django/fixtures.py +++ b/pytest_django/fixtures.py @@ -29,6 +29,7 @@ def _django_db_setup(request, skip_if_no_django() if _is_xdist_one_db_enabled(request.config): + _reuse_db() return from .compat import setup_databases, teardown_databases @@ -46,27 +47,32 @@ def _django_db_setup(request, if request.config.getvalue('nomigrations'): _disable_native_migrations() - db_args = {} with _django_cursor_wrapper: - if (request.config.getvalue('reuse_db') and - not request.config.getvalue('create_db')): - if get_django_version() >= (1, 8): - db_args['keepdb'] = True - else: - monkey_patch_creation_for_db_reuse() - - # Create the database - db_cfg = setup_databases(verbosity=pytest.config.option.verbose, - interactive=False, **db_args) + if request.config.getvalue('_reuse_db') and not request.config.getvalue('create_db'): + db_cfg = _reuse_db() + else: + db_cfg = setup_databases(verbosity=pytest.config.option.verbose, interactive=False,) def teardown_database(): with _django_cursor_wrapper: teardown_databases(db_cfg) - if not request.config.getvalue('reuse_db'): + if not request.config.getvalue('_reuse_db'): request.addfinalizer(teardown_database) +def _reuse_db(): + from .compat import setup_databases + db_args = {} + + if get_django_version() >= (1, 8): + db_args['keepdb'] = True + else: + monkey_patch_creation_for_db_reuse() + + return setup_databases(verbosity=pytest.config.option.verbose, interactive=False, **db_args) + + def _is_xdist_one_db_enabled(config): from django.conf import settings is_sqlite = (settings.DATABASES['default']['ENGINE'] == 'django.db.backends.sqlite3') From f56e012d06c6fcc070c71978c8581e6edf3068f1 Mon Sep 17 00:00:00 2001 From: Dmitriy Trochshenko Date: Fri, 17 Jun 2016 17:28:37 +0600 Subject: [PATCH 8/9] _django_cursor_wrapper is essential --- pytest_django/fixtures.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytest_django/fixtures.py b/pytest_django/fixtures.py index 38d050c7c..ea2c27d0c 100644 --- a/pytest_django/fixtures.py +++ b/pytest_django/fixtures.py @@ -29,7 +29,8 @@ def _django_db_setup(request, skip_if_no_django() if _is_xdist_one_db_enabled(request.config): - _reuse_db() + with _django_cursor_wrapper: + _reuse_db() return from .compat import setup_databases, teardown_databases From 7df098bcbd0cfda8760bf3193ec48d45ab81ce6e Mon Sep 17 00:00:00 2001 From: Dmitriy Trochshenko Date: Fri, 17 Jun 2016 17:41:45 +0600 Subject: [PATCH 9/9] rename reuse_db function to not be confused with option name --- pytest_django/fixtures.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pytest_django/fixtures.py b/pytest_django/fixtures.py index ea2c27d0c..ca2b4987a 100644 --- a/pytest_django/fixtures.py +++ b/pytest_django/fixtures.py @@ -30,7 +30,7 @@ def _django_db_setup(request, if _is_xdist_one_db_enabled(request.config): with _django_cursor_wrapper: - _reuse_db() + _setup_reused_databases() return from .compat import setup_databases, teardown_databases @@ -49,8 +49,8 @@ def _django_db_setup(request, _disable_native_migrations() with _django_cursor_wrapper: - if request.config.getvalue('_reuse_db') and not request.config.getvalue('create_db'): - db_cfg = _reuse_db() + if request.config.getvalue('reuse_db') and not request.config.getvalue('create_db'): + db_cfg = _setup_reused_databases() else: db_cfg = setup_databases(verbosity=pytest.config.option.verbose, interactive=False,) @@ -58,11 +58,11 @@ def teardown_database(): with _django_cursor_wrapper: teardown_databases(db_cfg) - if not request.config.getvalue('_reuse_db'): + if not request.config.getvalue('reuse_db'): request.addfinalizer(teardown_database) -def _reuse_db(): +def _setup_reused_databases(): from .compat import setup_databases db_args = {}