TST: use xdist for multiple cpu testing

Author: jreback

.travis.yml

@@ -320,7 +320,8 @@ before_script:
 script:
   - echo "script start"
   - ci/run_build_docs.sh
-  - ci/script.sh
+  - ci/script_single.sh
+  - ci/script_multi.sh
   - ci/lint.sh
   - echo "script done"
 

ci/script_multi.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+echo "[script multi]"
+
+source activate pandas
+
+# don't run the tests for the doc build
+if [ x"$DOC_BUILD" != x"" ]; then
+    exit 0
+fi
+
+if [ -n "$LOCALE_OVERRIDE" ]; then
+    export LC_ALL="$LOCALE_OVERRIDE";
+    echo "Setting LC_ALL to $LOCALE_OVERRIDE"
+
+    pycmd='import pandas; print("pandas detected console encoding: %s" % pandas.get_option("display.encoding"))'
+    python -c "$pycmd"
+fi
+
+if [ "$BUILD_TEST" ]; then
+    echo "We are not running pytest as this is simply a build test."
+elif [ "$COVERAGE" ]; then
+    echo pytest -s -n 2 -m "not single" --cov=pandas --cov-append --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
+    pytest -s -n 2 -m "not single" --cov=pandas --cov-append --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
+else
+    echo pytest -n 2 -m "not single" $TEST_ARGS pandas
+    pytest -n 2 -m "not single" $TEST_ARGS pandas # TODO: doctest
+fi
+
+RET="$?"
+
+exit "$RET"

ci/script.sh renamed to ci/script_single.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-echo "inside $0"
+echo "[script_single]"
 
 source activate pandas
 
@@ -20,11 +20,11 @@ fi
 if [ "$BUILD_TEST" ]; then
     echo "We are not running pytest as this is simply a build test."
 elif [ "$COVERAGE" ]; then
-    echo pytest -s --cov=pandas --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
-    pytest -s --cov=pandas --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
+    echo pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
+    pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/pytest.xml $TEST_ARGS pandas
 else
-    echo pytest $TEST_ARGS pandas
-    pytest $TEST_ARGS pandas # TODO: doctest
+    echo pytest -m "single" $TEST_ARGS pandas
+    pytest -m "single" $TEST_ARGS pandas # TODO: doctest
 fi
 
 RET="$?"

pandas/tests/io/test_clipboard.py

@@ -20,6 +20,7 @@
     _DEPS_INSTALLED = 0
 
 
+@pytest.mark.single
 @pytest.mark.skipif(not _DEPS_INSTALLED,
                     reason="clipboard primitives not installed")
 class TestClipboard(tm.TestCase):

pandas/tests/io/test_pytables.py

@@ -36,12 +36,6 @@
 from pandas import concat, Timestamp
 from pandas import compat
 from pandas.compat import range, lrange, u
-
-try:
-    import tables
-except ImportError:
-    pytest.skip('no pytables')
-
 from distutils.version import LooseVersion
 
 _default_compressor = ('blosc' if LooseVersion(tables.__version__) >= '2.2'
@@ -165,6 +159,7 @@ def tearDown(self):
         pass
 
 
+@pytest.mark.single
 class TestHDFStore(Base, tm.TestCase):
 
     def test_factory_fun(self):
@@ -5086,6 +5081,7 @@ def test_query_long_float_literal(self):
             tm.assert_frame_equal(expected, result)
 
 
+@pytest.mark.single
 class TestHDFComplexValues(Base):
     # GH10447
 
@@ -5231,6 +5227,7 @@ def test_complex_append(self):
             assert_frame_equal(pd.concat([df, df], 0), result)
 
 
+@pytest.mark.single
 class TestTimezones(Base, tm.TestCase):
 
     def _compare_with_tz(self, a, b):

pandas/tests/io/test_sql.py

@@ -18,6 +18,7 @@
 """
 
 from __future__ import print_function
+import pytest
 import unittest
 import sqlite3
 import csv
@@ -839,6 +840,7 @@ def test_unicode_column_name(self):
         df.to_sql('test_unicode', self.conn, index=False)
 
 
+@pytest.mark.single
 class TestSQLApi(SQLAlchemyMixIn, _TestSQLApi, unittest.TestCase):
     """
     Test the public API as it would be used directly
@@ -1024,10 +1026,12 @@ def tearDown(self):
         super(_EngineToConnMixin, self).tearDown()
 
 
+@pytest.mark.single
 class TestSQLApiConn(_EngineToConnMixin, TestSQLApi, unittest.TestCase):
     pass
 
 
+@pytest.mark.single
 class TestSQLiteFallbackApi(SQLiteMixIn, _TestSQLApi, unittest.TestCase):
     """
     Test the public sqlite connection fallback API
@@ -1875,30 +1879,36 @@ def test_schema_support(self):
             tm.assert_frame_equal(res1, res2)
 
 
+@pytest.mark.single
 class TestMySQLAlchemy(_TestMySQLAlchemy, _TestSQLAlchemy, unittest.TestCase):
     pass
 
 
+@pytest.mark.single
 class TestMySQLAlchemyConn(_TestMySQLAlchemy, _TestSQLAlchemyConn,
                            unittest.TestCase):
     pass
 
 
+@pytest.mark.single
 class TestPostgreSQLAlchemy(_TestPostgreSQLAlchemy, _TestSQLAlchemy,
                             unittest.TestCase):
     pass
 
 
+@pytest.mark.single
 class TestPostgreSQLAlchemyConn(_TestPostgreSQLAlchemy, _TestSQLAlchemyConn,
                                 unittest.TestCase):
     pass
 
 
+@pytest.mark.single
 class TestSQLiteAlchemy(_TestSQLiteAlchemy, _TestSQLAlchemy,
                         unittest.TestCase):
     pass
 
 
+@pytest.mark.single
 class TestSQLiteAlchemyConn(_TestSQLiteAlchemy, _TestSQLAlchemyConn,
                             unittest.TestCase):
     pass
@@ -1907,6 +1917,7 @@ class TestSQLiteAlchemyConn(_TestSQLiteAlchemy, _TestSQLAlchemyConn,
 # -----------------------------------------------------------------------------
 # -- Test Sqlite / MySQL fallback
 
+@pytest.mark.single
 class TestSQLiteFallback(SQLiteMixIn, PandasSQLTest, unittest.TestCase):
     """
     Test the fallback mode against an in-memory sqlite database.
@@ -2133,6 +2144,7 @@ def _skip_if_no_pymysql():
         pytest.skip('pymysql not installed, skipping')
 
 
+@pytest.mark.single
 class TestXSQLite(SQLiteMixIn, tm.TestCase):
 
     def setUp(self):
@@ -2343,6 +2355,7 @@ def clean_up(test_table_to_drop):
         clean_up(table_name)
 
 
+@pytest.mark.single
 class TestSQLFlavorDeprecation(tm.TestCase):
     """
     gh-13611: test that the 'flavor' parameter
@@ -2367,8 +2380,9 @@ def test_deprecated_flavor(self):
                 getattr(sql, func)(self.con, flavor='sqlite')
 
 
-@unittest.skip("gh-13611: there is no support for MySQL "
-               "if SQLAlchemy is not installed")
+@pytest.mark.single
+@pytest.mark.skip(reason="gh-13611: there is no support for MySQL "
+                  "if SQLAlchemy is not installed")
 class TestXMySQL(MySQLMixIn, tm.TestCase):
 
     @classmethod

setup.cfg

@@ -25,3 +25,5 @@ split_penalty_logical_operator = 30
 # Silencing the warning until then
 addopts = --disable-pytest-warnings
 testpaths = pandas
+markers =
+    single: mark a test as single cpu only

test_fast.sh

@@ -1,2 +1 @@
-# nosetests -A "not slow and not network" pandas --with-id $*
-pytest pandas --skip-slow
+pytest pandas --skip-slow --skip-network -m "not single" -n 4