CLN: move top-level dirs

xref #13634

pandas.types -> pandas.core.dtypes
pandas.computation -> pandas.core.computation
pandas.sparse -> pandas.core.sparse

Author: Jeff Reback <jeff.reback@twosigma.com>

Closes #15998 from jreback/move_types and squashes the following commits:

5fe80ae [Jeff Reback] move fixtures to top-level conftest
9d36104 [Jeff Reback] moar ci
d165a3f [Jeff Reback] more verbose built test
3ca5ba3 [Jeff Reback] pep
f1505d7 [Jeff Reback] try with pyargs
f63b76c [Jeff Reback] CLN: pandas.sparse -> pandas.core.sparse
751fb33 [Jeff Reback] move privacy changes to development section
221a7b5 [Jeff Reback] rename .typed -> .dtypes
290315e [Jeff Reback] move pandas.api.lib.infer_dtype -> pandas.api.types
ba32641 [Jeff Reback] CLN: move pandas.computation -> pandas.core.computation
bbdb1ee [Jeff Reback] CLN: move pandas.types -> pandas.core.typed

Author: jreback

asv_bench/benchmarks/binary_ops.py

@@ -1,5 +1,5 @@
 from .pandas_vb_common import *
-import pandas.computation.expressions as expr
+import pandas.core.computation.expressions as expr
 
 
 class Ops(object):

asv_bench/benchmarks/categoricals.py

@@ -1,6 +1,6 @@
 from .pandas_vb_common import *
 try:
-    from pandas.types.concat import union_categoricals
+    from pandas.core.dtypes.concat import union_categoricals
 except ImportError:
     pass
 

asv_bench/benchmarks/eval.py

@@ -1,6 +1,6 @@
 from .pandas_vb_common import *
 import pandas as pd
-import pandas.computation.expressions as expr
+import pandas.core.computation.expressions as expr
 
 
 class Eval(object):

asv_bench/benchmarks/indexing.py

@@ -1,6 +1,6 @@
 from .pandas_vb_common import *
 try:
-    import pandas.computation.expressions as expr
+    import pandas.core.computation.expressions as expr
 except:
     expr = None
 

asv_bench/benchmarks/sparse.py

@@ -1,5 +1,5 @@
 from .pandas_vb_common import *
-import pandas.sparse.series
+import pandas.core.sparse.series
 import scipy.sparse
 from pandas.core.sparse import SparseSeries, SparseDataFrame
 from pandas.core.sparse import SparseDataFrame
@@ -37,7 +37,7 @@ def setup(self):
         self.A = scipy.sparse.coo_matrix(([3.0, 1.0, 2.0], ([1, 0, 0], [0, 2, 3])), shape=(100, 100))
 
     def time_sparse_series_from_coo(self):
-        self.ss = pandas.sparse.series.SparseSeries.from_coo(self.A)
+        self.ss = pandas.core.sparse.series.SparseSeries.from_coo(self.A)
 
 
 class sparse_series_to_coo(object):

ci/install_travis.sh

@@ -123,6 +123,7 @@ if [ "$BUILD_TEST" ]; then
 
     # build & install testing
     echo ["Starting installation test."]
+    rm -rf dist
     python setup.py clean
     python setup.py build_ext --inplace
     python setup.py sdist --formats=gztar

ci/script_multi.sh

@@ -19,7 +19,11 @@ export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 429496
 echo PYTHONHASHSEED=$PYTHONHASHSEED
 
 if [ "$BUILD_TEST" ]; then
+    echo "build-test"
     cd /tmp
+    pwd
+    conda list pandas
+    echo "running"
     python -c "import pandas; pandas.test(['-n 2'])"
 elif [ "$DOC" ]; then
     echo "We are not running pytest as this is a doc-build"

doc/source/categorical.rst

@@ -673,7 +673,7 @@ will be the union of the categories being combined.
 
 .. ipython:: python
 
-    from pandas.types.concat import union_categoricals
+    from pandas.api.types import union_categoricals
     a = pd.Categorical(["b", "c"])
     b = pd.Categorical(["a", "b"])
     union_categoricals([a, b])

doc/source/whatsnew/v0.20.0.txt

@@ -21,8 +21,11 @@ Highlights include:
 - Support for S3 handling now uses ``s3fs``, see :ref:`here <whatsnew_0200.api_breaking.s3>`
 - Google BigQuery support now uses the ``pandas-gbq`` library, see :ref:`here <whatsnew_0200.api_breaking.gbq>`
 - Switched the test framework to use `pytest <http://doc.pytest.org/en/latest>`__ (:issue:`13097`)
-- The ``pandas.tools.plotting`` module has been deprecated, moved to ``pandas.plotting``. See :ref:`here <whatsnew_0200.api_breaking.plotting>`
 
+.. warning::
+
+  Pandas has changed the internal structure and layout of the codebase.
+  This can affect imports that are not from the top-level ``pandas.*`` namespace, please see the changes :ref:`here <whatsnew_0200.privacy>`.
 
 Check the :ref:`API Changes <whatsnew_0200.api_breaking>` and :ref:`deprecations <whatsnew_0200.deprecations>` before updating.
 
@@ -489,7 +492,7 @@ Other Enhancements
 
 
 - ``TimedeltaIndex`` now has a custom datetick formatter specifically designed for nanosecond level precision (:issue:`8711`)
-- ``pd.types.concat.union_categoricals`` gained the ``ignore_ordered`` argument to allow ignoring the ordered attribute of unioned categoricals (:issue:`13410`). See the :ref:`categorical union docs <categorical.union>` for more information.
+- ``pd.api.types.union_categoricals`` gained the ``ignore_ordered`` argument to allow ignoring the ordered attribute of unioned categoricals (:issue:`13410`). See the :ref:`categorical union docs <categorical.union>` for more information.
 - ``DataFrame.to_latex()`` and ``DataFrame.to_string()`` now allow optional header aliases. (:issue:`15536`)
 - Re-enable the ``parse_dates`` keyword of ``pd.read_excel()`` to parse string columns as dates (:issue:`14326`)
 - Added ``.empty`` property to subclasses of ``Index``. (:issue:`15270`)
@@ -558,31 +561,6 @@ Using ``.iloc``. Here we will get the location of the 'A' column, then use *posi
   df.iloc[[0, 2], df.columns.get_loc('A')]
 
 
-.. _whatsnew_0200.api_breaking.deprecate_plotting
-
-Deprecate .plotting
-^^^^^^^^^^^^^^^^^^^
-
-The ``pandas.tools.plotting`` module has been deprecated,  in favor of the top level ``pandas.plotting`` module. All the public plotting functions are now available
-from ``pandas.plotting`` (:issue:`12548`).
-
-Furthermore, the top-level ``pandas.scatter_matrix`` and ``pandas.plot_params`` are deprecated.
-Users can import these from ``pandas.plotting`` as well.
-
-Previous script:
-
-.. code-block:: python
-
-   pd.tools.plotting.scatter_matrix(df)
-   pd.scatter_matrix(df)
-
-Should be changed to:
-
-.. code-block:: python
-
-    pd.plotting.scatter_matrix(df)
-
-
 .. _whatsnew_0200.api_breaking.deprecate_panel:
 
 Deprecate Panel
@@ -1026,34 +1004,6 @@ New Behavior:
    In [11]: index.memory_usage(deep=True)
    Out[11]: 260
 
-.. _whatsnew_0200.api_breaking.extensions:
-
-Extension Modules Moved
-^^^^^^^^^^^^^^^^^^^^^^^
-
-Some formerly public c/c++/cython extension modules have been moved and/or renamed. These are all removed from the public API.
-If indicated, a deprecation warning will be issued if you reference that module. (:issue:`12588`)
-
-.. csv-table::
-    :header: "Previous Location", "New Location", "Deprecated"
-    :widths: 30, 30, 4
-
-    "pandas.lib", "pandas._libs.lib", "X"
-    "pandas.tslib", "pandas._libs.tslib", "X"
-    "pandas._join", "pandas._libs.join", ""
-    "pandas._period", "pandas._libs.period", ""
-    "pandas.msgpack", "pandas.io.msgpack", ""
-    "pandas.index", "pandas._libs.index", ""
-    "pandas.algos", "pandas._libs.algos", ""
-    "pandas.hashtable", "pandas._libs.hashtable", ""
-    "pandas.json", "pandas.io.json.libjson", "X"
-    "pandas.parser", "pandas.io.libparsers", "X"
-    "pandas.io.sas.saslib", "pandas.io.sas.libsas", ""
-    "pandas._testing", "pandas.util.libtesting", ""
-    "pandas._sparse", "pandas.sparse.libsparse", ""
-    "pandas._hash", "pandas.tools.libhash", ""
-    "pandas._window", "pandas.core.libwindow", ""
-
 .. _whatsnew_0200.api_breaking.sort_index:
 
 DataFrame.sort_index changes
@@ -1354,10 +1304,74 @@ Other API Changes
 - ``DataFrame`` and ``Panel`` constructors with invalid input will now raise ``ValueError`` rather than ``pandas.core.common.PandasError``, if called with scalar inputs and not axes; The exception ``PandasError`` is removed as well. (:issue:`15541`)
 - The exception ``pandas.core.common.AmbiguousIndexError`` is removed as it is not referenced (:issue:`15541`)
 
-.. _whatsnew_0200.develop:
+.. _whatsnew_0200.privacy:
+
+Privacy Changes
+~~~~~~~~~~~~~~~
+
+.. _whatsnew_0200.privacy.extensions:
+
+Modules Privacy Has Changed
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Some formerly public python/c/c++/cython extension modules have been moved and/or renamed. These are all removed from the public API.
+Furthermore, the ``pandas.core``, ``pandas.io``, and ``pandas.util`` top-level modules are now considered to be PRIVATE.
+If indicated, a deprecation warning will be issued if you reference theses modules. (:issue:`12588`)
+
+.. csv-table::
+    :header: "Previous Location", "New Location", "Deprecated"
+    :widths: 30, 30, 4
+
+    "pandas.lib", "pandas._libs.lib", "X"
+    "pandas.tslib", "pandas._libs.tslib", "X"
+    "pandas.computation", "pandas.core.computation", ""
+    "pandas.msgpack", "pandas.io.msgpack", ""
+    "pandas.index", "pandas._libs.index", ""
+    "pandas.algos", "pandas._libs.algos", ""
+    "pandas.hashtable", "pandas._libs.hashtable", ""
+    "pandas.json", "pandas.io.json.libjson", "X"
+    "pandas.parser", "pandas.io.libparsers", "X"
+    "pandas.sparse", "pandas.core.sparse", ""
+    "pandas.types", "pandas.core.dtypes", ""
+    "pandas.io.sas.saslib", "pandas.io.sas.libsas", ""
+    "pandas._join", "pandas._libs.join", ""
+    "pandas._hash", "pandas.tools.libhash", ""
+    "pandas._period", "pandas._libs.period", ""
+    "pandas._sparse", "pandas.core.sparse.libsparse", ""
+    "pandas._testing", "pandas.util.libtesting", ""
+    "pandas._window", "pandas.core.libwindow", ""
+
+- The function :func:`~pandas.api.type.union_categoricals` is now importable from ``pandas.api.types``, formerly from ``pandas.types.concat`` (:issue:`15998`)
+
+.. _whatsnew_0200.privacy.deprecate_plotting
+
+Deprecate .plotting
+^^^^^^^^^^^^^^^^^^^
+
+The ``pandas.tools.plotting`` module has been deprecated,  in favor of the top level ``pandas.plotting`` module. All the public plotting functions are now available
+from ``pandas.plotting`` (:issue:`12548`).
+
+Furthermore, the top-level ``pandas.scatter_matrix`` and ``pandas.plot_params`` are deprecated.
+Users can import these from ``pandas.plotting`` as well.
+
+Previous script:
+
+.. code-block:: python
+
+   pd.tools.plotting.scatter_matrix(df)
+   pd.scatter_matrix(df)
+
+Should be changed to:
+
+.. code-block:: python
+
+    pd.plotting.scatter_matrix(df)
+
+
+.. _whatsnew_0200.privacy.development:
 
-Development Changes
-~~~~~~~~~~~~~~~~~~~
+Other Developement Changes
+^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 - Building pandas for development now requires ``cython >= 0.23`` (:issue:`14831`)
 - Require at least 0.23 version of cython to avoid problems with character encodings (:issue:`14699`)

pandas/__init__.py

@@ -40,10 +40,10 @@
 import pandas.core.config_init
 
 from pandas.core.api import *
-from pandas.sparse.api import *
+from pandas.core.sparse.api import *
 from pandas.stats.api import *
 from pandas.tseries.api import *
-from pandas.computation.api import *
+from pandas.core.computation.api import *
 
 from pandas.tools.concat import concat
 from pandas.tools.merge import (merge, ordered_merge,

pandas/api/types/__init__.py

@@ -1,4 +1,5 @@
 """ public toolkit API """
 
-from pandas.types.api import *  # noqa
+from pandas.core.dtypes.api import *  # noqa
+from pandas.core.dtypes.concat import union_categoricals  # noqa
 del np  # noqa

pandas/compat/numpy/function.py

@@ -22,7 +22,7 @@
 from pandas.util.validators import (validate_args, validate_kwargs,
                                     validate_args_and_kwargs)
 from pandas.errors import UnsupportedFunctionCall
-from pandas.types.common import is_integer, is_bool
+from pandas.core.dtypes.common import is_integer, is_bool
 from pandas.compat import OrderedDict
 
 

pandas/compat/pickle_compat.py

@@ -67,16 +67,24 @@ def load_reduce(self):
     ('pandas.core.series', 'TimeSeries'):
         ('pandas.core.series', 'Series'),
     ('pandas.sparse.series', 'SparseTimeSeries'):
-        ('pandas.sparse.series', 'SparseSeries'),
+        ('pandas.core.sparse.series', 'SparseSeries'),
 
     # 12588, extensions moving
     ('pandas._sparse', 'BlockIndex'):
-        ('pandas.sparse.libsparse', 'BlockIndex'),
+        ('pandas.core.sparse.libsparse', 'BlockIndex'),
     ('pandas.tslib', 'Timestamp'):
         ('pandas._libs.tslib', 'Timestamp'),
     ('pandas.tslib', '__nat_unpickle'):
         ('pandas._libs.tslib', '__nat_unpickle'),
-    ('pandas._period', 'Period'): ('pandas._libs.period', 'Period')
+    ('pandas._period', 'Period'): ('pandas._libs.period', 'Period'),
+
+    # 15998 top-level dirs moving
+    ('pandas.sparse.array', 'SparseArray'):
+        ('pandas.core.sparse.array', 'SparseArray'),
+    ('pandas.sparse.series', 'SparseSeries'):
+        ('pandas.core.sparse.series', 'SparseSeries'),
+    ('pandas.sparse.frame', 'SparseDataFrame'):
+        ('pandas.core.sparse.frame', 'SparseDataFrame')
 }
 
 

pandas/conftest.py

@@ -2,6 +2,7 @@
 
 import numpy
 import pandas
+import pandas.util.testing as tm
 
 
 def pytest_addoption(parser):
@@ -30,3 +31,10 @@ def pytest_runtest_setup(item):
 def add_imports(doctest_namespace):
     doctest_namespace['np'] = numpy
     doctest_namespace['pd'] = pandas
+
+
+@pytest.fixture(params=['bsr', 'coo', 'csc', 'csr', 'dia', 'dok', 'lil'])
+def spmatrix(request):
+    tm._skip_if_no_scipy()
+    from scipy import sparse
+    return getattr(sparse, request.param + '_matrix')

pandas/core/algorithms.py

@@ -7,10 +7,11 @@
 import numpy as np
 
 from pandas import compat, _np_version_under1p8
-from pandas.types.cast import maybe_promote
-from pandas.types.generic import (ABCSeries, ABCIndex,
-                                  ABCIndexClass, ABCCategorical)
-from pandas.types.common import (
+from pandas.core.dtypes.cast import maybe_promote
+from pandas.core.dtypes.generic import (
+    ABCSeries, ABCIndex,
+    ABCIndexClass, ABCCategorical)
+from pandas.core.dtypes.common import (
     is_unsigned_integer_dtype, is_signed_integer_dtype,
     is_integer_dtype, is_complex_dtype,
     is_categorical_dtype, is_sparse,
@@ -25,9 +26,9 @@
     _ensure_float64, _ensure_uint64,
     _ensure_int64)
 from pandas.compat.numpy import _np_version_under1p10
-from pandas.types.missing import isnull
+from pandas.core.dtypes.missing import isnull
 
-import pandas.core.common as com
+from pandas.core import common as com
 from pandas.compat import string_types
 from pandas._libs import algos, lib, hashtable as htable
 from pandas._libs.tslib import iNaT

pandas/core/api.py

@@ -5,7 +5,7 @@
 import numpy as np
 
 from pandas.core.algorithms import factorize, unique, value_counts
-from pandas.types.missing import isnull, notnull
+from pandas.core.dtypes.missing import isnull, notnull
 from pandas.core.categorical import Categorical
 from pandas.core.groupby import Grouper
 from pandas.formats.format import set_eng_float_format

pandas/core/base.py

@@ -6,9 +6,9 @@
 from pandas.compat import builtins
 import numpy as np
 
-from pandas.types.missing import isnull
-from pandas.types.generic import ABCDataFrame, ABCSeries, ABCIndexClass
-from pandas.types.common import is_object_dtype, is_list_like, is_scalar
+from pandas.core.dtypes.missing import isnull
+from pandas.core.dtypes.generic import ABCDataFrame, ABCSeries, ABCIndexClass
+from pandas.core.dtypes.common import is_object_dtype, is_list_like, is_scalar
 from pandas.util.validators import validate_bool_kwarg
 
 from pandas.core import common as com
@@ -725,7 +725,7 @@ def _aggregate_multiple_funcs(self, arg, _level, _axis):
             # we are concatting non-NDFrame objects,
             # e.g. a list of scalars
 
-            from pandas.types.cast import is_nested_object
+            from pandas.core.dtypes.cast import is_nested_object
             from pandas import Series
             result = Series(results, index=keys, name=self.name)
             if is_nested_object(result):