Merge branch 'master' into issue-37643

Author: OlehKSS

Dockerfile

@@ -43,5 +43,5 @@ RUN conda env update -n base -f "$pandas_home/environment.yml"
 
 # Build C extensions and pandas
 RUN cd "$pandas_home" \
-    && python setup.py build_ext --inplace -j 4 \
+    && python setup.py build_ext -j 4 \
     && python -m pip install -e .

Makefile

@@ -9,7 +9,7 @@ clean_pyc:
 	-find . -name '*.py[co]' -exec rm {} \;
 
 build: clean_pyc
-	python setup.py build_ext --inplace
+	python setup.py build_ext
 
 lint-diff:
 	git diff upstream/master --name-only -- "*.py" | xargs flake8

README.md

@@ -60,7 +60,7 @@ Here are just a few of the things that pandas does well:
     and saving/loading data from the ultrafast [**HDF5 format**][hdfstore]
   - [**Time series**][timeseries]-specific functionality: date range
     generation and frequency conversion, moving window statistics,
-    date shifting and lagging.
+    date shifting and lagging
 
 
    [missing-data]: https://pandas.pydata.org/pandas-docs/stable/missing_data.html#working-with-missing-data

asv_bench/benchmarks/groupby.py

@@ -486,7 +486,7 @@ def setup(self):
         tmp2 = (np.random.random(10000) * 10.0).astype(np.float32)
         tmp = np.concatenate((tmp1, tmp2))
         arr = np.repeat(tmp, 10)
-        self.df = DataFrame(dict(a=arr, b=arr))
+        self.df = DataFrame({"a": arr, "b": arr})
 
     def time_sum(self):
         self.df.groupby(["a"])["b"].sum()

asv_bench/benchmarks/indexing.py

@@ -358,6 +358,14 @@ def time_assign_with_setitem(self):
         for i in range(100):
             self.df[i] = np.random.randn(self.N)
 
+    def time_assign_list_like_with_setitem(self):
+        np.random.seed(1234)
+        self.df[list(range(100))] = np.random.randn(self.N, 100)
+
+    def time_assign_list_of_columns_concat(self):
+        df = DataFrame(np.random.randn(self.N, 100))
+        concat([self.df, df], axis=1)
+
 
 class ChainIndexing:
 

asv_bench/benchmarks/join_merge.py

@@ -132,6 +132,9 @@ def time_join_dataframe_index_single_key_small(self, sort):
     def time_join_dataframe_index_shuffle_key_bigger_sort(self, sort):
         self.df_shuf.join(self.df_key2, on="key2", sort=sort)
 
+    def time_join_dataframes_cross(self, sort):
+        self.df.loc[:2000].join(self.df_key1, how="cross", sort=sort)
+
 
 class JoinIndex:
     def setup(self):
@@ -205,6 +208,9 @@ def time_merge_dataframe_integer_2key(self, sort):
     def time_merge_dataframe_integer_key(self, sort):
         merge(self.df, self.df2, on="key1", sort=sort)
 
+    def time_merge_dataframes_cross(self, sort):
+        merge(self.left.loc[:2000], self.right.loc[:2000], how="cross", sort=sort)
+
 
 class I8Merge:
 

asv_bench/benchmarks/reshape.py

@@ -103,7 +103,10 @@ def setup(self):
         nidvars = 20
         N = 5000
         self.letters = list("ABCD")
-        yrvars = [l + str(num) for l, num in product(self.letters, range(1, nyrs + 1))]
+        yrvars = [
+            letter + str(num)
+            for letter, num in product(self.letters, range(1, nyrs + 1))
+        ]
         columns = [str(i) for i in range(nidvars)] + yrvars
         self.df = DataFrame(np.random.randn(N, nidvars + len(yrvars)), columns=columns)
         self.df["id"] = self.df.index

asv_bench/benchmarks/series_methods.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 
-from pandas import NaT, Series, date_range
+from pandas import Categorical, NaT, Series, date_range
 
 from .pandas_vb_common import tm
 
@@ -36,6 +36,28 @@ def time_isin(self, dtypes):
         self.s.isin(self.values)
 
 
+class IsInDatetime64:
+    def setup(self):
+        dti = date_range(
+            start=datetime(2015, 10, 26), end=datetime(2016, 1, 1), freq="50s"
+        )
+        self.ser = Series(dti)
+        self.subset = self.ser._values[::3]
+        self.cat_subset = Categorical(self.subset)
+
+    def time_isin(self):
+        self.ser.isin(self.subset)
+
+    def time_isin_cat_values(self):
+        self.ser.isin(self.cat_subset)
+
+    def time_isin_mismatched_dtype(self):
+        self.ser.isin([1, 2])
+
+    def time_isin_empty(self):
+        self.ser.isin([])
+
+
 class IsInFloat64:
     def setup(self):
         self.small = Series([1, 2], dtype=np.float64)

azure-pipelines.yml

@@ -40,7 +40,7 @@ jobs:
         . ~/virtualenvs/pandas-dev/bin/activate && \
         python -m pip install --no-deps -U pip wheel setuptools && \
         pip install cython numpy python-dateutil pytz pytest pytest-xdist hypothesis pytest-azurepipelines && \
-        python setup.py build_ext -q -i -j2 && \
+        python setup.py build_ext -q -j2 && \
         python -m pip install --no-build-isolation -e . && \
         pytest -m 'not slow and not network and not clipboard' pandas --junitxml=test-data.xml"
       displayName: 'Run 32-bit manylinux2014 Docker Build / Tests'

ci/azure/windows.yml

@@ -34,7 +34,7 @@ jobs:
     - bash: |
         source activate pandas-dev
         conda list
-        python setup.py build_ext -q -i -j 4
+        python setup.py build_ext -q -j 4
         python -m pip install --no-build-isolation -e .
       displayName: 'Build'
 

ci/deps/azure-39.yaml

@@ -15,3 +15,8 @@ dependencies:
   - numpy
   - python-dateutil
   - pytz
+
+  # optional dependencies
+  - pytables
+  - scipy
+  - pyarrow=1.0

ci/deps/travis-37-locale.yaml

@@ -34,7 +34,7 @@ dependencies:
   - pyarrow>=0.17
   - pytables>=3.5.1
   - scipy
-  - xarray=0.12.0
+  - xarray=0.12.3
   - xlrd
   - xlsxwriter
   - xlwt

ci/setup_env.sh

@@ -108,6 +108,12 @@ fi
 echo "activate pandas-dev"
 source activate pandas-dev
 
+# Explicitly set an environment variable indicating that this is pandas' CI environment.
+#
+# This allows us to enable things like -Werror that shouldn't be activated in
+# downstream CI jobs that may also build pandas from source.
+export PANDAS_CI=1
+
 echo
 echo "remove any installed pandas package"
 echo "w/o removing anything else"
@@ -131,7 +137,7 @@ conda list pandas
 # Make sure any error below is reported as such
 
 echo "[Build extensions]"
-python setup.py build_ext -q -i -j2
+python setup.py build_ext -q -j2
 
 echo "[Updating pip]"
 python -m pip install --no-deps -U pip wheel setuptools

doc/source/development/contributing.rst

@@ -146,7 +146,7 @@ Creating a development environment
 ----------------------------------
 
 To test out code changes, you'll need to build pandas from source, which
-requires a C compiler and Python environment. If you're making documentation
+requires a C/C++ compiler and Python environment. If you're making documentation
 changes, you can skip to :ref:`contributing.documentation` but you won't be able
 to build the documentation locally before pushing your changes.
 
@@ -183,7 +183,7 @@ See https://www.jetbrains.com/help/pycharm/docker.html for details.
 
 Note that you might need to rebuild the C extensions if/when you merge with upstream/master using::
 
-    python setup.py build_ext --inplace -j 4
+    python setup.py build_ext -j 4
 
 .. _contributing.dev_c:
 
@@ -195,6 +195,13 @@ operations. To install pandas from source, you need to compile these C
 extensions, which means you need a C compiler. This process depends on which
 platform you're using.
 
+If you have setup your environment using ``conda``, the packages ``c-compiler``
+and ``cxx-compiler`` will install a fitting compiler for your platform that is
+compatible with the remaining conda packages. On Windows and macOS, you will
+also need to install the SDKs as they have to be distributed separately.
+These packages will be automatically installed by using ``pandas``'s
+``environment.yml``.
+
 **Windows**
 
 You will need `Build Tools for Visual Studio 2017
@@ -206,12 +213,33 @@ You will need `Build Tools for Visual Studio 2017
 	scrolling down to "All downloads" -> "Tools for Visual Studio 2019".
 	In the installer, select the "C++ build tools" workload.
 
+You can install the necessary components on the commandline using
+`vs_buildtools.exe <https://aka.ms/vs/16/release/vs_buildtools.exe>`_:
+
+.. code::
+
+    vs_buildtools.exe --quiet --wait --norestart --nocache ^
+        --installPath C:\BuildTools ^
+        --add "Microsoft.VisualStudio.Workload.VCTools;includeRecommended" ^
+        --add Microsoft.VisualStudio.Component.VC.v141 ^
+        --add Microsoft.VisualStudio.Component.VC.v141.x86.x64 ^
+        --add Microsoft.VisualStudio.Component.Windows10SDK.17763
+
+To setup the right paths on the commandline, call
+``"C:\BuildTools\VC\Auxiliary\Build\vcvars64.bat" -vcvars_ver=14.16 10.0.17763.0``.
+
 **macOS**
 
-Information about compiler installation can be found here:
+To use the ``conda``-based compilers, you will need to install the
+Developer Tools using ``xcode-select --install``. Otherwise
+information about compiler installation can be found here:
 https://devguide.python.org/setup/#macos
 
-**Unix**
+**Linux**
+
+For Linux-based ``conda`` installations, you won't have to install any
+additional components outside of the conda environment. The instructions
+below are only needed if your setup isn't based on conda environments.
 
 Some Linux distributions will come with a pre-installed C compiler. To find out
 which compilers (and versions) are installed on your system::
@@ -243,11 +271,10 @@ Let us know if you have any difficulties by opening an issue or reaching out on
 Creating a Python environment
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Now that you have a C compiler, create an isolated pandas development
-environment:
+Now create an isolated pandas development environment:
 
-* Install either `Anaconda <https://www.anaconda.com/download/>`_ or `miniconda
-  <https://conda.io/miniconda.html>`_
+* Install either `Anaconda <https://www.anaconda.com/download/>`_, `miniconda
+  <https://conda.io/miniconda.html>`_, or `miniforge <https://github.com/conda-forge/miniforge>`_
 * Make sure your conda is up to date (``conda update conda``)
 * Make sure that you have :ref:`cloned the repository <contributing.forking>`
 * ``cd`` to the pandas source directory
@@ -268,7 +295,7 @@ We'll now kick off a three-step process:
    source activate pandas-dev
 
    # Build and install pandas
-   python setup.py build_ext --inplace -j 4
+   python setup.py build_ext -j 4
    python -m pip install -e . --no-build-isolation --no-use-pep517
 
 At this point you should be able to import pandas from your locally built version::
@@ -315,7 +342,7 @@ You'll need to have at least Python 3.6.1 installed on your system.
    python -m pip install -r requirements-dev.txt
 
    # Build and install pandas
-   python setup.py build_ext --inplace -j 4
+   python setup.py build_ext -j 4
    python -m pip install -e . --no-build-isolation --no-use-pep517
 
 **Unix**/**macOS with pyenv**
@@ -339,7 +366,7 @@ Consult the docs for setting up pyenv `here <https://github.com/pyenv/pyenv>`__.
    python -m pip install -r requirements-dev.txt
 
    # Build and install pandas
-   python setup.py build_ext --inplace -j 4
+   python setup.py build_ext -j 4
    python -m pip install -e . --no-build-isolation --no-use-pep517
 
 **Windows**
@@ -365,7 +392,7 @@ should already exist.
    python -m pip install -r requirements-dev.txt
 
    # Build and install pandas
-   python setup.py build_ext --inplace -j 4
+   python setup.py build_ext -j 4
    python -m pip install -e . --no-build-isolation --no-use-pep517
 
 Creating a branch

doc/source/development/policies.rst

@@ -35,7 +35,7 @@ We will not introduce new deprecations in patch releases.
 Deprecations will only be enforced in **major** releases. For example, if a
 behavior is deprecated in pandas 1.2.0, it will continue to work, with a
 warning, for all releases in the 1.x series. The behavior will change and the
-deprecation removed in the next next major release (2.0.0).
+deprecation removed in the next major release (2.0.0).
 
 .. note::
 

doc/source/getting_started/install.rst

@@ -284,7 +284,7 @@ pyxlsb                    1.0.6              Reading for xlsb files
 qtpy                                         Clipboard I/O
 s3fs                      0.4.0              Amazon S3 access
 tabulate                  0.8.3              Printing in Markdown-friendly format (see `tabulate`_)
-xarray                    0.12.0             pandas-like API for N-dimensional data
+xarray                    0.12.3             pandas-like API for N-dimensional data
 xclip                                        Clipboard I/O on linux
 xlrd                      1.2.0              Excel reading
 xlwt                      1.3.0              Excel writing

doc/source/reference/index.rst

@@ -30,7 +30,6 @@ public functions related to data types in pandas.
    series
    frame
    arrays
-   panel
    indexing
    offset_frequency
    window

doc/source/reference/panel.rst

@@ -36,6 +36,7 @@ Style application
    Styler.where
    Styler.format
    Styler.set_precision
+   Styler.set_td_classes
    Styler.set_table_styles
    Styler.set_table_attributes
    Styler.set_caption

doc/source/reference/style.rst

@@ -439,7 +439,7 @@ Data Classes as introduced in `PEP557 <https://www.python.org/dev/peps/pep-0557>
 can be passed into the DataFrame constructor.
 Passing a list of dataclasses is equivalent to passing a list of dictionaries.
 
-Please be aware, that that all values in the list should be dataclasses, mixing
+Please be aware, that all values in the list should be dataclasses, mixing
 types in the list would result in a TypeError.
 
 .. ipython:: python

doc/source/user_guide/dsintro.rst

@@ -524,6 +524,15 @@ index are the group names and whose values are the sizes of each group.
 
    grouped.describe()
 
+Another aggregation example is to compute the number of unique values of each group. This is similar to the ``value_counts`` function, except that it only counts unique values.
+
+.. ipython:: python
+
+   ll = [['foo', 1], ['foo', 2], ['foo', 2], ['bar', 1], ['bar', 1]]
+   df4 = pd.DataFrame(ll, columns=["A", "B"])
+   df4
+   df4.groupby("A")["B"].nunique()
+
 .. note::
 
    Aggregation functions **will not** return the groups that you are aggregating over