Merge remote-tracking branch 'upstream/master' into shiny-new-feature

Author: daniellebrown

.github/workflows/ci.yml

@@ -15,12 +15,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
+    - name: Setting conda path
+      run: echo "::add-path::${HOME}/miniconda3/bin"
+
     - name: Checkout
       uses: actions/checkout@v1
 
-    - name: Setting conda path
-      run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"
-
     - name: Looking for unwanted patterns
       run: ci/code_checks.sh patterns
       if: true

asv_bench/benchmarks/index_object.py

@@ -7,6 +7,7 @@
     Float64Index,
     Index,
     IntervalIndex,
+    MultiIndex,
     RangeIndex,
     Series,
     date_range,
@@ -111,6 +112,18 @@ def time_get_loc_dec(self):
         self.idx_dec.get_loc(100000)
 
 
+class IndexEquals:
+    def setup(self):
+        idx_large_fast = RangeIndex(100000)
+        idx_small_slow = date_range(start="1/1/2012", periods=1)
+        self.mi_large_slow = MultiIndex.from_product([idx_large_fast, idx_small_slow])
+
+        self.idx_non_object = RangeIndex(1)
+
+    def time_non_object_equals_multiindex(self):
+        self.idx_non_object.equals(self.mi_large_slow)
+
+
 class IndexAppend:
     def setup(self):
 

asv_bench/benchmarks/multiindex_object.py

@@ -2,7 +2,7 @@
 
 import numpy as np
 
-from pandas import DataFrame, MultiIndex, date_range
+from pandas import DataFrame, MultiIndex, RangeIndex, date_range
 import pandas.util.testing as tm
 
 
@@ -147,4 +147,16 @@ def time_categorical_level(self):
         self.df.set_index(["a", "b"])
 
 
+class Equals:
+    def setup(self):
+        idx_large_fast = RangeIndex(100000)
+        idx_small_slow = date_range(start="1/1/2012", periods=1)
+        self.mi_large_slow = MultiIndex.from_product([idx_large_fast, idx_small_slow])
+
+        self.idx_non_object = RangeIndex(1)
+
+    def time_equals_non_object_index(self):
+        self.mi_large_slow.equals(self.idx_non_object)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/stat_ops.py

@@ -7,20 +7,14 @@
 
 class FrameOps:
 
-    params = [ops, ["float", "int"], [0, 1], [True, False]]
-    param_names = ["op", "dtype", "axis", "use_bottleneck"]
+    params = [ops, ["float", "int"], [0, 1]]
+    param_names = ["op", "dtype", "axis"]
 
-    def setup(self, op, dtype, axis, use_bottleneck):
+    def setup(self, op, dtype, axis):
         df = pd.DataFrame(np.random.randn(100000, 4)).astype(dtype)
-        try:
-            pd.options.compute.use_bottleneck = use_bottleneck
-        except TypeError:
-            from pandas.core import nanops
-
-            nanops._USE_BOTTLENECK = use_bottleneck
         self.df_func = getattr(df, op)
 
-    def time_op(self, op, dtype, axis, use_bottleneck):
+    def time_op(self, op, dtype, axis):
         self.df_func(axis=axis)
 
 
@@ -46,20 +40,14 @@ def time_op(self, level, op):
 
 class SeriesOps:
 
-    params = [ops, ["float", "int"], [True, False]]
-    param_names = ["op", "dtype", "use_bottleneck"]
+    params = [ops, ["float", "int"]]
+    param_names = ["op", "dtype"]
 
-    def setup(self, op, dtype, use_bottleneck):
+    def setup(self, op, dtype):
         s = pd.Series(np.random.randn(100000)).astype(dtype)
-        try:
-            pd.options.compute.use_bottleneck = use_bottleneck
-        except TypeError:
-            from pandas.core import nanops
-
-            nanops._USE_BOTTLENECK = use_bottleneck
         self.s_func = getattr(s, op)
 
-    def time_op(self, op, dtype, use_bottleneck):
+    def time_op(self, op, dtype):
         self.s_func()
 
 
@@ -101,61 +89,49 @@ def time_average_old(self, constructor, pct):
 
 class Correlation:
 
-    params = [["spearman", "kendall", "pearson"], [True, False]]
-    param_names = ["method", "use_bottleneck"]
+    params = [["spearman", "kendall", "pearson"]]
+    param_names = ["method"]
 
-    def setup(self, method, use_bottleneck):
-        try:
-            pd.options.compute.use_bottleneck = use_bottleneck
-        except TypeError:
-            from pandas.core import nanops
+    def setup(self, method):
+        self.df = pd.DataFrame(np.random.randn(500, 15))
+        self.df2 = pd.DataFrame(np.random.randn(500, 15))
+        self.df_wide = pd.DataFrame(np.random.randn(500, 100))
+        self.df_wide_nans = self.df_wide.where(np.random.random((500, 100)) < 0.9)
+        self.s = pd.Series(np.random.randn(500))
+        self.s2 = pd.Series(np.random.randn(500))
 
-            nanops._USE_BOTTLENECK = use_bottleneck
-        self.df = pd.DataFrame(np.random.randn(1000, 30))
-        self.df2 = pd.DataFrame(np.random.randn(1000, 30))
-        self.df_wide = pd.DataFrame(np.random.randn(1000, 200))
-        self.df_wide_nans = self.df_wide.where(np.random.random((1000, 200)) < 0.9)
-        self.s = pd.Series(np.random.randn(1000))
-        self.s2 = pd.Series(np.random.randn(1000))
-
-    def time_corr(self, method, use_bottleneck):
+    def time_corr(self, method):
         self.df.corr(method=method)
 
-    def time_corr_wide(self, method, use_bottleneck):
+    def time_corr_wide(self, method):
         self.df_wide.corr(method=method)
 
-    def time_corr_wide_nans(self, method, use_bottleneck):
+    def time_corr_wide_nans(self, method):
         self.df_wide_nans.corr(method=method)
 
-    def peakmem_corr_wide(self, method, use_bottleneck):
+    def peakmem_corr_wide(self, method):
         self.df_wide.corr(method=method)
 
-    def time_corr_series(self, method, use_bottleneck):
+    def time_corr_series(self, method):
         self.s.corr(self.s2, method=method)
 
-    def time_corrwith_cols(self, method, use_bottleneck):
+    def time_corrwith_cols(self, method):
         self.df.corrwith(self.df2, method=method)
 
-    def time_corrwith_rows(self, method, use_bottleneck):
+    def time_corrwith_rows(self, method):
         self.df.corrwith(self.df2, axis=1, method=method)
 
 
 class Covariance:
 
-    params = [[True, False]]
-    param_names = ["use_bottleneck"]
-
-    def setup(self, use_bottleneck):
-        try:
-            pd.options.compute.use_bottleneck = use_bottleneck
-        except TypeError:
-            from pandas.core import nanops
+    params = []
+    param_names = []
 
-            nanops._USE_BOTTLENECK = use_bottleneck
+    def setup(self):
         self.s = pd.Series(np.random.randn(100000))
         self.s2 = pd.Series(np.random.randn(100000))
 
-    def time_cov_series(self, use_bottleneck):
+    def time_cov_series(self):
         self.s.cov(self.s2)
 
 

ci/azure/posix.yml

@@ -20,14 +20,14 @@ jobs:
           CONDA_PY: "36"
           PATTERN: "not slow and not network"
         py36_locale_slow_old_np:
-          ENV_FILE: ci/deps/azure-36-locale.yaml
+          ENV_FILE: ci/deps/azure-36-locale_slow.yaml
           CONDA_PY: "36"
           PATTERN: "slow"
           LOCALE_OVERRIDE: "zh_CN.UTF-8"
           EXTRA_APT: "language-pack-zh-hans"
 
-        py36_locale_slow:
-          ENV_FILE: ci/deps/azure-36-locale_slow.yaml
+        py36_locale:
+          ENV_FILE: ci/deps/azure-36-locale.yaml
           CONDA_PY: "36"
           PATTERN: "not slow and not network"
           LOCALE_OVERRIDE: "it_IT.UTF-8"
@@ -44,13 +44,15 @@ jobs:
           PATTERN: "not slow and not network"
           LOCALE_OVERRIDE: "zh_CN.UTF-8"
 
-        py37_np_dev:
-          ENV_FILE: ci/deps/azure-37-numpydev.yaml
-          CONDA_PY: "37"
-          PATTERN: "not slow and not network"
-          TEST_ARGS: "-W error"
-          PANDAS_TESTING_MODE: "deprecate"
-          EXTRA_APT: "xsel"
+        # Disabled for NumPy object-dtype warning.
+        # https://github.com/pandas-dev/pandas/issues/30043
+        # py37_np_dev:
+        #   ENV_FILE: ci/deps/azure-37-numpydev.yaml
+        #   CONDA_PY: "37"
+        #   PATTERN: "not slow and not network"
+        #   TEST_ARGS: "-W error"
+        #   PANDAS_TESTING_MODE: "deprecate"
+        #   EXTRA_APT: "xsel"
 
   steps:
     - script: |

ci/code_checks.sh

@@ -105,7 +105,12 @@ if [[ -z "$CHECK" || "$CHECK" == "lint" ]]; then
 
     # Imports - Check formatting using isort see setup.cfg for settings
     MSG='Check import format using isort ' ; echo $MSG
-    isort --recursive --check-only pandas asv_bench
+    ISORT_CMD="isort --recursive --check-only pandas asv_bench"
+    if [[ "$GITHUB_ACTIONS" == "true" ]]; then
+        eval $ISORT_CMD | awk '{print "##[error]" $0}'; RET=$(($RET + ${PIPESTATUS[0]}))
+    else
+        eval $ISORT_CMD
+    fi
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/deps/azure-36-locale.yaml

@@ -13,20 +13,27 @@ dependencies:
   - pytest-azurepipelines
 
   # pandas dependencies
-  - beautifulsoup4==4.6.0
-  - bottleneck=1.2.*
+  - beautifulsoup4
+  - gcsfs
+  - html5lib
+  - ipython
+  - jinja2
   - lxml
-  - matplotlib=2.2.2
-  - numpy=1.14.*
-  - openpyxl=2.4.8
+  - matplotlib=3.0.*
+  - nomkl
+  - numexpr
+  - numpy=1.15.*
+  - openpyxl
+  # lowest supported version of pyarrow (putting it here instead of in
+  # azure-36-minimum_versions because it needs numpy >= 1.14)
+  - pyarrow=0.12
+  - pytables
   - python-dateutil
-  - python-blosc
-  - pytz=2017.2
+  - pytz
+  - s3fs
   - scipy
-  - sqlalchemy=1.1.4
-  - xlrd=1.1.0
-  - xlsxwriter=0.9.8
-  - xlwt=1.2.0
-  - pip
-  - pip:
-    - html5lib==1.0b2
+  - xarray
+  - xlrd
+  - xlsxwriter
+  - xlwt
+  - moto

ci/deps/azure-36-locale_slow.yaml

@@ -13,24 +13,20 @@ dependencies:
   - pytest-azurepipelines
 
   # pandas dependencies
-  - beautifulsoup4
-  - gcsfs
-  - html5lib
-  - ipython
-  - jinja2
+  - beautifulsoup4==4.6.0
+  - bottleneck=1.2.*
   - lxml
-  - matplotlib=3.0.*
-  - nomkl
-  - numexpr
-  - numpy=1.15.*
-  - openpyxl
-  - pytables
+  - matplotlib=2.2.2
+  - numpy=1.14.*
+  - openpyxl=2.4.8
   - python-dateutil
-  - pytz
-  - s3fs
+  - python-blosc
+  - pytz=2017.2
   - scipy
-  - xarray
-  - xlrd
-  - xlsxwriter
-  - xlwt
-  - moto
+  - sqlalchemy=1.1.4
+  - xlrd=1.1.0
+  - xlsxwriter=0.9.8
+  - xlwt=1.2.0
+  - pip
+  - pip:
+    - html5lib==1.0b2

ci/deps/azure-37-locale.yaml

@@ -1,6 +1,5 @@
 name: pandas-dev
 channels:
-  - defaults
   - conda-forge
 dependencies:
   - python=3.7.*
@@ -33,3 +32,4 @@ dependencies:
   - xlrd
   - xlsxwriter
   - xlwt
+  - pyarrow>=0.15

ci/deps/travis-36-cov.yaml

@@ -33,8 +33,7 @@ dependencies:
   # https://github.com/pydata/pandas-gbq/issues/271
   - google-cloud-bigquery<=1.11
   - psycopg2
-  # pyarrow segfaults on load: https://github.com/pandas-dev/pandas/issues/26716
-  # - pyarrow=0.9.0
+  - pyarrow>=0.12.0
   - pymysql
   - pytables
   - python-snappy

ci/print_skipped.py

@@ -5,12 +5,12 @@
 
 def main(filename):
     if not os.path.isfile(filename):
-        return
+        raise RuntimeError(f"Could not find junit file {repr(filename)}")
 
     tree = et.parse(filename)
     root = tree.getroot()
     current_class = ""
-    for el in root.findall("testcase"):
+    for el in root.iter("testcase"):
         cn = el.attrib["classname"]
         for sk in el.findall("skipped"):
             old_class = current_class
@@ -32,8 +32,7 @@ def main(filename):
             print("-" * 80)
         else:
             print(
-                "#{i} {class_name}.{test_name}: {message}".format(
-                    **dict(test_data, i=i)
-                )
+                f"#{i} {test_data['class_name']}."
+                f"{test_data['test_name']}: {test_data['message']}"
             )
             i += 1