Merge branch 'master' into fix/combine-first

Author: simonjayhawkins

.github/workflows/ci.yml

@@ -4,7 +4,9 @@ on:
   push:
     branches: master
   pull_request:
-    branches: master
+    branches:
+      - master
+      - 1.1.x
 
 env:
   ENV_FILE: environment.yml

.github/workflows/stale-pr.yml

@@ -0,0 +1,21 @@
+name: "Stale PRs"
+on:
+  schedule:
+  # * is a special character in YAML so you have to quote this string
+  - cron: "0 */6 * * *"
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-pr-message: "This pull request is stale because it has been open for thirty days with no activity."
+        skip-stale-pr-message: false
+        stale-pr-label: "Stale"
+        exempt-pr-labels: "Needs Review,Blocked"
+        days-before-stale: 30
+        days-before-close: -1
+        remove-stale-when-updated: true
+        debug-only: true

.pre-commit-config.yaml

@@ -30,15 +30,3 @@ repos:
     -   id: isort
         language: python_venv
         exclude: ^pandas/__init__\.py$|^pandas/core/api\.py$
--   repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.730
-    hooks:
-     -  id: mypy
-        args:
-          # As long as a some files are excluded from check-untyped-defs
-          # we have to exclude it from the pre-commit hook as the configuration
-          # is based on modules but the hook runs on files.
-          - --no-check-untyped-defs
-          - --follow-imports
-          - skip
-        files: pandas/

.travis.yml

@@ -45,7 +45,7 @@ matrix:
         - JOB="3.7, arm64" PYTEST_WORKERS=8 ENV_FILE="ci/deps/travis-37-arm64.yaml" PATTERN="(not slow and not network and not clipboard)"
 
     - env:
-        - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
+        - JOB="3.7, locale" ENV_FILE="ci/deps/travis-37-locale.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8" SQL="1"
       services:
         - mysql
         - postgresql
@@ -54,7 +54,7 @@ matrix:
         # Enabling Deprecations when running tests
         # PANDAS_TESTING_MODE="deprecate" causes DeprecationWarning messages to be displayed in the logs
         # See pandas/_testing.py for more details.
-        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1"
+        - JOB="3.7, coverage" ENV_FILE="ci/deps/travis-37-cov.yaml" PATTERN="((not slow and not network and not clipboard) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true SQL="1"
       services:
         - mysql
         - postgresql

Makefile

@@ -25,3 +25,16 @@ doc:
 	cd doc; \
 	python make.py clean; \
 	python make.py html
+
+check:
+	python3 scripts/validate_unwanted_patterns.py \
+		--validation-type="private_function_across_module" \
+		--included-file-extensions="py" \
+		--excluded-file-paths=pandas/tests,asv_bench/,pandas/_vendored \
+		pandas/
+
+	python3 scripts/validate_unwanted_patterns.py \
+		--validation-type="private_import_across_module" \
+		--included-file-extensions="py" \
+		--excluded-file-paths=pandas/tests,asv_bench/,pandas/_vendored,doc/
+		pandas/

README.md

@@ -32,7 +32,7 @@ its way towards this goal.
 Here are just a few of the things that pandas does well:
 
   - Easy handling of [**missing data**][missing-data] (represented as
-    `NaN`) in floating point as well as non-floating point data
+    `NaN`, `NA`, or `NaT`) in floating point as well as non-floating point data
   - Size mutability: columns can be [**inserted and
     deleted**][insertion-deletion] from DataFrame and higher dimensional
     objects

asv_bench/asv.conf.json

@@ -26,7 +26,7 @@
     // The Pythons you'd like to test against.  If not provided, defaults
     // to the current version of Python used to run `asv`.
     // "pythons": ["2.7", "3.4"],
-    "pythons": ["3.6"],
+    "pythons": ["3.8"],
 
     // The matrix of dependencies to test.  Each key is the name of a
     // package (in PyPI) and the values are version numbers.  An empty

asv_bench/benchmarks/frame_ctor.py

@@ -6,7 +6,7 @@
 from .pandas_vb_common import tm
 
 try:
-    from pandas.tseries.offsets import Nano, Hour
+    from pandas.tseries.offsets import Hour, Nano
 except ImportError:
     # For compatibility with older versions
     from pandas.core.datetools import *  # noqa

asv_bench/benchmarks/gil.py

@@ -7,14 +7,14 @@
 
 try:
     from pandas import (
-        rolling_median,
+        rolling_kurt,
+        rolling_max,
         rolling_mean,
+        rolling_median,
         rolling_min,
-        rolling_max,
-        rolling_var,
         rolling_skew,
-        rolling_kurt,
         rolling_std,
+        rolling_var,
     )
 
     have_rolling_methods = True

asv_bench/benchmarks/groupby.py

@@ -627,49 +627,63 @@ def time_first(self):
 
 
 class TransformEngine:
-    def setup(self):
+
+    param_names = ["parallel"]
+    params = [[True, False]]
+
+    def setup(self, parallel):
         N = 10 ** 3
         data = DataFrame(
             {0: [str(i) for i in range(100)] * N, 1: list(range(100)) * N},
             columns=[0, 1],
         )
+        self.parallel = parallel
         self.grouper = data.groupby(0)
 
-    def time_series_numba(self):
+    def time_series_numba(self, parallel):
         def function(values, index):
             return values * 5
 
-        self.grouper[1].transform(function, engine="numba")
+        self.grouper[1].transform(
+            function, engine="numba", engine_kwargs={"parallel": self.parallel}
+        )
 
-    def time_series_cython(self):
+    def time_series_cython(self, parallel):
         def function(values):
             return values * 5
 
         self.grouper[1].transform(function, engine="cython")
 
-    def time_dataframe_numba(self):
+    def time_dataframe_numba(self, parallel):
         def function(values, index):
             return values * 5
 
-        self.grouper.transform(function, engine="numba")
+        self.grouper.transform(
+            function, engine="numba", engine_kwargs={"parallel": self.parallel}
+        )
 
-    def time_dataframe_cython(self):
+    def time_dataframe_cython(self, parallel):
         def function(values):
             return values * 5
 
         self.grouper.transform(function, engine="cython")
 
 
 class AggEngine:
-    def setup(self):
+
+    param_names = ["parallel"]
+    params = [[True, False]]
+
+    def setup(self, parallel):
         N = 10 ** 3
         data = DataFrame(
             {0: [str(i) for i in range(100)] * N, 1: list(range(100)) * N},
             columns=[0, 1],
         )
+        self.parallel = parallel
         self.grouper = data.groupby(0)
 
-    def time_series_numba(self):
+    def time_series_numba(self, parallel):
         def function(values, index):
             total = 0
             for i, value in enumerate(values):
@@ -679,9 +693,11 @@ def function(values, index):
                     total += value * 2
             return total
 
-        self.grouper[1].agg(function, engine="numba")
+        self.grouper[1].agg(
+            function, engine="numba", engine_kwargs={"parallel": self.parallel}
+        )
 
-    def time_series_cython(self):
+    def time_series_cython(self, parallel):
         def function(values):
             total = 0
             for i, value in enumerate(values):
@@ -693,7 +709,7 @@ def function(values):
 
         self.grouper[1].agg(function, engine="cython")
 
-    def time_dataframe_numba(self):
+    def time_dataframe_numba(self, parallel):
         def function(values, index):
             total = 0
             for i, value in enumerate(values):
@@ -703,9 +719,11 @@ def function(values, index):
                     total += value * 2
             return total
 
-        self.grouper.agg(function, engine="numba")
+        self.grouper.agg(
+            function, engine="numba", engine_kwargs={"parallel": self.parallel}
+        )
 
-    def time_dataframe_cython(self):
+    def time_dataframe_cython(self, parallel):
         def function(values):
             total = 0
             for i, value in enumerate(values):

asv_bench/benchmarks/index_object.py

@@ -57,8 +57,8 @@ def time_datetime_difference_disjoint(self):
 
 class Range:
     def setup(self):
-        self.idx_inc = RangeIndex(start=0, stop=10 ** 7, step=3)
-        self.idx_dec = RangeIndex(start=10 ** 7, stop=-1, step=-3)
+        self.idx_inc = RangeIndex(start=0, stop=10 ** 6, step=3)
+        self.idx_dec = RangeIndex(start=10 ** 6, stop=-1, step=-3)
 
     def time_max(self):
         self.idx_inc.max()
@@ -73,15 +73,23 @@ def time_min_trivial(self):
         self.idx_inc.min()
 
     def time_get_loc_inc(self):
-        self.idx_inc.get_loc(900000)
+        self.idx_inc.get_loc(900_000)
 
     def time_get_loc_dec(self):
-        self.idx_dec.get_loc(100000)
+        self.idx_dec.get_loc(100_000)
+
+    def time_iter_inc(self):
+        for _ in self.idx_inc:
+            pass
+
+    def time_iter_dec(self):
+        for _ in self.idx_dec:
+            pass
 
 
 class IndexEquals:
     def setup(self):
-        idx_large_fast = RangeIndex(100000)
+        idx_large_fast = RangeIndex(100_000)
         idx_small_slow = date_range(start="1/1/2012", periods=1)
         self.mi_large_slow = MultiIndex.from_product([idx_large_fast, idx_small_slow])
 
@@ -94,7 +102,7 @@ def time_non_object_equals_multiindex(self):
 class IndexAppend:
     def setup(self):
 
-        N = 10000
+        N = 10_000
         self.range_idx = RangeIndex(0, 100)
         self.int_idx = self.range_idx.astype(int)
         self.obj_idx = self.int_idx.astype(str)
@@ -168,7 +176,7 @@ def time_get_loc_non_unique_sorted(self, dtype):
 class Float64IndexMethod:
     # GH 13166
     def setup(self):
-        N = 100000
+        N = 100_000
         a = np.arange(N)
         self.ind = Float64Index(a * 4.8000000418824129e-08)
 
@@ -212,7 +220,7 @@ class GC:
     params = [1, 2, 5]
 
     def create_use_drop(self):
-        idx = Index(list(range(1000 * 1000)))
+        idx = Index(list(range(1_000_000)))
         idx._engine
 
     def peakmem_gc_instances(self, N):

asv_bench/benchmarks/io/parsers.py

@@ -2,8 +2,8 @@
 
 try:
     from pandas._libs.tslibs.parsing import (
-        concat_date_cols,
         _does_string_look_like_datetime,
+        concat_date_cols,
     )
 except ImportError:
     # Avoid whole benchmark suite import failure on asv (currently 0.4)

asv_bench/benchmarks/package.py

@@ -4,22 +4,16 @@
 import subprocess
 import sys
 
-from pandas.compat import PY37
-
 
 class TimeImport:
     def time_import(self):
-        if PY37:
-            # on py37+ we the "-X importtime" usage gives us a more precise
-            #  measurement of the import time we actually care about,
-            #  without the subprocess or interpreter overhead
-            cmd = [sys.executable, "-X", "importtime", "-c", "import pandas as pd"]
-            p = subprocess.run(cmd, stderr=subprocess.PIPE)
-
-            line = p.stderr.splitlines()[-1]
-            field = line.split(b"|")[-2].strip()
-            total = int(field)  # microseconds
-            return total
+        # on py37+ we the "-X importtime" usage gives us a more precise
+        #  measurement of the import time we actually care about,
+        #  without the subprocess or interpreter overhead
+        cmd = [sys.executable, "-X", "importtime", "-c", "import pandas as pd"]
+        p = subprocess.run(cmd, stderr=subprocess.PIPE)
 
-        cmd = [sys.executable, "-c", "import pandas as pd"]
-        subprocess.run(cmd, stderr=subprocess.PIPE)
+        line = p.stderr.splitlines()[-1]
+        field = line.split(b"|")[-2].strip()
+        total = int(field)  # microseconds
+        return total

asv_bench/benchmarks/strings.py

@@ -7,6 +7,21 @@
 from .pandas_vb_common import tm
 
 
+class Construction:
+
+    params = ["str", "string"]
+    param_names = ["dtype"]
+
+    def setup(self, dtype):
+        self.data = tm.rands_array(nchars=10 ** 5, size=10)
+
+    def time_construction(self, dtype):
+        Series(self.data, dtype=dtype)
+
+    def peakmem_construction(self, dtype):
+        Series(self.data, dtype=dtype)
+
+
 class Methods:
     def setup(self):
         self.s = Series(tm.makeStringIndex(10 ** 5))

asv_bench/benchmarks/tslibs/normalize.py

@@ -1,5 +1,5 @@
 try:
-    from pandas._libs.tslibs import normalize_i8_timestamps, is_date_array_normalized
+    from pandas._libs.tslibs import is_date_array_normalized, normalize_i8_timestamps
 except ImportError:
     from pandas._libs.tslibs.conversion import (
         normalize_i8_timestamps,

azure-pipelines.yml

@@ -1,9 +1,11 @@
 # Adapted from https://github.com/numba/numba/blob/master/azure-pipelines.yml
 trigger:
 - master
+- 1.1.x
 
 pr:
 - master
+- 1.1.x
 
 variables:
   PYTEST_WORKERS: auto