Merge remote-tracking branch 'upstream/master' into value_counts_part1

Author: DataInformer

.travis.yml

@@ -58,12 +58,6 @@ matrix:
       services:
         - mysql
         - postgresql
-
-    - env:
-        - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" SQL="1"
-      services:
-        - mysql
-        - postgresql
   allow_failures:
     - arch: arm64
       env:

LICENSES/XARRAY_LICENSE

@@ -1,3 +1,7 @@
+Copyright 2014-2019, xarray Developers
+
+--------------------------------------------------------------------------------
+
 Apache License
 Version 2.0, January 2004
 http://www.apache.org/licenses/

asv_bench/asv.conf.json

@@ -53,6 +53,7 @@
         "xlwt": [],
         "odfpy": [],
         "pytest": [],
+        "jinja2": [],
         // If using Windows with python 2.7 and want to build using the
         // mingw toolchain (rather than MSVC), uncomment the following line.
         // "libpython": [],

asv_bench/benchmarks/io/style.py

@@ -0,0 +1,34 @@
+import numpy as np
+
+from pandas import DataFrame
+
+
+class RenderApply:
+
+    params = [[12, 24, 36], [12, 120]]
+    param_names = ["cols", "rows"]
+
+    def setup(self, cols, rows):
+        self.df = DataFrame(
+            np.random.randn(rows, cols),
+            columns=[f"float_{i+1}" for i in range(cols)],
+            index=[f"row_{i+1}" for i in range(rows)],
+        )
+        self._style_apply()
+
+    def time_render(self, cols, rows):
+        self.st.render()
+
+    def peakmem_apply(self, cols, rows):
+        self._style_apply()
+
+    def peakmem_render(self, cols, rows):
+        self.st.render()
+
+    def _style_apply(self):
+        def _apply_func(s):
+            return [
+                "background-color: lightcyan" if s.name == "row_1" else "" for v in s
+            ]
+
+        self.st = self.df.style.apply(_apply_func, axis=1)

asv_bench/benchmarks/rolling.py

@@ -91,11 +91,18 @@ class EWMMethods:
     def setup(self, constructor, window, dtype, method):
         N = 10 ** 5
         arr = (100 * np.random.random(N)).astype(dtype)
+        times = pd.date_range("1900", periods=N, freq="23s")
         self.ewm = getattr(pd, constructor)(arr).ewm(halflife=window)
+        self.ewm_times = getattr(pd, constructor)(arr).ewm(
+            halflife="1 Day", times=times
+        )
 
     def time_ewm(self, constructor, window, dtype, method):
         getattr(self.ewm, method)()
 
+    def time_ewm_times(self, constructor, window, dtype, method):
+        self.ewm.mean()
+
 
 class VariableWindowMethods(Methods):
     params = (

asv_bench/benchmarks/tslibs/fields.py

@@ -0,0 +1,74 @@
+import numpy as np
+
+from pandas._libs.tslibs.fields import (
+    get_date_field,
+    get_start_end_field,
+    get_timedelta_field,
+)
+
+from .tslib import _sizes
+
+
+class TimeGetTimedeltaField:
+    params = [
+        _sizes,
+        ["days", "h", "s", "seconds", "ms", "microseconds", "us", "ns", "nanoseconds"],
+    ]
+    param_names = ["size", "field"]
+
+    def setup(self, size, field):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        self.i8data = arr
+
+    def time_get_timedelta_field(self, size, field):
+        get_timedelta_field(self.i8data, field)
+
+
+class TimeGetDateField:
+    params = [
+        _sizes,
+        [
+            "Y",
+            "M",
+            "D",
+            "h",
+            "m",
+            "s",
+            "us",
+            "ns",
+            "doy",
+            "dow",
+            "woy",
+            "q",
+            "dim",
+            "is_leap_year",
+        ],
+    ]
+    param_names = ["size", "field"]
+
+    def setup(self, size, field):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        self.i8data = arr
+
+    def time_get_date_field(self, size, field):
+        get_date_field(self.i8data, field)
+
+
+class TimeGetStartEndField:
+    params = [
+        _sizes,
+        ["start", "end"],
+        ["month", "quarter", "year"],
+        ["B", None, "QS"],
+        [12, 3, 5],
+    ]
+    param_names = ["size", "side", "period", "freqstr", "month_kw"]
+
+    def setup(self, size, side, period, freqstr, month_kw):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        self.i8data = arr
+
+        self.attrname = f"is_{period}_{side}"
+
+    def time_get_start_end_field(self, size, side, period, freqstr, month_kw):
+        get_start_end_field(self.i8data, self.attrname, freqstr, month_kw=month_kw)

asv_bench/benchmarks/tslibs/normalize.py

@@ -0,0 +1,32 @@
+try:
+    from pandas._libs.tslibs import normalize_i8_timestamps, is_date_array_normalized
+except ImportError:
+    from pandas._libs.tslibs.conversion import (
+        normalize_i8_timestamps,
+        is_date_array_normalized,
+    )
+
+import pandas as pd
+
+from .tslib import _sizes, _tzs
+
+
+class Normalize:
+    params = [
+        _sizes,
+        _tzs,
+    ]
+    param_names = ["size", "tz"]
+
+    def setup(self, size, tz):
+        # use an array that will have is_date_array_normalized give True,
+        #  so we do not short-circuit early.
+        dti = pd.date_range("2016-01-01", periods=10, tz=tz).repeat(size // 10)
+        self.i8data = dti.asi8
+
+    def time_normalize_i8_timestamps(self, size, tz):
+        normalize_i8_timestamps(self.i8data, tz)
+
+    def time_is_date_array_normalized(self, size, tz):
+        # TODO: cases with different levels of short-circuiting
+        is_date_array_normalized(self.i8data, tz)

asv_bench/benchmarks/tslibs/period.py

@@ -2,10 +2,20 @@
 Period benchmarks that rely only on tslibs.  See benchmarks.period for
 Period benchmarks that rely on other parts fo pandas.
 """
-from pandas import Period
+
+import numpy as np
+
+from pandas._libs.tslibs.period import Period, periodarr_to_dt64arr
 
 from pandas.tseries.frequencies import to_offset
 
+from .tslib import _sizes, _tzs
+
+try:
+    from pandas._libs.tslibs.vectorized import dt64arr_to_periodarr
+except ImportError:
+    from pandas._libs.tslibs.period import dt64arr_to_periodarr
+
 
 class PeriodProperties:
 
@@ -68,3 +78,53 @@ def setup(self, freq, is_offset):
 
     def time_period_constructor(self, freq, is_offset):
         Period("2012-06-01", freq=freq)
+
+
+_freq_ints = [
+    1000,
+    1011,  # Annual - November End
+    2000,
+    2011,  # Quarterly - November End
+    3000,
+    4000,
+    4006,  # Weekly - Saturday End
+    5000,
+    6000,
+    7000,
+    8000,
+    9000,
+    10000,
+    11000,
+    12000,
+]
+
+
+class TimePeriodArrToDT64Arr:
+    params = [
+        _sizes,
+        _freq_ints,
+    ]
+    param_names = ["size", "freq"]
+
+    def setup(self, size, freq):
+        arr = np.arange(10, dtype="i8").repeat(size // 10)
+        self.i8values = arr
+
+    def time_periodarray_to_dt64arr(self, size, freq):
+        periodarr_to_dt64arr(self.i8values, freq)
+
+
+class TimeDT64ArrToPeriodArr:
+    params = [
+        _sizes,
+        _freq_ints,
+        _tzs,
+    ]
+    param_names = ["size", "freq", "tz"]
+
+    def setup(self, size, freq, tz):
+        arr = np.arange(10, dtype="i8").repeat(size // 10)
+        self.i8values = arr
+
+    def time_dt64arr_to_periodarr(self, size, freq, tz):
+        dt64arr_to_periodarr(self.i8values, freq, tz)

asv_bench/benchmarks/tslibs/resolution.py

@@ -23,7 +23,10 @@
 import numpy as np
 import pytz
 
-from pandas._libs.tslibs.resolution import get_resolution
+try:
+    from pandas._libs.tslibs import get_resolution
+except ImportError:
+    from pandas._libs.tslibs.resolution import get_resolution
 
 
 class TimeResolution:

asv_bench/benchmarks/tslibs/tslib.py

@@ -21,7 +21,10 @@
 import numpy as np
 import pytz
 
-from pandas._libs.tslib import ints_to_pydatetime
+try:
+    from pandas._libs.tslibs import ints_to_pydatetime
+except ImportError:
+    from pandas._libs.tslib import ints_to_pydatetime
 
 _tzs = [
     None,

asv_bench/benchmarks/tslibs/tz_convert.py

@@ -0,0 +1,43 @@
+import numpy as np
+from pytz import UTC
+
+from pandas._libs.tslibs.tzconversion import tz_localize_to_utc
+
+from .tslib import _sizes, _tzs
+
+try:
+    old_sig = False
+    from pandas._libs.tslibs.tzconversion import tz_convert_from_utc
+except ImportError:
+    old_sig = True
+    from pandas._libs.tslibs.tzconversion import tz_convert as tz_convert_from_utc
+
+
+class TimeTZConvert:
+    params = [
+        _sizes,
+        [x for x in _tzs if x is not None],
+    ]
+    param_names = ["size", "tz"]
+
+    def setup(self, size, tz):
+        arr = np.random.randint(0, 10, size=size, dtype="i8")
+        self.i8data = arr
+
+    def time_tz_convert_from_utc(self, size, tz):
+        # effectively:
+        #  dti = DatetimeIndex(self.i8data, tz=tz)
+        #  dti.tz_localize(None)
+        if size >= 10 ** 6 and str(tz) == "tzlocal()":
+            # asv fill will because each call takes 8+seconds
+            return
+        if old_sig:
+            tz_convert_from_utc(self.i8data, UTC, tz)
+        else:
+            tz_convert_from_utc(self.i8data, tz)
+
+    def time_tz_localize_to_utc(self, size, tz):
+        # effectively:
+        #  dti = DatetimeIndex(self.i8data)
+        #  dti.tz_localize(tz, ambiguous="NaT", nonexistent="NaT")
+        tz_localize_to_utc(self.i8data, tz, ambiguous="NaT", nonexistent="NaT")

ci/azure/posix.yml

@@ -30,6 +30,11 @@ jobs:
           LC_ALL: "zh_CN.utf8"
           EXTRA_APT: "language-pack-zh-hans"
 
+        py36_slow:
+          ENV_FILE: ci/deps/azure-36-slow.yaml
+          CONDA_PY: "36"
+          PATTERN: "slow"
+
         py36_locale:
           ENV_FILE: ci/deps/azure-36-locale.yaml
           CONDA_PY: "36"

ci/deps/azure-36-32bit.yaml

@@ -23,4 +23,4 @@ dependencies:
   - pip
   - pip:
     - cython>=0.29.16
-    - pytest>=5.0.1
+    - pytest>=5.0.1,<6.0.0rc0

ci/deps/azure-36-locale.yaml

@@ -7,7 +7,7 @@ dependencies:
 
   # tools
   - cython>=0.29.16
-  - pytest>=5.0.1
+  - pytest>=5.0.1,<6.0.0rc0
   - pytest-xdist>=1.21
   - pytest-asyncio
   - hypothesis>=3.58.0

ci/deps/azure-36-locale_slow.yaml

@@ -7,7 +7,7 @@ dependencies:
 
   # tools
   - cython>=0.29.16
-  - pytest>=5.0.1
+  - pytest>=5.0.1,<6.0.0rc0
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
   - pytest-azurepipelines

ci/deps/travis-36-slow.yaml renamed to ci/deps/azure-36-slow.yaml

@@ -7,7 +7,7 @@ dependencies:
 
   # tools
   - cython>=0.29.16
-  - pytest>=5.0.1
+  - pytest>=5.0.1,<6.0.0rc0
   - pytest-xdist>=1.21
   - hypothesis>=3.58.0
 

ci/deps/azure-37-locale.yaml

@@ -6,7 +6,7 @@ dependencies:
 
   # tools
   - cython>=0.29.16
-  - pytest>=5.0.1
+  - pytest>=5.0.1,<6.0.0rc0
   - pytest-xdist>=1.21
   - pytest-asyncio
   - hypothesis>=3.58.0
@@ -18,7 +18,7 @@ dependencies:
   - ipython
   - jinja2
   - lxml
-  - matplotlib
+  - matplotlib>=3.3.0
   - moto
   - nomkl
   - numexpr