Merge branch 'master' into get_loc-nan

Author: proost

.travis.yml

@@ -30,6 +30,12 @@ matrix:
       - python: 3.5
 
     include:
+    - dist: bionic
+      # 18.04
+      python: 3.8-dev
+      env:
+        - JOB="3.8-dev" PATTERN="(not slow and not network)"
+
     - dist: trusty
       env:
         - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)"
@@ -71,24 +77,27 @@ before_install:
   # This overrides travis and tells it to look nowhere.
   - export BOTO_CONFIG=/dev/null
 
+
 install:
   - echo "install start"
   - ci/prep_cython_cache.sh
   - ci/setup_env.sh
   - ci/submit_cython_cache.sh
   - echo "install done"
 
+
 before_script:
   # display server (for clipboard functionality) needs to be started here,
   # does not work if done in install:setup_env.sh (GH-26103)
   - export DISPLAY=":99.0"
   - echo "sh -e /etc/init.d/xvfb start"
-  - sh -e /etc/init.d/xvfb start
+  - if [ "$JOB" != "3.8-dev" ]; then sh -e /etc/init.d/xvfb start; fi
   - sleep 3
 
 script:
   - echo "script start"
-  - source activate pandas-dev
+  - echo "$JOB"
+  - if [ "$JOB" != "3.8-dev" ]; then source activate pandas-dev; fi
   - ci/run_tests.sh
 
 after_script:

asv_bench/benchmarks/categoricals.py

@@ -282,4 +282,18 @@ def time_sort_values(self):
         self.index.sort_values(ascending=False)
 
 
+class SearchSorted:
+    def setup(self):
+        N = 10 ** 5
+        self.ci = tm.makeCategoricalIndex(N).sort_values()
+        self.c = self.ci.values
+        self.key = self.ci.categories[1]
+
+    def time_categorical_index_contains(self):
+        self.ci.searchsorted(self.key)
+
+    def time_categorical_contains(self):
+        self.c.searchsorted(self.key)
+
+
 from .pandas_vb_common import setup  # noqa: F401 isort:skip

asv_bench/benchmarks/ctors.py

@@ -67,7 +67,7 @@ class SeriesConstructors:
     def setup(self, data_fmt, with_index, dtype):
         if data_fmt in (gen_of_str, gen_of_tuples) and with_index:
             raise NotImplementedError(
-                "Series constructors do not support " "using generators with indexes"
+                "Series constructors do not support using generators with indexes"
             )
         N = 10 ** 4
         if dtype == "float":

asv_bench/benchmarks/eval.py

@@ -27,7 +27,7 @@ def time_add(self, engine, threads):
 
     def time_and(self, engine, threads):
         pd.eval(
-            "(self.df > 0) & (self.df2 > 0) & " "(self.df3 > 0) & (self.df4 > 0)",
+            "(self.df > 0) & (self.df2 > 0) & (self.df3 > 0) & (self.df4 > 0)",
             engine=engine,
         )
 

asv_bench/benchmarks/io/hdf.py

@@ -88,11 +88,11 @@ def time_write_store_table_dc(self):
 
     def time_query_store_table_wide(self):
         self.store.select(
-            "table_wide", where="index > self.start_wide and " "index < self.stop_wide"
+            "table_wide", where="index > self.start_wide and index < self.stop_wide"
         )
 
     def time_query_store_table(self):
-        self.store.select("table", where="index > self.start and " "index < self.stop")
+        self.store.select("table", where="index > self.start and index < self.stop")
 
     def time_store_repr(self):
         repr(self.store)

asv_bench/benchmarks/join_merge.py

@@ -273,10 +273,10 @@ def time_merge_ordered(self):
 
 
 class MergeAsof:
-    params = [["backward", "forward", "nearest"]]
-    param_names = ["direction"]
+    params = [["backward", "forward", "nearest"], [None, 5]]
+    param_names = ["direction", "tolerance"]
 
-    def setup(self, direction):
+    def setup(self, direction, tolerance):
         one_count = 200000
         two_count = 1000000
 
@@ -303,6 +303,9 @@ def setup(self, direction):
         df1["time32"] = np.int32(df1.time)
         df2["time32"] = np.int32(df2.time)
 
+        df1["timeu64"] = np.uint64(df1.time)
+        df2["timeu64"] = np.uint64(df2.time)
+
         self.df1a = df1[["time", "value1"]]
         self.df2a = df2[["time", "value2"]]
         self.df1b = df1[["time", "key", "value1"]]
@@ -313,22 +316,52 @@ def setup(self, direction):
         self.df2d = df2[["time32", "value2"]]
         self.df1e = df1[["time", "key", "key2", "value1"]]
         self.df2e = df2[["time", "key", "key2", "value2"]]
+        self.df1f = df1[["timeu64", "value1"]]
+        self.df2f = df2[["timeu64", "value2"]]
+
+    def time_on_int(self, direction, tolerance):
+        merge_asof(
+            self.df1a, self.df2a, on="time", direction=direction, tolerance=tolerance
+        )
 
-    def time_on_int(self, direction):
-        merge_asof(self.df1a, self.df2a, on="time", direction=direction)
+    def time_on_int32(self, direction, tolerance):
+        merge_asof(
+            self.df1d, self.df2d, on="time32", direction=direction, tolerance=tolerance
+        )
 
-    def time_on_int32(self, direction):
-        merge_asof(self.df1d, self.df2d, on="time32", direction=direction)
+    def time_on_uint64(self, direction, tolerance):
+        merge_asof(
+            self.df1f, self.df2f, on="timeu64", direction=direction, tolerance=tolerance
+        )
 
-    def time_by_object(self, direction):
-        merge_asof(self.df1b, self.df2b, on="time", by="key", direction=direction)
+    def time_by_object(self, direction, tolerance):
+        merge_asof(
+            self.df1b,
+            self.df2b,
+            on="time",
+            by="key",
+            direction=direction,
+            tolerance=tolerance,
+        )
 
-    def time_by_int(self, direction):
-        merge_asof(self.df1c, self.df2c, on="time", by="key2", direction=direction)
+    def time_by_int(self, direction, tolerance):
+        merge_asof(
+            self.df1c,
+            self.df2c,
+            on="time",
+            by="key2",
+            direction=direction,
+            tolerance=tolerance,
+        )
 
-    def time_multiby(self, direction):
+    def time_multiby(self, direction, tolerance):
         merge_asof(
-            self.df1e, self.df2e, on="time", by=["key", "key2"], direction=direction
+            self.df1e,
+            self.df2e,
+            on="time",
+            by=["key", "key2"],
+            direction=direction,
+            tolerance=tolerance,
         )
 
 

ci/build38.sh

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+# Special build for python3.8 until numpy puts its own wheels up
+
+sudo apt-get install build-essential gcc xvfb
+pip install --no-deps -U pip wheel setuptools
+pip install python-dateutil pytz cython pytest pytest-xdist hypothesis
+
+# Possible alternative for getting numpy:
+pip install --pre -f https://7933911d6844c6c53a7d-47bd50c35cd79bd838daf386af554a83.ssl.cf2.rackcdn.com/ numpy
+
+python setup.py build_ext -inplace
+python -m pip install --no-build-isolation -e .
+
+python -c "import sys; print(sys.version_info)"
+python -c "import pandas as pd"
+python -c "import hypothesis"
+
+# TODO: Is there anything else in setup_env that we really want to do?
+# ci/setup_env.sh

ci/code_checks.sh

@@ -122,7 +122,11 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     MSG='Check for non-standard imports' ; echo $MSG
     invgrep -R --include="*.py*" -E "from pandas.core.common import " pandas
     invgrep -R --include="*.py*" -E "from collections.abc import " pandas
-    # invgrep -R --include="*.py*" -E "from numpy import nan " pandas  # GH#24822 not yet implemented since the offending imports have not all been removed
+    invgrep -R --include="*.py*" -E "from numpy import nan " pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
+    MSG='Check for use of exec' ; echo $MSG
+    invgrep -R --include="*.py*" -E "[^a-zA-Z0-9_]exec\(" pandas
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
     MSG='Check for pytest warns' ; echo $MSG
@@ -184,7 +188,7 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -R --include="*.rst" ".. ipython ::" doc/source
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
-    MSG='Check that no file in the repo contains tailing whitespaces' ; echo $MSG
+    MSG='Check that no file in the repo contains trailing whitespaces' ; echo $MSG
     set -o pipefail
     if [[ "$AZURE" == "true" ]]; then
         # we exclude all c/cpp files as the c/cpp files of pandas code base are tested when Linting .c and .h files
@@ -262,13 +266,17 @@ if [[ -z "$CHECK" || "$CHECK" == "doctests" ]]; then
         -k"-from_arrays -from_breaks -from_intervals -from_tuples -set_closed -to_tuples -interval_range"
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Doctests arrays/string_.py' ; echo $MSG
+    pytest -q --doctest-modules pandas/core/arrays/string_.py
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
 fi
 
 ### DOCSTRINGS ###
 if [[ -z "$CHECK" || "$CHECK" == "docstrings" ]]; then
 
-    MSG='Validate docstrings (GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS04, SS05, PR03, PR04, PR05, PR10, EX04, RT01, RT04, RT05, SA05)' ; echo $MSG
-    $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS04,SS05,PR03,PR04,PR05,PR10,EX04,RT01,RT04,RT05,SA05
+    MSG='Validate docstrings (GL03, GL04, GL05, GL06, GL07, GL09, GL10, SS04, SS05, PR03, PR04, PR05, PR10, EX04, RT01, RT04, RT05, SA01, SA02, SA03, SA05)' ; echo $MSG
+    $BASE_DIR/scripts/validate_docstrings.py --format=azure --errors=GL03,GL04,GL05,GL06,GL07,GL09,GL10,SS04,SS05,PR03,PR04,PR05,PR10,EX04,RT01,RT04,RT05,SA01,SA02,SA03,SA05
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
 fi

ci/deps/azure-36-32bit.yaml

@@ -3,6 +3,7 @@ channels:
   - defaults
   - conda-forge
 dependencies:
+  - attrs=19.1.0
   - gcc_linux-32
   - gcc_linux-32
   - gxx_linux-32
@@ -11,7 +12,7 @@ dependencies:
   - python=3.6.*
   - pytz=2017.2
   # universal
-  - pytest>=4.0.2,<5.0.0
+  - pytest
   - pytest-xdist
   - pytest-mock
   - pytest-azurepipelines

ci/run_tests.sh

@@ -43,10 +43,9 @@ do
     # if no tests are found (the case of "single and slow"), pytest exits with code 5, and would make the script fail, if not for the below code
     sh -c "$PYTEST_CMD; ret=\$?; [ \$ret = 5 ] && exit 0 || exit \$ret"
 
-    # 2019-08-21 disabling because this is hitting HTTP 400 errors GH#27602
-    # if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then
-    #    echo "uploading coverage for $TYPE tests"
-    #    echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME"
-    #          bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME
-    # fi
+    if [[ "$COVERAGE" && $? == 0 && "$TRAVIS_BRANCH" == "master" ]]; then
+        echo "uploading coverage for $TYPE tests"
+        echo "bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME"
+              bash <(curl -s https://codecov.io/bash) -Z -c -F $TYPE -f $COVERAGE_FNAME
+    fi
 done

ci/setup_env.sh

@@ -1,5 +1,9 @@
 #!/bin/bash -e
 
+if [ "$JOB" == "3.8-dev" ]; then
+    /bin/bash ci/build38.sh
+    exit 0
+fi
 
 # edit the locale file if needed
 if [ -n "$LOCALE_OVERRIDE" ]; then
@@ -51,6 +55,7 @@ echo
 echo "update conda"
 conda config --set ssl_verify false
 conda config --set quiet true --set always_yes true --set changeps1 false
+conda install pip  # create conda to create a historical artifact for pip & setuptools
 conda update -n base conda
 
 echo "conda info -a"

doc/.gitignore

@@ -0,0 +1,4 @@
+data/
+timeseries.csv
+timeseries.parquet
+timeseries_wide.parquet

doc/redirects.csv

@@ -6,6 +6,7 @@ whatsnew,whatsnew/index
 release,whatsnew/index
 
 # getting started
+install,getting_started/install
 10min,getting_started/10min
 basics,getting_started/basics
 comparison_with_r,getting_started/comparison/comparison_with_r
@@ -1577,3 +1578,6 @@ generated/pandas.unique,../reference/api/pandas.unique
 generated/pandas.util.hash_array,../reference/api/pandas.util.hash_array
 generated/pandas.util.hash_pandas_object,../reference/api/pandas.util.hash_pandas_object
 generated/pandas.wide_to_long,../reference/api/pandas.wide_to_long
+
+# Cached searches
+reference/api/pandas.DataFrame.from_csv,pandas.read_csv

doc/source/conf.py

@@ -120,6 +120,9 @@
 plot_pre_code = """import numpy as np
 import pandas as pd"""
 
+# nbsphinx do not use requirejs (breaks bootstrap)
+nbsphinx_requirejs_path = ""
+
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ["../_templates"]
 
@@ -191,7 +194,7 @@
 
 # The theme to use for HTML and HTML Help pages.  Major themes that come with
 # Sphinx are currently 'default' and 'sphinxdoc'.
-html_theme = "nature_with_gtoc"
+html_theme = "pandas_sphinx_theme"
 
 # The style sheet to use for HTML and HTML Help pages. A file of that name
 # must exist either in Sphinx' static/ path, or in one of the custom paths
@@ -204,7 +207,7 @@
 # html_theme_options = {}
 
 # Add any paths that contain custom themes here, relative to this directory.
-html_theme_path = ["themes"]
+# html_theme_path = ["themes"]
 
 # The name for this set of Sphinx documents.  If None, it defaults to
 # "<project> v<release> documentation".
@@ -628,11 +631,11 @@ def linkcode_resolve(domain, info):
     fn = os.path.relpath(fn, start=os.path.dirname(pandas.__file__))
 
     if "+" in pandas.__version__:
-        return "http://github.com/pandas-dev/pandas/blob/master/pandas/" "{}{}".format(
+        return "http://github.com/pandas-dev/pandas/blob/master/pandas/{}{}".format(
             fn, linespec
         )
     else:
-        return "http://github.com/pandas-dev/pandas/blob/" "v{}/pandas/{}{}".format(
+        return "http://github.com/pandas-dev/pandas/blob/v{}/pandas/{}{}".format(
             pandas.__version__, fn, linespec
         )
 

doc/source/development/contributing.rst

@@ -172,7 +172,7 @@ installed (or you wish to install a newer version) you can install a compiler
     yum groupinstall "Development Tools"
 
 For other Linux distributions, consult your favourite search engine for
-commpiler installation instructions.
+compiler installation instructions.
 
 Let us know if you have any difficulties by opening an issue or reaching out on
 `Gitter`_.
@@ -949,10 +949,13 @@ the expected correct result::
 
         assert_frame_equal(pivoted, expected)
 
+Please remember to add the Github Issue Number as a comment to a new test.
+E.g. "# brief comment, see GH#28907"
+
 Transitioning to ``pytest``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-*pandas* existing test structure is *mostly* classed based, meaning that you will typically find tests wrapped in a class.
+*pandas* existing test structure is *mostly* class-based, meaning that you will typically find tests wrapped in a class.
 
 .. code-block:: python
 

doc/source/ecosystem.rst

@@ -1,3 +1,5 @@
+:orphan:
+
 .. _ecosystem:
 
 {{ header }}