Merge branch 'master' into ndarray_tolerance

Author: buntwo

.github/ISSUE_TEMPLATE.md

@@ -8,11 +8,15 @@
 
 [this should explain **why** the current behaviour is a problem and why the expected output is a better solution.]
 
-**Note**: We receive a lot of issues on our GitHub tracker, so it is very possible that your issue has been posted before.
-Please check first before submitting so that we do not have to handle and close duplicates!
+**Note**: We receive a lot of issues on our GitHub tracker, so it is very possible that your issue has been posted before. Please check first before submitting so that we do not have to handle and close duplicates!
 
-**Note**: Many problems can be resolved by simply upgrading `pandas` to the latest version. Before submitting, please check
-if that solution works for you. If possible, you may want to check if `master` addresses this issue, but that is not necessary.
+**Note**: Many problems can be resolved by simply upgrading `pandas` to the latest version. Before submitting, please check if that solution works for you. If possible, you may want to check if `master` addresses this issue, but that is not necessary.
+
+For documentation-related issues, you can check the latest versions of the docs on `master` here:
+
+https://pandas-docs.github.io/pandas-docs-travis/
+
+If the issue has not been resolved there, go ahead and file it in the issue tracker.
 
 #### Expected Output
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,4 +1,4 @@
- - [ ] closes #xxxx
- - [ ] tests added / passed
- - [ ] passes ``git diff upstream/master -u -- "*.py" | flake8 --diff``
- - [ ] whatsnew entry
+- [ ] closes #xxxx
+- [ ] tests added / passed
+- [ ] passes `git diff upstream/master -u -- "*.py" | flake8 --diff`
+- [ ] whatsnew entry

.travis.yml

@@ -37,7 +37,7 @@ matrix:
         - JOB="3.5_OSX" TEST_ARGS="--skip-slow --skip-network"
     - dist: trusty
       env:
-        - JOB="2.7_LOCALE" TEST_ARGS="--only-slow --skip-network" LOCALE_OVERRIDE="zh_CN.UTF-8"
+        - JOB="2.7_LOCALE" LOCALE_OVERRIDE="zh_CN.UTF-8" SLOW=true
       addons:
         apt:
           packages:
@@ -62,7 +62,7 @@ matrix:
     # In allow_failures
     - dist: trusty
       env:
-        - JOB="2.7_SLOW" TEST_ARGS="--only-slow --skip-network"
+        - JOB="2.7_SLOW" SLOW=true
     # In allow_failures
     - dist: trusty
       env:
@@ -82,7 +82,7 @@ matrix:
     allow_failures:
       - dist: trusty
         env:
-          - JOB="2.7_SLOW" TEST_ARGS="--only-slow --skip-network"
+          - JOB="2.7_SLOW" SLOW=true
       - dist: trusty
         env:
           - JOB="2.7_BUILD_TEST" TEST_ARGS="--skip-slow" BUILD_TEST=true

appveyor.yml

@@ -72,7 +72,7 @@ install:
   - cmd: conda info -a
 
   # create our env
-  - cmd: conda create -n pandas python=%PYTHON_VERSION% cython pytest pytest-xdist
+  - cmd: conda create -n pandas python=%PYTHON_VERSION% cython pytest>=3.1.0 pytest-xdist
   - cmd: activate pandas
   - SET REQ=ci\requirements-%PYTHON_VERSION%_WIN.run
   - cmd: echo "installing requirements from %REQ%"

asv_bench/asv.conf.json

@@ -117,8 +117,10 @@
     // with results. If the commit is `null`, regression detection is
     // skipped for the matching benchmark.
     //
-    // "regressions_first_commits": {
-    //    "some_benchmark": "352cdf",  // Consider regressions only after this commit
-    //    "another_benchmark": null,   // Skip regression detection altogether
-    // }
+    "regressions_first_commits": {
+        ".*": "v0.20.0"
+    },
+    "regression_thresholds": {
+        ".*": 0.05
+    }
 }

asv_bench/benchmarks/stat_ops.py

@@ -1,92 +1,36 @@
 from .pandas_vb_common import *
 
 
-class stat_ops_frame_mean_float_axis_0(object):
-    goal_time = 0.2
-
-    def setup(self):
-        self.df = DataFrame(np.random.randn(100000, 4))
-        self.dfi = DataFrame(np.random.randint(1000, size=self.df.shape))
-
-    def time_stat_ops_frame_mean_float_axis_0(self):
-        self.df.mean()
-
-
-class stat_ops_frame_mean_float_axis_1(object):
-    goal_time = 0.2
-
-    def setup(self):
-        self.df = DataFrame(np.random.randn(100000, 4))
-        self.dfi = DataFrame(np.random.randint(1000, size=self.df.shape))
-
-    def time_stat_ops_frame_mean_float_axis_1(self):
-        self.df.mean(1)
-
-
-class stat_ops_frame_mean_int_axis_0(object):
-    goal_time = 0.2
-
-    def setup(self):
-        self.df = DataFrame(np.random.randn(100000, 4))
-        self.dfi = DataFrame(np.random.randint(1000, size=self.df.shape))
-
-    def time_stat_ops_frame_mean_int_axis_0(self):
-        self.dfi.mean()
-
-
-class stat_ops_frame_mean_int_axis_1(object):
-    goal_time = 0.2
+def _set_use_bottleneck_False():
+    try:
+        pd.options.compute.use_bottleneck = False
+    except:
+        from pandas.core import nanops
+        nanops._USE_BOTTLENECK = False
 
-    def setup(self):
-        self.df = DataFrame(np.random.randn(100000, 4))
-        self.dfi = DataFrame(np.random.randint(1000, size=self.df.shape))
-
-    def time_stat_ops_frame_mean_int_axis_1(self):
-        self.dfi.mean(1)
-
-
-class stat_ops_frame_sum_float_axis_0(object):
-    goal_time = 0.2
 
-    def setup(self):
-        self.df = DataFrame(np.random.randn(100000, 4))
-        self.dfi = DataFrame(np.random.randint(1000, size=self.df.shape))
-
-    def time_stat_ops_frame_sum_float_axis_0(self):
-        self.df.sum()
-
-
-class stat_ops_frame_sum_float_axis_1(object):
+class FrameOps(object):
     goal_time = 0.2
 
-    def setup(self):
-        self.df = DataFrame(np.random.randn(100000, 4))
-        self.dfi = DataFrame(np.random.randint(1000, size=self.df.shape))
+    param_names = ['op', 'use_bottleneck', 'dtype', 'axis']
+    params = [['mean', 'sum', 'median'],
+              [True, False],
+              ['float', 'int'],
+              [0, 1]]
 
-    def time_stat_ops_frame_sum_float_axis_1(self):
-        self.df.sum(1)
+    def setup(self, op, use_bottleneck, dtype, axis):
+        if dtype == 'float':
+            self.df = DataFrame(np.random.randn(100000, 4))
+        elif dtype == 'int':
+            self.df = DataFrame(np.random.randint(1000, size=(100000, 4)))
 
+        if not use_bottleneck:
+            _set_use_bottleneck_False()
 
-class stat_ops_frame_sum_int_axis_0(object):
-    goal_time = 0.2
-
-    def setup(self):
-        self.df = DataFrame(np.random.randn(100000, 4))
-        self.dfi = DataFrame(np.random.randint(1000, size=self.df.shape))
-
-    def time_stat_ops_frame_sum_int_axis_0(self):
-        self.dfi.sum()
-
-
-class stat_ops_frame_sum_int_axis_1(object):
-    goal_time = 0.2
-
-    def setup(self):
-        self.df = DataFrame(np.random.randn(100000, 4))
-        self.dfi = DataFrame(np.random.randint(1000, size=self.df.shape))
+        self.func = getattr(self.df, op)
 
-    def time_stat_ops_frame_sum_int_axis_1(self):
-        self.dfi.sum(1)
+    def time_op(self, op, use_bottleneck, dtype, axis):
+        self.func(axis=axis)
 
 
 class stat_ops_level_frame_sum(object):

asv_bench/benchmarks/timeseries.py

@@ -510,3 +510,17 @@ def time_begin_incr_rng(self):
 
     def time_begin_decr_rng(self):
         self.rng - self.semi_month_begin
+
+
+class DatetimeAccessor(object):
+    def setup(self):
+        self.N = 100000
+        self.series = pd.Series(
+            pd.date_range(start='1/1/2000', periods=self.N, freq='T')
+        )
+
+    def time_dt_accessor(self):
+        self.series.dt
+
+    def time_dt_accessor_normalize(self):
+        self.series.dt.normalize()

ci/install_circle.sh

@@ -64,7 +64,7 @@ fi
 # create envbuild deps
 echo "[create env: ${REQ_BUILD}]"
 time conda create -n pandas -q --file=${REQ_BUILD} || exit 1
-time conda install -n pandas pytest || exit 1
+time conda install -n pandas pytest>=3.1.0 || exit 1
 
 source activate pandas
 

ci/install_travis.sh

@@ -47,7 +47,7 @@ which conda
 echo
 echo "[update conda]"
 conda config --set ssl_verify false || exit 1
-conda config --set always_yes true --set changeps1 false || exit 1
+conda config --set quiet true --set always_yes true --set changeps1 false || exit 1
 conda update -q conda
 
 echo
@@ -103,7 +103,7 @@ if [ -e ${REQ} ]; then
     time bash $REQ || exit 1
 fi
 
-time conda install -n pandas pytest
+time conda install -n pandas pytest>=3.1.0
 time pip install pytest-xdist
 
 if [ "$LINT" ]; then

ci/requirements-2.7_COMPAT.build

@@ -1,5 +1,5 @@
 python=2.7*
-numpy=1.7.1
+numpy=1.9.2
 cython=0.23
 dateutil=1.5
 pytz=2013b

ci/requirements-2.7_COMPAT.run

@@ -1,11 +1,12 @@
-numpy=1.7.1
+numpy=1.9.2
 dateutil=1.5
 pytz=2013b
-scipy=0.11.0
+scipy=0.14.0
 xlwt=0.7.5
 xlrd=0.9.2
-numexpr=2.2.2
-pytables=3.0.0
+bottleneck=1.0.0
+numexpr=2.4.4 # we test that we correctly don't use an unsupported numexpr
+pytables=3.2.2
 psycopg2
 pymysql=0.6.0
 sqlalchemy=0.7.8

ci/requirements-2.7_LOCALE.build

@@ -1,5 +1,5 @@
 python=2.7*
 python-dateutil
 pytz=2013b
-numpy=1.8.2
+numpy=1.9.2
 cython=0.23

ci/requirements-2.7_LOCALE.run

@@ -1,11 +1,12 @@
 python-dateutil
 pytz=2013b
-numpy=1.8.2
+numpy=1.9.2
 xlwt=0.7.5
 openpyxl=1.6.2
 xlsxwriter=0.5.2
 xlrd=0.9.2
-matplotlib=1.3.1
+bottleneck=1.0.0
+matplotlib=1.4.3
 sqlalchemy=0.8.1
 lxml=3.2.1
 scipy

ci/requirements-2.7_SLOW.build

@@ -1,5 +1,5 @@
 python=2.7*
 python-dateutil
 pytz
-numpy=1.8.2
+numpy=1.10*
 cython

ci/requirements-2.7_SLOW.run

@@ -1,7 +1,7 @@
 python-dateutil
 pytz
-numpy=1.8.2
-matplotlib=1.3.1
+numpy=1.10*
+matplotlib=1.4.3
 scipy
 patsy
 xlwt

ci/requirements_all.txt

@@ -1,4 +1,4 @@
-pytest
+pytest>=3.1.0
 pytest-cov
 pytest-xdist
 flake8

ci/requirements_dev.txt

@@ -2,6 +2,6 @@ python-dateutil
 pytz
 numpy
 cython
-pytest
+pytest>=3.1.0
 pytest-cov
 flake8

ci/script_multi.sh

@@ -36,9 +36,15 @@ elif [ "$COVERAGE" ]; then
     echo pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
     pytest -s -n 2 -m "not single" --cov=pandas --cov-report xml:/tmp/cov-multiple.xml --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
 
+elif [ "$SLOW" ]; then
+    TEST_ARGS="--only-slow --skip-network"
+    echo pytest -r xX -m "not single and slow" -v --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
+    pytest -r xX -m "not single and slow" -v --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
+
 else
     echo pytest -n 2 -r xX -m "not single" --junitxml=/tmp/multiple.xml $TEST_ARGS pandas
     pytest -n 2 -r xX -m "not single" --junitxml=/tmp/multiple.xml $TEST_ARGS pandas # TODO: doctest
+
 fi
 
 RET="$?"

ci/script_single.sh

@@ -12,16 +12,24 @@ if [ -n "$LOCALE_OVERRIDE" ]; then
     python -c "$pycmd"
 fi
 
+if [ "$SLOW" ]; then
+    TEST_ARGS="--only-slow --skip-network"
+fi
+
 if [ "$BUILD_TEST" ]; then
     echo "We are not running pytest as this is a build test."
+
 elif [ "$DOC" ]; then
     echo "We are not running pytest as this is a doc-build"
+
 elif [ "$COVERAGE" ]; then
     echo pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
     pytest -s -m "single" --cov=pandas --cov-report xml:/tmp/cov-single.xml --junitxml=/tmp/single.xml $TEST_ARGS pandas
+
 else
     echo pytest -m "single" -r xX --junitxml=/tmp/single.xml $TEST_ARGS pandas
     pytest -m "single" -r xX  --junitxml=/tmp/single.xml $TEST_ARGS pandas # TODO: doctest
+
 fi
 
 RET="$?"

doc/source/api.rst

@@ -109,6 +109,14 @@ Feather
 
    read_feather
 
+Parquet
+~~~~~~~
+
+.. autosummary::
+   :toctree: generated/
+
+   read_parquet
+
 SAS
 ~~~