Merge branch 'master' into feature/groupby-repr-ellipses-1135

Author: benjaminarjun

.gitignore

@@ -61,6 +61,7 @@ dist
 .coverage
 coverage.xml
 coverage_html_report
+.mypy_cache
 *.pytest_cache
 # hypothesis test database
 .hypothesis/

.travis.yml

@@ -1,6 +1,5 @@
 sudo: false
 language: python
-# Default Python version is usually 2.7
 python: 3.5
 
 # To turn off cached cython files and compiler cache
@@ -36,21 +35,13 @@ matrix:
       env:
         - JOB="3.7" ENV_FILE="ci/deps/travis-37.yaml" PATTERN="(not slow and not network)"
 
-    - dist: trusty
-      env:
-        - JOB="2.7" ENV_FILE="ci/deps/travis-27.yaml" PATTERN="(not slow or (single and db))"
-      addons:
-        apt:
-          packages:
-          - python-gtk2
-
     - dist: trusty
       env:
         - JOB="3.6, locale" ENV_FILE="ci/deps/travis-36-locale.yaml" PATTERN="((not slow and not network) or (single and db))" LOCALE_OVERRIDE="zh_CN.UTF-8"
 
     - dist: trusty
       env:
-        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true
+        - JOB="3.6, coverage" ENV_FILE="ci/deps/travis-36-cov.yaml" PATTERN="((not slow and not network) or (single and db))" PANDAS_TESTING_MODE="deprecate" COVERAGE=true
 
     # In allow_failures
     - dist: trusty

README.md

@@ -164,9 +164,9 @@ pip install pandas
 ```
 
 ## Dependencies
-- [NumPy](https://www.numpy.org): 1.12.0 or higher
+- [NumPy](https://www.numpy.org): 1.13.3 or higher
 - [python-dateutil](https://labix.org/python-dateutil): 2.5.0 or higher
-- [pytz](https://pythonhosted.org/pytz): 2011k or higher
+- [pytz](https://pythonhosted.org/pytz): 2015.4 or higher
 
 See the [full installation instructions](https://pandas.pydata.org/pandas-docs/stable/install.html#dependencies)
 for recommended and optional dependencies.

asv_bench/benchmarks/io/stata.py

@@ -12,12 +12,12 @@ class Stata(BaseIO):
 
     def setup(self, convert_dates):
         self.fname = '__test__.dta'
-        N = 100000
-        C = 5
+        N = self.N = 100000
+        C = self.C = 5
         self.df = DataFrame(np.random.randn(N, C),
                             columns=['float{}'.format(i) for i in range(C)],
                             index=date_range('20000101', periods=N, freq='H'))
-        self.df['object'] = tm.makeStringIndex(N)
+        self.df['object'] = tm.makeStringIndex(self.N)
         self.df['int8_'] = np.random.randint(np.iinfo(np.int8).min,
                                              np.iinfo(np.int8).max - 27, N)
         self.df['int16_'] = np.random.randint(np.iinfo(np.int16).min,
@@ -36,4 +36,14 @@ def time_write_stata(self, convert_dates):
         self.df.to_stata(self.fname, self.convert_dates)
 
 
+class StataMissing(Stata):
+    def setup(self, convert_dates):
+        super(StataMissing, self).setup(convert_dates)
+        for i in range(10):
+            missing_data = np.random.randn(self.N)
+            missing_data[missing_data < 0] = np.nan
+            self.df['missing_{0}'.format(i)] = missing_data
+        self.df.to_stata(self.fname, self.convert_dates)
+
+
 from ..pandas_vb_common import setup  # noqa: F401

azure-pipelines.yml

@@ -10,7 +10,6 @@ jobs:
     name: Linux
     vmImage: ubuntu-16.04
 
-# Windows Python 2.7 needs VC 9.0 installed, handled in the template
 - template: ci/azure/windows.yml
   parameters:
     name: Windows
@@ -89,6 +88,13 @@ jobs:
     displayName: 'Docstring validation'
     condition: true
 
+  - script: |
+      export PATH=$HOME/miniconda3/bin:$PATH
+      source activate pandas-dev
+      ci/code_checks.sh typing
+    displayName: 'Typing validation'
+    condition: true
+
   - script: |
       export PATH=$HOME/miniconda3/bin:$PATH
       source activate pandas-dev

ci/azure/posix.yml

@@ -9,20 +9,20 @@ jobs:
   strategy:
     matrix:
       ${{ if eq(parameters.name, 'macOS') }}:
-        py35_np_120:
+        py35_macos:
           ENV_FILE: ci/deps/azure-macos-35.yaml
           CONDA_PY: "35"
           PATTERN: "not slow and not network"
 
       ${{ if eq(parameters.name, 'Linux') }}:
-        py27_np_120:
-          ENV_FILE: ci/deps/azure-27-compat.yaml
-          CONDA_PY: "27"
+        py35_compat:
+          ENV_FILE: ci/deps/azure-35-compat.yaml
+          CONDA_PY: "35"
           PATTERN: "not slow and not network"
 
-        py27_locale_slow_old_np:
-          ENV_FILE: ci/deps/azure-27-locale.yaml
-          CONDA_PY: "27"
+        py36_locale_slow_old_np:
+          ENV_FILE: ci/deps/azure-36-locale.yaml
+          CONDA_PY: "36"
           PATTERN: "slow"
           LOCALE_OVERRIDE: "zh_CN.UTF-8"
           EXTRA_APT: "language-pack-zh-hans"

ci/azure/windows.yml

@@ -8,27 +8,20 @@ jobs:
     vmImage: ${{ parameters.vmImage }}
   strategy:
     matrix:
-      py36_np14:
+      py36_np15:
         ENV_FILE: ci/deps/azure-windows-36.yaml
         CONDA_PY: "36"
 
-      py27_np121:
-        ENV_FILE: ci/deps/azure-windows-27.yaml
-        CONDA_PY: "27"
+      py37_np141:
+        ENV_FILE: ci/deps/azure-windows-37.yaml
+        CONDA_PY: "37"
 
   steps:
     - task: CondaEnvironment@1
       inputs:
         updateConda: no
         packageSpecs: ''
 
-    - powershell: |
-        $wc = New-Object net.webclient
-        $wc.Downloadfile("https://download.microsoft.com/download/7/9/6/796EF2E4-801B-4FC4-AB28-B59FBF6D907B/VCForPython27.msi", "VCForPython27.msi")
-        Start-Process "VCForPython27.msi" /qn -Wait
-      displayName: 'Install VC 9.0 only for Python 2.7'
-      condition: eq(variables.CONDA_PY, '27')
-
     - script: |
         ci\\incremental\\setup_conda_environment.cmd
       displayName: 'Before Install'

ci/code_checks.sh

@@ -16,9 +16,10 @@
 #   $ ./ci/code_checks.sh doctests      # run doctests
 #   $ ./ci/code_checks.sh docstrings    # validate docstring errors
 #   $ ./ci/code_checks.sh dependencies  # check that dependencies are consistent
+#   $ ./ci/code_checks.sh typing	# run static type analysis
 
-[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "dependencies" ]] || \
-    { echo "Unknown command $1. Usage: $0 [lint|patterns|code|doctests|docstrings|dependencies]"; exit 9999; }
+[[ -z "$1" || "$1" == "lint" || "$1" == "patterns" || "$1" == "code" || "$1" == "doctests" || "$1" == "docstrings" || "$1" == "dependencies" || "$1" == "typing" ]] || \
+    { echo "Unknown command $1. Usage: $0 [lint|patterns|code|doctests|docstrings|dependencies|typing]"; exit 9999; }
 
 BASE_DIR="$(dirname $0)/.."
 RET=0
@@ -119,6 +120,10 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -r -E --include '*.py' 'pytest\.warns' pandas/tests/
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Check for pytest raises without context' ; echo $MSG
+    invgrep -r -E --include '*.py' "[[:space:]] pytest.raises" pandas/tests/
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     # Check for the following code in testing: `np.testing` and `np.array_equal`
     MSG='Check for invalid testing' ; echo $MSG
     invgrep -r -E --include '*.py' --exclude testing.py '(numpy|np)(\.testing|\.array_equal)' pandas/tests/
@@ -175,9 +180,9 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     set -o pipefail
     if [[ "$AZURE" == "true" ]]; then
         # we exclude all c/cpp files as the c/cpp files of pandas code base are tested when Linting .c and .h files
-        ! grep -n '--exclude=*.'{svg,c,cpp,html} -RI "\s$" * | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Tailing whitespaces found: " $3}'
+        ! grep -n '--exclude=*.'{svg,c,cpp,html} --exclude-dir=env -RI "\s$" * | awk -F ":" '{print "##vso[task.logissue type=error;sourcepath=" $1 ";linenumber=" $2 ";] Tailing whitespaces found: " $3}'
     else
-        ! grep -n '--exclude=*.'{svg,c,cpp,html}  -RI "\s$" * | awk -F ":" '{print $1 ":" $2 ":Tailing whitespaces found: " $3}'
+        ! grep -n '--exclude=*.'{svg,c,cpp,html} --exclude-dir=env -RI "\s$" * | awk -F ":" '{print $1 ":" $2 ":Tailing whitespaces found: " $3}'
     fi
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 fi
@@ -256,4 +261,16 @@ if [[ -z "$CHECK" || "$CHECK" == "dependencies" ]]; then
 
 fi
 
+### TYPING ###
+if [[ -z "$CHECK" || "$CHECK" == "typing" ]]; then
+
+    echo "mypy --version"
+    mypy --version
+
+    MSG='Performing static analysis using mypy' ; echo $MSG
+    mypy pandas
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+fi
+
+
 exit $RET

ci/deps/azure-27-compat.yaml

@@ -0,0 +1,29 @@
+name: pandas-dev
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - beautifulsoup4=4.4.1
+  - bottleneck=1.2.1
+  - jinja2=2.8
+  - numexpr=2.6.2
+  - numpy=1.13.3
+  - openpyxl=2.4.0
+  - pytables=3.4.2
+  - python-dateutil=2.5.0
+  - python=3.5.*
+  - pytz=2015.4
+  - scipy=0.19.0
+  - xlrd=1.0.0
+  - xlsxwriter=0.7.7
+  - xlwt=1.0.0
+  # universal
+  - cython=0.28.2
+  - hypothesis>=3.58.0
+  - pytest-xdist
+  - pytest-mock
+  - isort
+  - pip:
+    # for python 3.5, pytest>=4.0.2 is not available in conda
+    - pytest>=4.0.2
+    - html5lib==1.0b2

ci/deps/azure-35-compat.yaml

@@ -3,22 +3,22 @@ channels:
   - defaults
   - conda-forge
 dependencies:
-  - bottleneck=1.2.0
+  - beautifulsoup4==4.5.1
+  - bottleneck=1.2.*
   - cython=0.28.2
   - lxml
-  - matplotlib=2.0.0
-  - numpy=1.12.0
+  - matplotlib=2.2.2
+  - numpy=1.14.*
   - openpyxl=2.4.0
   - python-dateutil
   - python-blosc
-  - python=2.7
-  - pytz
-  - pytz=2013b
+  - python=3.6.*
+  - pytz=2016.10
   - scipy
-  - sqlalchemy=0.8.1
+  - sqlalchemy=1.1.4
   - xlrd=1.0.0
-  - xlsxwriter=0.5.2
-  - xlwt=0.7.5
+  - xlsxwriter=0.9.4
+  - xlwt=1.2.0
   # universal
   - pytest>=4.0.2
   - pytest-xdist
@@ -27,4 +27,3 @@ dependencies:
   - isort
   - pip:
     - html5lib==1.0b2
-    - beautifulsoup4==4.2.1

ci/deps/azure-27-locale.yaml renamed to ci/deps/azure-36-locale.yaml

@@ -10,14 +10,14 @@ dependencies:
   - ipython
   - jinja2
   - lxml
-  - matplotlib
+  - matplotlib=3.0.*
   - nomkl
   - numexpr
-  - numpy
+  - numpy=1.15.*
   - openpyxl
   - pytables
   - python-dateutil
-  - python=3.6*
+  - python=3.6.*
   - pytz
   - s3fs
   - scipy

ci/deps/azure-36-locale_slow.yaml

@@ -16,7 +16,7 @@ dependencies:
   - openpyxl
   - pytables
   - python-dateutil
-  - python=3.7*
+  - python=3.7.*
   - pytz
   - s3fs
   - scipy

ci/deps/azure-37-locale.yaml

@@ -2,7 +2,7 @@ name: pandas-dev
 channels:
   - defaults
 dependencies:
-  - python=3.7*
+  - python=3.7.*
   - pytz
   - Cython>=0.28.2
   # universal

ci/deps/azure-37-numpydev.yaml

@@ -8,14 +8,14 @@ dependencies:
   - html5lib
   - jinja2
   - lxml
-  - matplotlib=2.2.0
+  - matplotlib=2.2.3
   - nomkl
   - numexpr
-  - numpy=1.12.0
-  - openpyxl=2.5.5
+  - numpy=1.13.3
+  - openpyxl
   - pyarrow
   - pytables
-  - python=3.5*
+  - python=3.5.*
   - pytz
   - xarray
   - xlrd

ci/deps/azure-macos-35.yaml

@@ -7,15 +7,15 @@ dependencies:
   - bottleneck
   - boost-cpp<1.67
   - fastparquet>=0.2.1
-  - matplotlib
+  - matplotlib=3.0.2
   - numexpr
-  - numpy=1.14*
+  - numpy=1.15.*
   - openpyxl
   - parquet-cpp
   - pyarrow
   - pytables
   - python-dateutil
-  - python=3.6.6
+  - python=3.6.*
   - pytz
   - scipy
   - xlrd