Merge in master

Author: AlexKirko

.github/workflows/ci.yml

@@ -23,53 +23,53 @@ jobs:
 
     - name: Looking for unwanted patterns
       run: ci/code_checks.sh patterns
-      if: true
+      if: always()
 
     - name: Setup environment and build pandas
       run: ci/setup_env.sh
-      if: true
+      if: always()
 
     - name: Linting
       run: |
         source activate pandas-dev
         ci/code_checks.sh lint
-      if: true
+      if: always()
 
     - name: Dependencies consistency
       run: |
         source activate pandas-dev
         ci/code_checks.sh dependencies
-      if: true
+      if: always()
 
     - name: Checks on imported code
       run: |
         source activate pandas-dev
         ci/code_checks.sh code
-      if: true
+      if: always()
 
     - name: Running doctests
       run: |
         source activate pandas-dev
         ci/code_checks.sh doctests
-      if: true
+      if: always()
 
     - name: Docstring validation
       run: |
         source activate pandas-dev
         ci/code_checks.sh docstrings
-      if: true
+      if: always()
 
     - name: Typing validation
       run: |
         source activate pandas-dev
         ci/code_checks.sh typing
-      if: true
+      if: always()
 
     - name: Testing docstring validation script
       run: |
         source activate pandas-dev
         pytest --capture=no --strict scripts
-      if: true
+      if: always()
 
     - name: Running benchmarks
       run: |
@@ -87,11 +87,73 @@ jobs:
         else
             echo "Benchmarks did not run, no changes detected"
         fi
-      if: true
+      if: always()
 
     - name: Publish benchmarks artifact
       uses: actions/upload-artifact@master
       with:
         name: Benchmarks log
         path: asv_bench/benchmarks.log
       if: failure()
+
+  web_and_docs:
+    name: Web and docs
+    runs-on: ubuntu-latest
+    steps:
+
+    - name: Setting conda path
+      run: echo "::set-env name=PATH::${HOME}/miniconda3/bin:${PATH}"
+
+    - name: Checkout
+      uses: actions/checkout@v1
+
+    - name: Setup environment and build pandas
+      run: ci/setup_env.sh
+
+    - name: Build website
+      run: |
+        source activate pandas-dev
+        python web/pandas_web.py web/pandas --target-path=web/build
+
+    - name: Build documentation
+      run: |
+        source activate pandas-dev
+        doc/make.py --warnings-are-errors | tee sphinx.log ; exit ${PIPESTATUS[0]}
+
+    # This can be removed when the ipython directive fails when there are errors,
+    # including the `tee sphinx.log` in te previous step (https://github.com/ipython/ipython/issues/11547)
+    - name: Check ipython directive errors
+      run: "! grep -B1 \"^<<<-------------------------------------------------------------------------$\" sphinx.log"
+
+    - name: Merge website and docs
+      run: |
+        mkdir -p pandas_web/docs
+        cp -r web/build/* pandas_web/
+        cp -r doc/build/html/* pandas_web/docs/
+      if: github.event_name == 'push'
+
+    - name: Install Rclone
+      run: sudo apt install rclone -y
+      if: github.event_name == 'push'
+
+    - name: Set up Rclone
+      run: |
+        RCLONE_CONFIG_PATH=$HOME/.config/rclone/rclone.conf
+        mkdir -p `dirname $RCLONE_CONFIG_PATH`
+        echo "[ovh_cloud_pandas_web]" > $RCLONE_CONFIG_PATH
+        echo "type = swift" >> $RCLONE_CONFIG_PATH
+        echo "env_auth = false" >> $RCLONE_CONFIG_PATH
+        echo "auth_version = 3" >> $RCLONE_CONFIG_PATH
+        echo "auth = https://auth.cloud.ovh.net/v3/" >> $RCLONE_CONFIG_PATH
+        echo "endpoint_type = public" >> $RCLONE_CONFIG_PATH
+        echo "tenant_domain = default" >> $RCLONE_CONFIG_PATH
+        echo "tenant = 2977553886518025" >> $RCLONE_CONFIG_PATH
+        echo "domain = default" >> $RCLONE_CONFIG_PATH
+        echo "user = w4KGs3pmDxpd" >> $RCLONE_CONFIG_PATH
+        echo "key = ${{ secrets.ovh_object_store_key }}" >> $RCLONE_CONFIG_PATH
+        echo "region = BHS" >> $RCLONE_CONFIG_PATH
+      if: github.event_name == 'push'
+
+    - name: Sync web
+      run: rclone sync pandas_web ovh_cloud_pandas_web:dev
+      if: github.event_name == 'push'

.travis.yml

@@ -48,17 +48,12 @@ matrix:
         - mysql
         - postgresql
 
-    # In allow_failures
     - env:
         - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" SQL="1"
       services:
         - mysql
         - postgresql
 
-    allow_failures:
-      - env:
-          - JOB="3.6, slow" ENV_FILE="ci/deps/travis-36-slow.yaml" PATTERN="slow" SQL="1"
-
 before_install:
   - echo "before_install"
   # set non-blocking IO on travis

ci/azure/windows.yml

@@ -34,7 +34,7 @@ jobs:
     - bash: |
         source activate pandas-dev
         conda list
-        python setup.py build_ext -q -i
+        python setup.py build_ext -q -i -j 4
         python -m pip install --no-build-isolation -e .
       displayName: 'Build'
 

ci/deps/azure-36-locale_slow.yaml

@@ -13,7 +13,7 @@ dependencies:
   - pytest-azurepipelines
 
   # pandas dependencies
-  - beautifulsoup4==4.6.0
+  - beautifulsoup4=4.6.0
   - bottleneck=1.2.*
   - lxml
   - matplotlib=2.2.2

doc/source/getting_started/10min.rst

@@ -697,8 +697,9 @@ Plotting
 
 See the :ref:`Plotting <visualization>` docs.
 
+We use the standard convention for referencing the matplotlib API:
+
 .. ipython:: python
-   :suppress:
 
    import matplotlib.pyplot as plt
    plt.close('all')

doc/source/user_guide/integer_na.rst

@@ -15,6 +15,10 @@ Nullable integer data type
    IntegerArray is currently experimental. Its API or implementation may
    change without warning.
 
+.. versionchanged:: 1.0.0
+
+   Now uses :attr:`pandas.NA` as the missing value rather
+   than :attr:`numpy.nan`.
 
 In :ref:`missing_data`, we saw that pandas primarily uses ``NaN`` to represent
 missing data. Because ``NaN`` is a float, this forces an array of integers with
@@ -23,6 +27,9 @@ much. But if your integer column is, say, an identifier, casting to float can
 be problematic. Some integers cannot even be represented as floating point
 numbers.
 
+Construction
+------------
+
 Pandas can represent integer data with possibly missing values using
 :class:`arrays.IntegerArray`. This is an :ref:`extension types <extending.extension-types>`
 implemented within pandas.
@@ -39,6 +46,12 @@ NumPy's ``'int64'`` dtype:
 
    pd.array([1, 2, np.nan], dtype="Int64")
 
+All NA-like values are replaced with :attr:`pandas.NA`.
+
+.. ipython:: python
+
+   pd.array([1, 2, np.nan, None, pd.NA], dtype="Int64")
+
 This array can be stored in a :class:`DataFrame` or :class:`Series` like any
 NumPy array.
 
@@ -78,6 +91,9 @@ with the dtype.
    In the future, we may provide an option for :class:`Series` to infer a
    nullable-integer dtype.
 
+Operations
+----------
+
 Operations involving an integer array will behave similar to NumPy arrays.
 Missing values will be propagated, and the data will be coerced to another
 dtype if needed.
@@ -123,3 +139,15 @@ Reduction and groupby operations such as 'sum' work as well.
 
    df.sum()
    df.groupby('B').A.sum()
+
+Scalar NA Value
+---------------
+
+:class:`arrays.IntegerArray` uses :attr:`pandas.NA` as its scalar
+missing value. Slicing a single element that's missing will return
+:attr:`pandas.NA`
+
+.. ipython:: python
+
+   a = pd.array([1, None], dtype="Int64")
+   a[1]

doc/source/whatsnew/v1.0.0.rst

@@ -221,8 +221,8 @@ Other enhancements
 - DataFrame constructor preserve `ExtensionArray` dtype with `ExtensionArray` (:issue:`11363`)
 - :meth:`DataFrame.sort_values` and :meth:`Series.sort_values` have gained ``ignore_index`` keyword to be able to reset index after sorting (:issue:`30114`)
 - :meth:`DataFrame.to_markdown` and :meth:`Series.to_markdown` added (:issue:`11052`)
-
 - :meth:`DataFrame.drop_duplicates` has gained ``ignore_index`` keyword to reset index (:issue:`30114`)
+- Added new writer for exporting Stata dta files in version 118, ``StataWriter118``.  This format supports exporting strings containing Unicode characters (:issue:`23573`)
 
 Build Changes
 ^^^^^^^^^^^^^
@@ -365,6 +365,64 @@ The following methods now also correctly output values for unobserved categories
 
 As a reminder, you can specify the ``dtype`` to disable all inference.
 
+:class:`arrays.IntegerArray` now uses :attr:`pandas.NA`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+:class:`arrays.IntegerArray` now uses :attr:`pandas.NA` rather than
+:attr:`numpy.nan` as its missing value marker (:issue:`29964`).
+
+*pandas 0.25.x*
+
+.. code-block:: python
+
+   >>> a = pd.array([1, 2, None], dtype="Int64")
+   >>> a
+   <IntegerArray>
+   [1, 2, NaN]
+   Length: 3, dtype: Int64
+
+   >>> a[2]
+   nan
+
+*pandas 1.0.0*
+
+.. ipython:: python
+
+   a = pd.array([1, 2, None], dtype="Int64")
+   a[2]
+
+See :ref:`missing_data.NA` for more on the differences between :attr:`pandas.NA`
+and :attr:`numpy.nan`.
+
+:class:`arrays.IntegerArray` comparisons return :class:`arrays.BooleanArray`
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Comparison operations on a :class:`arrays.IntegerArray` now returns a
+:class:`arrays.BooleanArray` rather than a NumPy array (:issue:`29964`).
+
+*pandas 0.25.x*
+
+.. code-block:: python
+
+   >>> a = pd.array([1, 2, None], dtype="Int64")
+   >>> a
+   <IntegerArray>
+   [1, 2, NaN]
+   Length: 3, dtype: Int64
+
+   >>> a > 1
+   array([False,  True, False])
+
+*pandas 1.0.0*
+
+.. ipython:: python
+
+   a = pd.array([1, 2, None], dtype="Int64")
+   a > 1
+
+Note that missing values now propagate, rather than always comparing unequal
+like :attr:`numpy.nan`. See :ref:`missing_data.NA` for more.
+
 By default :meth:`Categorical.min` now returns the minimum instead of np.nan
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
@@ -737,6 +795,7 @@ Datetimelike
 - Bug in :class:`DatetimeIndex` addition when adding a non-optimized :class:`DateOffset` incorrectly dropping timezone information (:issue:`30336`)
 - Bug in :meth:`DataFrame.drop` where attempting to drop non-existent values from a DatetimeIndex would yield a confusing error message (:issue:`30399`)
 - Bug in :meth:`DataFrame.append` would remove the timezone-awareness of new data (:issue:`30238`)
+- Bug in :meth:`Series.cummin` and :meth:`Series.cummax` with timezone-aware dtype incorrectly dropping its timezone (:issue:`15553`)
 - Bug in :class:`DatetimeArray`, :class:`TimedeltaArray`, and :class:`PeriodArray` where inplace addition and subtraction did not actually operate inplace (:issue:`24115`)
 
 Timedelta
@@ -786,6 +845,7 @@ Interval
 
 - Bug in :meth:`IntervalIndex.get_indexer` where a :class:`Categorical` or :class:`CategoricalIndex` ``target`` would incorrectly raise a ``TypeError`` (:issue:`30063`)
 - Bug in ``pandas.core.dtypes.cast.infer_dtype_from_scalar`` where passing ``pandas_dtype=True`` did not infer :class:`IntervalDtype` (:issue:`30337`)
+- Bug in :class:`IntervalDtype` where the ``kind`` attribute was incorrectly set as ``None`` instead of ``"O"`` (:issue:`30568`)
 
 Indexing
 ^^^^^^^^
@@ -834,6 +894,7 @@ I/O
 - Bug in :func:`read_json` where default encoding was not set to ``utf-8`` (:issue:`29565`)
 - Bug in :class:`PythonParser` where str and bytes were being mixed when dealing with the decimal field (:issue:`29650`)
 - :meth:`read_gbq` now accepts ``progress_bar_type`` to display progress bar while the data downloads. (:issue:`29857`)
+- Bug in :func:`pandas.io.json.json_normalize` where a missing value in the location specified by `record_path` would raise a ``TypeError`` (:issue:`30148`)
 
 Plotting
 ^^^^^^^^
@@ -849,6 +910,7 @@ Plotting
 - :func:`set_option` now validates that the plot backend provided to ``'plotting.backend'`` implements the backend when the option is set, rather than when a plot is created (:issue:`28163`)
 - :meth:`DataFrame.plot` now allow a ``backend`` keyword argument to allow changing between backends in one session (:issue:`28619`).
 - Bug in color validation incorrectly raising for non-color styles (:issue:`29122`).
+- Allow :meth: `DataFrame.plot.scatter` to plot ``objects`` and ``datetime`` type data (:issue:`18755`, :issue:`30391`)
 - Bug in :meth:`DataFrame.hist`, ``xrot=0`` does not work with ``by`` and subplots (:issue:`30288`).
 
 Groupby/resample/rolling
@@ -871,6 +933,7 @@ Groupby/resample/rolling
 - Bug in :meth:`DataFrame.groupby` when using axis=1 and having a single level columns index (:issue:`30208`)
 - Bug in :meth:`DataFrame.groupby` when using nunique on axis=1 (:issue:`30253`)
 - Bug in :meth:`GroupBy.quantile` with multiple list-like q value and integer column names (:issue:`30289`)
+- Bug in :meth:`GroupBy.pct_change` and :meth:`SeriesGroupBy.pct_change` causes ``TypeError`` when ``fill_method`` is ``None`` (:issue:`30463`)
 
 Reshaping
 ^^^^^^^^^
@@ -913,13 +976,15 @@ Other
 - Bug in :meth:`Series.diff` where a boolean series would incorrectly raise a ``TypeError`` (:issue:`17294`)
 - :meth:`Series.append` will no longer raise a ``TypeError`` when passed a tuple of ``Series`` (:issue:`28410`)
 - Fix corrupted error message when calling ``pandas.libs._json.encode()`` on a 0d array (:issue:`18878`)
+- Bug in ``pd.core.util.hashing.hash_pandas_object`` where arrays containing tuples were incorrectly treated as non-hashable (:issue:`28969`)
 - Bug in :meth:`DataFrame.append` that raised ``IndexError`` when appending with empty list (:issue:`28769`)
 - Fix :class:`AbstractHolidayCalendar` to return correct results for
   years after 2030 (now goes up to 2200) (:issue:`27790`)
 - Fixed :class:`IntegerArray` returning ``inf`` rather than ``NaN`` for operations dividing by 0 (:issue:`27398`)
 - Fixed ``pow`` operations for :class:`IntegerArray` when the other value is ``0`` or ``1`` (:issue:`29997`)
 - Bug in :meth:`Series.count` raises if use_inf_as_na is enabled (:issue:`29478`)
-- Bug in :class:`Index` where a non-hashable name could be set without raising ``TypeError`` (:issue:29069`)
+- Bug in :class:`Index` where a non-hashable name could be set without raising ``TypeError`` (:issue:`29069`)
+
 
 .. _whatsnew_1000.contributors:
 

pandas/_config/config.py

@@ -197,7 +197,7 @@ def __setattr__(self, key, val):
         else:
             raise OptionError("You can only set the value of existing options")
 
-    def __getattr__(self, key):
+    def __getattr__(self, key: str):
         prefix = object.__getattribute__(self, "prefix")
         if prefix:
             prefix += "."

pandas/_libs/hashing.pyx

@@ -70,6 +70,12 @@ def hash_object_array(object[:] arr, object key, object encoding='utf8'):
             # null, stringify and encode
             data = <bytes>str(val).encode(encoding)
 
+        elif isinstance(val, tuple):
+            # GH#28969 we could have a tuple, but need to ensure that
+            #  the tuple entries are themselves hashable before converting
+            #  to str
+            hash(val)
+            data = <bytes>str(val).encode(encoding)
         else:
             raise TypeError(f"{val} of type {type(val)} is not a valid type "
                             "for hashing, must be string or null")