Merge remote-tracking branch 'upstream/master' into typing

Author: simonjayhawkins

ci/code_checks.sh

@@ -190,6 +190,10 @@ if [[ -z "$CHECK" || "$CHECK" == "patterns" ]]; then
     invgrep -R --include="*.rst" ".. ipython ::" doc/source
     RET=$(($RET + $?)) ; echo $MSG "DONE"
 
+    MSG='Check for extra blank lines after the class definition' ; echo $MSG
+    invgrep -R --include="*.py" --include="*.pyx" -E 'class.*:\n\n( )+"""' .
+    RET=$(($RET + $?)) ; echo $MSG "DONE"
+
     MSG='Check that no file in the repo contains trailing whitespaces' ; echo $MSG
     set -o pipefail
     if [[ "$AZURE" == "true" ]]; then

doc/source/user_guide/integer_na.rst

@@ -63,7 +63,7 @@ up with a ``float64`` dtype Series:
    pd.Series([1, 2, np.nan])
 
 Operations involving an integer array will behave similar to NumPy arrays.
-Missing values will be propagated, and and the data will be coerced to another
+Missing values will be propagated, and the data will be coerced to another
 dtype if needed.
 
 .. ipython:: python

doc/source/user_guide/io.rst

@@ -4671,7 +4671,6 @@ See the `Full Documentation <https://github.com/wesm/feather>`__.
 Write to a feather file.
 
 .. ipython:: python
-   :okwarning:
 
    df.to_feather('example.feather')
 
@@ -4748,7 +4747,6 @@ See the documentation for `pyarrow <https://arrow.apache.org/docs/python/>`__ an
 Write to a parquet file.
 
 .. ipython:: python
-   :okwarning:
 
    df.to_parquet('example_pa.parquet', engine='pyarrow')
    df.to_parquet('example_fp.parquet', engine='fastparquet')
@@ -4765,7 +4763,6 @@ Read from a parquet file.
 Read only certain columns of a parquet file.
 
 .. ipython:: python
-   :okwarning:
 
    result = pd.read_parquet('example_fp.parquet',
                             engine='fastparquet', columns=['a', 'b'])
@@ -4788,7 +4785,6 @@ Serializing a ``DataFrame`` to parquet may include the implicit index as one or
 more columns in the output file. Thus, this code:
 
 .. ipython:: python
-   :okwarning:
 
     df = pd.DataFrame({'a': [1, 2], 'b': [3, 4]})
     df.to_parquet('test.parquet', engine='pyarrow')
@@ -4805,7 +4801,6 @@ If you want to omit a dataframe's indexes when writing, pass ``index=False`` to
 :func:`~pandas.DataFrame.to_parquet`:
 
 .. ipython:: python
-   :okwarning:
 
     df.to_parquet('test.parquet', index=False)
 

doc/source/whatsnew/v0.20.0.rst

@@ -33,7 +33,7 @@ Check the :ref:`API Changes <whatsnew_0200.api_breaking>` and :ref:`deprecations
 
 .. note::
 
-   This is a combined release for 0.20.0 and and 0.20.1.
+   This is a combined release for 0.20.0 and 0.20.1.
    Version 0.20.1 contains one additional change for backwards-compatibility with downstream projects using pandas' ``utils`` routines. (:issue:`16250`)
 
 .. contents:: What's new in v0.20.0

doc/source/whatsnew/v1.0.0.rst

@@ -318,7 +318,7 @@ Datetimelike
 
 Timedelta
 ^^^^^^^^^
-
+- Bug in subtracting a :class:`TimedeltaIndex` or :class:`TimedeltaArray` from a ``np.datetime64`` object (:issue:`29558`)
 -
 -
 
@@ -400,6 +400,8 @@ I/O
 - Bug in :meth:`DataFrame.to_html` when using ``formatters=<list>`` and ``max_cols`` together. (:issue:`25955`)
 - Bug in :meth:`Styler.background_gradient` not able to work with dtype ``Int64`` (:issue:`28869`)
 - Bug in :meth:`DataFrame.to_clipboard` which did not work reliably in ipython (:issue:`22707`)
+- Bug in :func:`read_json` where default encoding was not set to ``utf-8`` (:issue:`29565`)
+-
 
 Plotting
 ^^^^^^^^

pandas/_libs/hashtable_class_helper.pxi.in

@@ -100,7 +100,7 @@ cdef class {{name}}Vector:
             PyMem_Free(self.data)
             self.data = NULL
 
-    def __len__(self):
+    def __len__(self) -> int:
         return self.data.n
 
     cpdef to_array(self):
@@ -168,7 +168,7 @@ cdef class StringVector:
             PyMem_Free(self.data)
             self.data = NULL
 
-    def __len__(self):
+    def __len__(self) -> int:
         return self.data.n
 
     def to_array(self):
@@ -212,7 +212,7 @@ cdef class ObjectVector:
         self.ao = np.empty(_INIT_VEC_CAP, dtype=object)
         self.data = <PyObject**>self.ao.data
 
-    def __len__(self):
+    def __len__(self) -> int:
         return self.n
 
     cdef inline append(self, object obj):
@@ -270,7 +270,7 @@ cdef class {{name}}HashTable(HashTable):
             size_hint = min(size_hint, _SIZE_HINT_LIMIT)
             kh_resize_{{dtype}}(self.table, size_hint)
 
-    def __len__(self):
+    def __len__(self) -> int:
         return self.table.size
 
     def __dealloc__(self):
@@ -897,7 +897,7 @@ cdef class PyObjectHashTable(HashTable):
             kh_destroy_pymap(self.table)
             self.table = NULL
 
-    def __len__(self):
+    def __len__(self) -> int:
         return self.table.size
 
     def __contains__(self, object key):

pandas/_libs/indexing.pyx

@@ -11,7 +11,7 @@ cdef class _NDFrameIndexerBase:
         self._ndim = None
 
     @property
-    def ndim(self):
+    def ndim(self) -> int:
         # Delay `ndim` instantiation until required as reading it
         # from `obj` isn't entirely cheap.
         ndim = self._ndim

pandas/_libs/internals.pyx

@@ -66,7 +66,7 @@ cdef class BlockPlacement:
     def __repr__(self) -> str:
         return str(self)
 
-    def __len__(self):
+    def __len__(self) -> int:
         cdef:
             slice s = self._ensure_has_slice()
         if s is not None:

pandas/_libs/lib.pyx

@@ -971,6 +971,7 @@ cdef class Seen:
         bint nat_             # seen nat
         bint bool_            # seen_bool
         bint null_            # seen_null
+        bint nan_             # seen_np.nan
         bint uint_            # seen_uint (unsigned integer)
         bint sint_            # seen_sint (signed integer)
         bint float_           # seen_float
@@ -995,6 +996,7 @@ cdef class Seen:
         self.nat_ = 0
         self.bool_ = 0
         self.null_ = 0
+        self.nan_ = 0
         self.uint_ = 0
         self.sint_ = 0
         self.float_ = 0
@@ -1953,10 +1955,37 @@ def maybe_convert_numeric(ndarray[object] values, set na_values,
 @cython.wraparound(False)
 def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
                           bint safe=0, bint convert_datetime=0,
-                          bint convert_timedelta=0):
+                          bint convert_timedelta=0,
+                          bint convert_to_nullable_integer=0):
     """
     Type inference function-- convert object array to proper dtype
+
+    Parameters
+    ----------
+    values : ndarray
+        Array of object elements to convert.
+    try_float : bool, default False
+        If an array-like object contains only float or NaN values is
+        encountered, whether to convert and return an array of float dtype.
+    safe : bool, default False
+        Whether to upcast numeric type (e.g. int cast to float). If set to
+        True, no upcasting will be performed.
+    convert_datetime : bool, default False
+        If an array-like object contains only datetime values or NaT is
+        encountered, whether to convert and return an array of M8[ns] dtype.
+    convert_timedelta : bool, default False
+        If an array-like object contains only timedelta values or NaT is
+        encountered, whether to convert and return an array of m8[ns] dtype.
+    convert_to_nullable_integer : bool, default False
+        If an array-like object contains only interger values (and NaN) is
+        encountered, whether to convert and return an IntegerArray.
+
+    Returns
+    -------
+    array : array of converted object values to more specific dtypes if
+    pplicable
     """
+
     cdef:
         Py_ssize_t i, n
         ndarray[float64_t] floats
@@ -1977,6 +2006,7 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
     ints = np.empty(n, dtype='i8')
     uints = np.empty(n, dtype='u8')
     bools = np.empty(n, dtype=np.uint8)
+    mask = np.full(n, False)
 
     if convert_datetime:
         datetimes = np.empty(n, dtype='M8[ns]')
@@ -1994,6 +2024,7 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
         if val is None:
             seen.null_ = 1
             floats[i] = complexes[i] = fnan
+            mask[i] = True
         elif val is NaT:
             seen.nat_ = 1
             if convert_datetime:
@@ -2003,6 +2034,10 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
             if not (convert_datetime or convert_timedelta):
                 seen.object_ = 1
                 break
+        elif val is np.nan:
+            seen.nan_ = 1
+            mask[i] = True
+            floats[i] = complexes[i] = val
         elif util.is_bool_object(val):
             seen.bool_ = 1
             bools[i] = val
@@ -2084,11 +2119,19 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
 
     if not seen.object_:
         if not safe:
-            if seen.null_:
+            if seen.null_ or seen.nan_:
                 if seen.is_float_or_complex:
                     if seen.complex_:
                         return complexes
-                    elif seen.float_ or seen.int_:
+                    elif seen.float_:
+                        return floats
+                    elif seen.int_:
+                        if convert_to_nullable_integer:
+                            from pandas.core.arrays import IntegerArray
+                            return IntegerArray(ints, mask)
+                        else:
+                            return floats
+                    elif seen.nan_:
                         return floats
             else:
                 if not seen.bool_:
@@ -2127,7 +2170,7 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
                     if seen.complex_:
                         if not seen.int_:
                             return complexes
-                    elif seen.float_:
+                    elif seen.float_ or seen.nan_:
                         if not seen.int_:
                             return floats
             else:
@@ -2151,7 +2194,7 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
                         if seen.complex_:
                             if not seen.int_:
                                 return complexes
-                        elif seen.float_:
+                        elif seen.float_ or seen.nan_:
                             if not seen.int_:
                                 return floats
                         elif seen.int_: