Storage options

doc/source/user_guide/io.rst

@@ -1649,8 +1649,10 @@ options include:
 Specifying any of the above options will produce a ``ParserWarning`` unless the
 python engine is selected explicitly using ``engine='python'``.
 
-Reading remote files
-''''''''''''''''''''
+.. _io.remote:
+
+Reading/writing remote files
+''''''''''''''''''''''''''''
 
 You can pass in a URL to read or write remote files to many of Pandas' IO
 functions - the following example shows reading a CSV file:
@@ -1686,6 +1688,8 @@ You can also pass parameters directly to the backend driver. For example,
 if you do *not* have S3 credentials, you can still access public data by
 specifying an anonymous connection, such as
 
+.. versionadded:: 1.2.0
+
 .. code-block:: python
 
    pd.read_csv("s3://ncei-wcsd-archive/data/processed/SH1305/18kHz/SaKe2013"
@@ -1696,6 +1700,8 @@ specifying an anonymous connection, such as
 archives, local caching of files, and more. To locally cache the above
 example, you would modify the call to
 
+.. code-block:: python
+
    pd.read_csv("simplecache::s3://ncei-wcsd-archive/data/processed/SH1305/18kHz/SaKe2013"
                "-D20130523-T080854_to_SaKe2013-D20130523-T085643.csv",
                storage_options={"s3": {"anon": True}})

doc/source/whatsnew/v1.2.0.rst

@@ -21,7 +21,8 @@ to pass a dictionary of parameters to the storage backend. This allows, for
 example, for passing credentials to S3 and GCS storage. The details of what
 parameters can be passed to which backends can be found in the documentation
 of the individual storage backends (detailed from the fsspec docs for
-`builtin implementations`_ and linked to `external ones`_).
+`builtin implementations`_ and linked to `external ones`_). See
+Section :ref:`io.remote`.
 
 .. _builtin implementations: https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations
 .. _external ones: https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations

pandas/_typing.py

@@ -106,3 +106,6 @@
     List[AggFuncTypeBase],
     Dict[Label, Union[AggFuncTypeBase, List[AggFuncTypeBase]]],
 ]
+
+# for arbitrary kwargs passed during reading/writing files
+StorageOptions = Optional[Dict[str, Any]]

pandas/core/frame.py

@@ -55,6 +55,7 @@
     Label,
     Level,
     Renamer,
+    StorageOptions,
     ValueKeyFunc,
 )
 from pandas.compat import PY37
@@ -2056,7 +2057,7 @@ def to_stata(
         version: Optional[int] = 114,
         convert_strl: Optional[Sequence[Label]] = None,
         compression: Union[str, Mapping[str, str], None] = "infer",
-        storage_options: Optional[Dict[str, Any]] = None,
+        storage_options: StorageOptions = None,
     ) -> None:
         """
         Export DataFrame object to Stata dta format.
@@ -2259,7 +2260,7 @@ def to_markdown(
         buf: Optional[Union[IO[str], str]] = None,
         mode: str = "wt",
         index: bool = True,
-        storage_options: Optional[Dict[str, Any]] = None,
+        storage_options: StorageOptions = None,
         **kwargs,
     ) -> Optional[str]:
         if "showindex" in kwargs:
@@ -2295,7 +2296,7 @@ def to_parquet(
         compression: Optional[str] = "snappy",
         index: Optional[bool] = None,
         partition_cols: Optional[List[str]] = None,
-        storage_options: Optional[Dict[str, Any]] = None,
+        storage_options: StorageOptions = None,
         **kwargs,
     ) -> None:
         """

pandas/core/generic.py

@@ -40,6 +40,7 @@
     Label,
     Level,
     Renamer,
+    StorageOptions,
     TimedeltaConvertibleTypes,
     TimestampConvertibleTypes,
     ValueKeyFunc,
@@ -2042,7 +2043,7 @@ def to_json(
         compression: Optional[str] = "infer",
         index: bool_t = True,
         indent: Optional[int] = None,
-        storage_options: Optional[Dict[str, Any]] = None,
+        storage_options: StorageOptions = None,
     ) -> Optional[str]:
         """
         Convert the object to a JSON string.
@@ -2629,7 +2630,7 @@ def to_pickle(
         path,
         compression: Optional[str] = "infer",
         protocol: int = pickle.HIGHEST_PROTOCOL,
-        storage_options: Optional[Dict[str, Any]] = None,
+        storage_options: StorageOptions = None,
     ) -> None:
         """
         Pickle (serialize) object to file.
@@ -3044,7 +3045,7 @@ def to_csv(
         escapechar: Optional[str] = None,
         decimal: Optional[str] = ".",
         errors: str = "strict",
-        storage_options: Optional[Dict[str, Any]] = None,
+        storage_options: StorageOptions = None,
     ) -> Optional[str]:
         r"""
         Write object to a comma-separated values (csv) file.

pandas/core/series.py

@@ -9,7 +9,6 @@
     TYPE_CHECKING,
     Any,
     Callable,
-    Dict,
     Iterable,
     List,
     Optional,
@@ -32,6 +31,7 @@
     FrameOrSeriesUnion,
     IndexKeyFunc,
     Label,
+    StorageOptions,
     ValueKeyFunc,
 )
 from pandas.compat.numpy import function as nv
@@ -1425,7 +1425,7 @@ def to_markdown(
         buf: Optional[IO[str]] = None,
         mode: str = "wt",
         index: bool = True,
-        storage_options: Optional[Dict[str, Any]] = None,
+        storage_options: StorageOptions = None,
         **kwargs,
     ) -> Optional[str]:
         """

pandas/io/common.py

@@ -29,7 +29,7 @@
 )
 import zipfile
 
-from pandas._typing import FilePathOrBuffer
+from pandas._typing import FilePathOrBuffer, StorageOptions
 from pandas.compat import _get_lzma_file, _import_lzma
 from pandas.compat._optional import import_optional_dependency
 
@@ -162,7 +162,7 @@ def get_filepath_or_buffer(
     encoding: Optional[str] = None,
     compression: Optional[str] = None,
     mode: Optional[str] = None,
-    storage_options: Optional[Dict[str, Any]] = None,
+    storage_options: StorageOptions = None,
 ):
     """
     If the filepath_or_buffer is a url, translate and return the buffer.

pandas/io/formats/csvs.py

@@ -5,13 +5,13 @@
 import csv as csvlib
 from io import StringIO, TextIOWrapper
 import os
-from typing import Any, Dict, Hashable, List, Mapping, Optional, Sequence, Union
+from typing import Hashable, List, Mapping, Optional, Sequence, Union
 import warnings
 
 import numpy as np
 
 from pandas._libs import writers as libwriters
-from pandas._typing import FilePathOrBuffer
+from pandas._typing import FilePathOrBuffer, StorageOptions
 
 from pandas.core.dtypes.generic import (
     ABCDatetimeIndex,
@@ -53,7 +53,7 @@ def __init__(
         doublequote: bool = True,
         escapechar: Optional[str] = None,
         decimal=".",
-        storage_options: Optional[Dict[str, Any]] = None,
+        storage_options: StorageOptions = None,
     ):
         self.obj = obj
 

pandas/io/json/_json.py

@@ -3,13 +3,13 @@
 from io import BytesIO, StringIO
 from itertools import islice
 import os
-from typing import Any, Callable, Dict, Optional, Type
+from typing import Any, Callable, Optional, Type
 
 import numpy as np
 
 import pandas._libs.json as json
 from pandas._libs.tslibs import iNaT
-from pandas._typing import JSONSerializable
+from pandas._typing import JSONSerializable, StorageOptions
 from pandas.errors import AbstractMethodError
 from pandas.util._decorators import deprecate_kwarg, deprecate_nonkeyword_arguments
 
@@ -44,7 +44,7 @@ def to_json(
     compression: Optional[str] = "infer",
     index: bool = True,
     indent: int = 0,
-    storage_options: Optional[Dict[str, Any]] = None,
+    storage_options: StorageOptions = None,
 ):
 
     if not index and orient not in ["split", "table"]:
@@ -371,7 +371,7 @@ def read_json(
     chunksize: Optional[int] = None,
     compression="infer",
     nrows: Optional[int] = None,
-    storage_options: Optional[Dict[str, Any]] = None,
+    storage_options: StorageOptions = None,
 ):
     """
     Convert a JSON string to pandas object.

pandas/io/parquet.py

@@ -3,7 +3,7 @@
 from typing import Any, AnyStr, Dict, List, Optional
 from warnings import catch_warnings
 
-from pandas._typing import FilePathOrBuffer
+from pandas._typing import FilePathOrBuffer, StorageOptions
 from pandas.compat._optional import import_optional_dependency
 from pandas.errors import AbstractMethodError
 
@@ -89,7 +89,7 @@ def write(
         path: FilePathOrBuffer[AnyStr],
         compression: Optional[str] = "snappy",
         index: Optional[bool] = None,
-        storage_options: Optional[Dict[str, Any]] = None,
+        storage_options: StorageOptions = None,
         partition_cols: Optional[List[str]] = None,
         **kwargs,
     ):
@@ -128,11 +128,7 @@ def write(
             self.api.parquet.write_table(table, path, compression=compression, **kwargs)
 
     def read(
-        self,
-        path,
-        columns=None,
-        storage_options: Optional[Dict[str, Any]] = None,
-        **kwargs,
+        self, path, columns=None, storage_options: StorageOptions = None, **kwargs,
     ):
         if is_fsspec_url(path) and "filesystem" not in kwargs:
             import_optional_dependency("fsspec")
@@ -178,7 +174,7 @@ def write(
         compression="snappy",
         index=None,
         partition_cols=None,
-        storage_options: Optional[Dict[str, Any]] = None,
+        storage_options: StorageOptions = None,
         **kwargs,
     ):
         self.validate_dataframe(df)
@@ -222,11 +218,7 @@ def write(
             )
 
     def read(
-        self,
-        path,
-        columns=None,
-        storage_options: Optional[Dict[str, Any]] = None,
-        **kwargs,
+        self, path, columns=None, storage_options: StorageOptions = None, **kwargs,
     ):
         if is_fsspec_url(path):
             fsspec = import_optional_dependency("fsspec")
@@ -248,7 +240,7 @@ def to_parquet(
     engine: str = "auto",
     compression: Optional[str] = "snappy",
     index: Optional[bool] = None,
-    storage_options: Optional[Dict[str, Any]] = None,
+    storage_options: StorageOptions = None,
     partition_cols: Optional[List[str]] = None,
     **kwargs,
 ):

pandas/io/pickle.py

@@ -1,9 +1,9 @@
 """ pickle compat """
 import pickle
-from typing import Any, Dict, Optional
+from typing import Any, Optional
 import warnings
 
-from pandas._typing import FilePathOrBuffer
+from pandas._typing import FilePathOrBuffer, StorageOptions
 from pandas.compat import pickle_compat as pc
 
 from pandas.io.common import get_filepath_or_buffer, get_handle
@@ -14,7 +14,7 @@ def to_pickle(
     filepath_or_buffer: FilePathOrBuffer,
     compression: Optional[str] = "infer",
     protocol: int = pickle.HIGHEST_PROTOCOL,
-    storage_options: Optional[Dict[str, Any]] = None,
+    storage_options: StorageOptions = None,
 ):
     """
     Pickle (serialize) object to file.
@@ -113,7 +113,7 @@ def to_pickle(
 def read_pickle(
     filepath_or_buffer: FilePathOrBuffer,
     compression: Optional[str] = "infer",
-    storage_options: Optional[Dict[str, Any]] = None,
+    storage_options: StorageOptions = None,
 ):
     """
     Load pickled pandas object (or any object) from file.

pandas/io/stata.py

@@ -35,7 +35,7 @@
 
 from pandas._libs.lib import infer_dtype
 from pandas._libs.writers import max_len_string_array
-from pandas._typing import FilePathOrBuffer, Label
+from pandas._typing import FilePathOrBuffer, Label, StorageOptions
 from pandas.util._decorators import Appender
 
 from pandas.core.dtypes.common import (
@@ -1035,7 +1035,7 @@ def __init__(
         columns: Optional[Sequence[str]] = None,
         order_categoricals: bool = True,
         chunksize: Optional[int] = None,
-        storage_options: Optional[Dict[str, Any]] = None,
+        storage_options: StorageOptions = None,
     ):
         super().__init__()
         self.col_sizes: List[int] = []
@@ -1910,7 +1910,7 @@ def read_stata(
     order_categoricals: bool = True,
     chunksize: Optional[int] = None,
     iterator: bool = False,
-    storage_options: Optional[Dict[str, Any]] = None,
+    storage_options: StorageOptions = None,
 ) -> Union[DataFrame, StataReader]:
 
     reader = StataReader(
@@ -1939,7 +1939,7 @@ def read_stata(
 def _open_file_binary_write(
     fname: FilePathOrBuffer,
     compression: Union[str, Mapping[str, str], None],
-    storage_options: Optional[Dict[str, Any]] = None,
+    storage_options: StorageOptions = None,
 ) -> Tuple[BinaryIO, bool, Optional[Union[str, Mapping[str, str]]]]:
     """
     Open a binary file or no-op if file-like.
@@ -2238,7 +2238,7 @@ def __init__(
         data_label: Optional[str] = None,
         variable_labels: Optional[Dict[Label, str]] = None,
         compression: Union[str, Mapping[str, str], None] = "infer",
-        storage_options: Optional[Dict[str, Any]] = None,
+        storage_options: StorageOptions = None,
     ):
         super().__init__()
         self._convert_dates = {} if convert_dates is None else convert_dates
@@ -3121,7 +3121,7 @@ def __init__(
         variable_labels: Optional[Dict[Label, str]] = None,
         convert_strl: Optional[Sequence[Label]] = None,
         compression: Union[str, Mapping[str, str], None] = "infer",
-        storage_options: Optional[Dict[str, Any]] = None,
+        storage_options: StorageOptions = None,
     ):
         # Copy to new list since convert_strl might be modified later
         self._convert_strl: List[Label] = []
@@ -3526,7 +3526,7 @@ def __init__(
         convert_strl: Optional[Sequence[Label]] = None,
         version: Optional[int] = None,
         compression: Union[str, Mapping[str, str], None] = "infer",
-        storage_options: Optional[Dict[str, Any]] = None,
+        storage_options: StorageOptions = None,
     ):
         if version is None:
             version = 118 if data.shape[1] <= 32767 else 119