From b2dd9b9afb270cb5b232fcad065efcaf9e6ee3c5 Mon Sep 17 00:00:00 2001 From: Srinivas Reddy Thatiparthy Date: Sun, 18 Nov 2018 23:30:07 +0530 Subject: [PATCH 1/8] Add _expand_user to _stringify_path as we need to allow tilde - ~, to expand to full path --- pandas/io/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/io/common.py b/pandas/io/common.py index 3a67238a66450..ad054d77b3bc8 100644 --- a/pandas/io/common.py +++ b/pandas/io/common.py @@ -157,7 +157,7 @@ def _stringify_path(filepath_or_buffer): return text_type(filepath_or_buffer) if _PY_PATH_INSTALLED and isinstance(filepath_or_buffer, LocalPath): return filepath_or_buffer.strpath - return filepath_or_buffer + return _expand_user(filepath_or_buffer) def is_s3_url(url): From 7c5a5b84392e1d6c28094d43072997e8799f82cf Mon Sep 17 00:00:00 2001 From: Srinivas Reddy Thatiparthy Date: Mon, 19 Nov 2018 00:36:07 +0530 Subject: [PATCH 2/8] Write whatsnew section about supporting tilde argument in path --- doc/source/whatsnew/v0.24.0.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 4ff3cc728f7f7..b81a45c7d60a9 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -293,6 +293,7 @@ Other Enhancements - :meth:`MultiIndex.to_flat_index` has been added to flatten multiple levels into a single-level :class:`Index` object. - :meth:`DataFrame.to_stata` and :class:` pandas.io.stata.StataWriter117` can write mixed sting columns to Stata strl format (:issue:`23633`) - :meth:`DataFrame.between_time` and :meth:`DataFrame.at_time` have gained the an ``axis`` parameter (:issue: `8839`) +- :meth:`DataFrame.to_json`, :meth:`DataFrame.to_csv`, :meth:`DataFrame.to_pickle`, and :meth:`DataFrame.to_XXX` etc. now support tilde(~) in path argument. (:issue:`23473`) .. _whatsnew_0240.api_breaking: From 19047a40d8e9bfb995303ce5a424f8302a0219f8 Mon Sep 17 00:00:00 2001 From: Srinivas Reddy Thatiparthy Date: Thu, 22 Nov 2018 19:18:49 +0530 Subject: [PATCH 3/8] Add test case for non existent file in user's home directory --- pandas/tests/io/test_common.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/pandas/tests/io/test_common.py b/pandas/tests/io/test_common.py index 2f2b792588a92..447dcff2f322a 100644 --- a/pandas/tests/io/test_common.py +++ b/pandas/tests/io/test_common.py @@ -150,6 +150,26 @@ def test_read_non_existant(self, reader, module, error_class, fn_ext): with pytest.raises(error_class): reader(path) + @pytest.mark.parametrize('reader, module, error_class, fn_ext', [ + (pd.read_csv, 'os', FileNotFoundError, 'csv'), + (pd.read_fwf, 'os', FileNotFoundError, 'txt'), + (pd.read_excel, 'xlrd', FileNotFoundError, 'xlsx'), + (pd.read_feather, 'feather', Exception, 'feather'), + (pd.read_hdf, 'tables', FileNotFoundError, 'h5'), + (pd.read_stata, 'os', FileNotFoundError, 'dta'), + (pd.read_sas, 'os', FileNotFoundError, 'sas7bdat'), + (pd.read_json, 'os', ValueError, 'json'), + (pd.read_msgpack, 'os', ValueError, 'mp'), + (pd.read_pickle, 'os', FileNotFoundError, 'pickle'), + ]) + def test_read_non_existant_user_home_dir(self, reader, module, + error_class, fn_ext): + pytest.importorskip(module) + + path = os.path.join('~', 'does_not_exist.' + fn_ext) + with pytest.raises(error_class): + reader(path) + def test_read_non_existant_read_table(self): path = os.path.join(HERE, 'data', 'does_not_exist.' + 'csv') with pytest.raises(FileNotFoundError): From 9a4a4bde894a9752c90935852698a70a28bebb3e Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 5 Jan 2019 08:37:51 -0800 Subject: [PATCH 4/8] Stronger assertion around expand_user --- pandas/tests/io/test_common.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pandas/tests/io/test_common.py b/pandas/tests/io/test_common.py index d04d4a1dd0cab..add8b30affe0c 100644 --- a/pandas/tests/io/test_common.py +++ b/pandas/tests/io/test_common.py @@ -160,12 +160,13 @@ def test_read_non_existant(self, reader, module, error_class, fn_ext): (pd.read_msgpack, 'os', ValueError, 'mp'), (pd.read_pickle, 'os', FileNotFoundError, 'pickle'), ]) - def test_read_non_existant_user_home_dir(self, reader, module, - error_class, fn_ext): + def test_read_expands_user_home_dir(self, reader, module, + error_class, fn_ext, monkeypatch): pytest.importorskip(module) path = os.path.join('~', 'does_not_exist.' + fn_ext) - with pytest.raises(error_class): + monkeypatch.setattr(icom, '_expand_user', lambda x: 'foo') + with pytest.raises(error_class, message=r".*foo/does_not_exist.*"): reader(path) def test_read_non_existant_read_table(self): From 5c74000542d3b13a3abd36f63002631af247377b Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 5 Jan 2019 10:07:27 -0800 Subject: [PATCH 5/8] Fixed issue with test case --- pandas/tests/io/test_common.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pandas/tests/io/test_common.py b/pandas/tests/io/test_common.py index add8b30affe0c..5e1cb13964efe 100644 --- a/pandas/tests/io/test_common.py +++ b/pandas/tests/io/test_common.py @@ -165,8 +165,10 @@ def test_read_expands_user_home_dir(self, reader, module, pytest.importorskip(module) path = os.path.join('~', 'does_not_exist.' + fn_ext) - monkeypatch.setattr(icom, '_expand_user', lambda x: 'foo') - with pytest.raises(error_class, message=r".*foo/does_not_exist.*"): + monkeypatch.setattr(icom, '_expand_user', + lambda x: os.path.join('foo', x)) + with pytest.raises(error_class, + message=r".*foo/does_not_exist." + fn_ext): reader(path) def test_read_non_existant_read_table(self): From 811395f283c6ef96cc7dac505782c37e474d2761 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 5 Jan 2019 10:08:21 -0800 Subject: [PATCH 6/8] Removed trailing whitespace from whatsnew --- doc/source/whatsnew/v0.24.0.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.24.0.rst b/doc/source/whatsnew/v0.24.0.rst index 7c6a53a10680b..4ff1f96777b1c 100644 --- a/doc/source/whatsnew/v0.24.0.rst +++ b/doc/source/whatsnew/v0.24.0.rst @@ -416,7 +416,7 @@ Other Enhancements - :class:`IntervalIndex` has gained the :attr:`~IntervalIndex.is_overlapping` attribute to indicate if the ``IntervalIndex`` contains any overlapping intervals (:issue:`23309`) - :func:`pandas.DataFrame.to_sql` has gained the ``method`` argument to control SQL insertion clause. See the :ref:`insertion method ` section in the documentation. (:issue:`8953`) - :meth:`DataFrame.corrwith` now supports Spearman's rank correlation, Kendall's tau as well as callable correlation methods. (:issue:`21925`) -- :meth:`DataFrame.to_json`, :meth:`DataFrame.to_csv`, :meth:`DataFrame.to_pickle`, and :meth:`DataFrame.to_XXX` etc. now support tilde(~) in path argument. (:issue:`23473`) +- :meth:`DataFrame.to_json`, :meth:`DataFrame.to_csv`, :meth:`DataFrame.to_pickle`, and :meth:`DataFrame.to_XXX` etc. now support tilde(~) in path argument. (:issue:`23473`) .. _whatsnew_0240.api_breaking: From fb31fa2cfd08aba0469366cda55ca33ef6678e5c Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 5 Jan 2019 10:17:09 -0800 Subject: [PATCH 7/8] More portable test --- pandas/tests/io/test_common.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pandas/tests/io/test_common.py b/pandas/tests/io/test_common.py index 5e1cb13964efe..53e6627f37efe 100644 --- a/pandas/tests/io/test_common.py +++ b/pandas/tests/io/test_common.py @@ -3,6 +3,7 @@ """ import mmap import os +import re import pytest @@ -167,8 +168,10 @@ def test_read_expands_user_home_dir(self, reader, module, path = os.path.join('~', 'does_not_exist.' + fn_ext) monkeypatch.setattr(icom, '_expand_user', lambda x: os.path.join('foo', x)) - with pytest.raises(error_class, - message=r".*foo/does_not_exist." + fn_ext): + + message = "".join(["foo", os.path.sep, "does_not_exist.", fn_ext]) + + with pytest.raises(error_class, message=re.escape(message)): reader(path) def test_read_non_existant_read_table(self): From 0a660e0e026af15b0e4177690055f106f21918b5 Mon Sep 17 00:00:00 2001 From: Will Ayd Date: Sat, 5 Jan 2019 10:39:19 -0800 Subject: [PATCH 8/8] LINT fixup --- pandas/tests/io/test_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pandas/tests/io/test_common.py b/pandas/tests/io/test_common.py index 53e6627f37efe..13a8b1a0edfd3 100644 --- a/pandas/tests/io/test_common.py +++ b/pandas/tests/io/test_common.py @@ -162,7 +162,7 @@ def test_read_non_existant(self, reader, module, error_class, fn_ext): (pd.read_pickle, 'os', FileNotFoundError, 'pickle'), ]) def test_read_expands_user_home_dir(self, reader, module, - error_class, fn_ext, monkeypatch): + error_class, fn_ext, monkeypatch): pytest.importorskip(module) path = os.path.join('~', 'does_not_exist.' + fn_ext)