From 751751cf8cb97167ed8e18cfc99d231980a0aef7 Mon Sep 17 00:00:00 2001 From: Kerby Shedden Date: Mon, 5 Sep 2016 15:37:30 -0400 Subject: [PATCH 1/5] Initial commit to fixe issue #11526 --- pandas/io/stata.py | 6 +++--- pandas/io/tests/test_stata.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/pandas/io/stata.py b/pandas/io/stata.py index 25f13048a73fd..985ea9c051505 100644 --- a/pandas/io/stata.py +++ b/pandas/io/stata.py @@ -34,8 +34,8 @@ from pandas.tslib import NaT, Timestamp _version_error = ("Version of given Stata file is not 104, 105, 108, " - "113 (Stata 8/9), 114 (Stata 10/11), 115 (Stata 12), " - "117 (Stata 13), or 118 (Stata 14)") + "111 (Stata 7SE), 113 (Stata 8/9), 114 (Stata 10/11), " + "115 (Stata 12), 117 (Stata 13), or 118 (Stata 14)") _statafile_processing_params1 = """\ convert_dates : boolean, defaults to True @@ -1183,7 +1183,7 @@ def _get_seek_variable_labels(self): def _read_old_header(self, first_char): self.format_version = struct.unpack('b', first_char)[0] - if self.format_version not in [104, 105, 108, 113, 114, 115]: + if self.format_version not in [104, 105, 108, 111, 113, 114, 115]: raise ValueError(_version_error) self.byteorder = struct.unpack('b', self.path_or_buf.read(1))[ 0] == 0x1 and '>' or '<' diff --git a/pandas/io/tests/test_stata.py b/pandas/io/tests/test_stata.py index 7752fff5247c0..34d175121fb06 100644 --- a/pandas/io/tests/test_stata.py +++ b/pandas/io/tests/test_stata.py @@ -82,6 +82,8 @@ def setUp(self): self.dta22_118 = os.path.join(self.dirpath, 'stata14_118.dta') self.dta23 = os.path.join(self.dirpath, 'stata15.dta') + self.dta24_111 = os.path.join(self.dirpath, 'stata7_111.dta') + def read_dta(self, file): # Legacy default reader configuration return read_stata(file, convert_dates=True) @@ -1219,6 +1221,19 @@ def test_repeated_column_labels(self): read_stata(self.dta23, convert_categoricals=True) tm.assertTrue('wolof' in cm.exception) + def test_stata_111(self): + # 111 is an old version but still used by current versions of + # SAS when exporting to Stata format. We do not know of any + # on-line documentation for this version. + df = read_stata(self.dta24_111) + original = pd.DataFrame({'y': [1,1,1,1,1,0,0,np.NaN,0,0], + 'x': [1,2,1,3,np.NaN,4,3,5,1,6], + 'w': [2,np.NaN,5,2,4,4,3,1,2,3], + 'z': ['a','b','c','d','e','','g','h','i','j']}) + original = original[['y', 'x', 'w', 'z']] + tm.assert_frame_equal(original, df) + + if __name__ == '__main__': nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'], exit=False) From a5a5842bea211d9da7285fb54ae75a075a34c959 Mon Sep 17 00:00:00 2001 From: Kerby Shedden Date: Mon, 5 Sep 2016 15:46:21 -0400 Subject: [PATCH 2/5] Added missing test data file --- pandas/io/tests/data/stata7_111.dta | Bin 0 -> 1024 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 pandas/io/tests/data/stata7_111.dta diff --git a/pandas/io/tests/data/stata7_111.dta b/pandas/io/tests/data/stata7_111.dta new file mode 100644 index 0000000000000000000000000000000000000000..e87fa3a72ff8e5b1486c8350bc19d6917429f8e7 GIT binary patch literal 1024 zcmd02Vq{=p;9{T)`2YVuQzaprDhMekC#0Z?kOG9fs-cmdK{|wwlWkPY08@j|Im#L) zA@IQ-;t?3l;E)LQ8A1di@Sr{kO$Mey#330?hQk3W;E;kQ^PnCgAmES+m4Q(L4lo&* o2#kiA#^C@>s|+wxM4);;*k`~LLur`X7#uRuWF(+^I2^JV08VNl4*&oF literal 0 HcmV?d00001 From 929d8af33f82e6f315265c05314f981c28cc3285 Mon Sep 17 00:00:00 2001 From: Kerby Shedden Date: Mon, 5 Sep 2016 20:23:48 -0400 Subject: [PATCH 3/5] Fix flake8 errors --- pandas/io/tests/test_stata.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pandas/io/tests/test_stata.py b/pandas/io/tests/test_stata.py index 34d175121fb06..1849b32a4a7c8 100644 --- a/pandas/io/tests/test_stata.py +++ b/pandas/io/tests/test_stata.py @@ -1226,10 +1226,11 @@ def test_stata_111(self): # SAS when exporting to Stata format. We do not know of any # on-line documentation for this version. df = read_stata(self.dta24_111) - original = pd.DataFrame({'y': [1,1,1,1,1,0,0,np.NaN,0,0], - 'x': [1,2,1,3,np.NaN,4,3,5,1,6], - 'w': [2,np.NaN,5,2,4,4,3,1,2,3], - 'z': ['a','b','c','d','e','','g','h','i','j']}) + original = pd.DataFrame({'y': [1, 1, 1, 1, 1, 0, 0, np.NaN, 0, 0], + 'x': [1, 2, 1, 3, np.NaN, 4, 3, 5, 1, 6], + 'w': [2, np.NaN, 5, 2, 4, 4, 3, 1, 2, 3], + 'z': ['a', 'b', 'c', 'd', 'e', '', 'g', 'h', + 'i', 'j']}) original = original[['y', 'x', 'w', 'z']] tm.assert_frame_equal(original, df) From 0ad1399ce473d6e1bc1832b7a77289eb5970a77e Mon Sep 17 00:00:00 2001 From: Kerby Shedden Date: Sat, 10 Sep 2016 14:59:16 -0400 Subject: [PATCH 4/5] Added to whatsnew --- doc/source/whatsnew/v0.20.0.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/source/whatsnew/v0.20.0.txt b/doc/source/whatsnew/v0.20.0.txt index 4aee6f72b1d53..71f7ed31a7cd7 100644 --- a/doc/source/whatsnew/v0.20.0.txt +++ b/doc/source/whatsnew/v0.20.0.txt @@ -30,7 +30,8 @@ New features Other enhancements ^^^^^^^^^^^^^^^^^^ - +`read_stata` can handle some format 111 files, which are produced by +SAS when generating Stata dta files (:issue:11526, :issue:14159) From 34e09ef9fafd6f25e6e50fcd611fcf28b776ca2b Mon Sep 17 00:00:00 2001 From: Kerby Shedden Date: Sat, 10 Sep 2016 15:00:06 -0400 Subject: [PATCH 5/5] Added to whatsnew --- doc/source/whatsnew/v0.20.0.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/doc/source/whatsnew/v0.20.0.txt b/doc/source/whatsnew/v0.20.0.txt index 71f7ed31a7cd7..e2e8ddcc47271 100644 --- a/doc/source/whatsnew/v0.20.0.txt +++ b/doc/source/whatsnew/v0.20.0.txt @@ -30,9 +30,6 @@ New features Other enhancements ^^^^^^^^^^^^^^^^^^ -`read_stata` can handle some format 111 files, which are produced by -SAS when generating Stata dta files (:issue:11526, :issue:14159) - @@ -83,3 +80,5 @@ Performance Improvements Bug Fixes ~~~~~~~~~ +`read_stata` can now handle some format 111 files, which are produced +by SAS when generating Stata dta files (:issue:11526, :issue:14159)