From 6265450003aa5b5b818d7cd063efcdfb9148cf70 Mon Sep 17 00:00:00 2001
From: Kevin Sheppard <kevin.sheppard@economics.ox.ac.uk>
Date: Tue, 22 Jul 2014 18:23:22 +0100
Subject: [PATCH] BUG: Fixed failure in StataReader when reading variable
 labels in 117 files

Stata's implementation does not match the online dta file format description.
The solution used here is to directly compute the offset rather than reading
it from the dta file.  If Stata fixes their implementation, the original code
can be restored.
closes #7816
---
 doc/source/v0.15.0.txt              |   2 +-
 pandas/io/stata.py                  |  11 +++++++++--
 pandas/io/tests/data/stata7_115.dta | Bin 0 -> 722 bytes
 pandas/io/tests/data/stata7_117.dta | Bin 0 -> 1159 bytes
 pandas/io/tests/test_stata.py       |  17 ++++++++++++++++-
 5 files changed, 26 insertions(+), 4 deletions(-)
 create mode 100644 pandas/io/tests/data/stata7_115.dta
 create mode 100644 pandas/io/tests/data/stata7_117.dta

diff --git a/doc/source/v0.15.0.txt b/doc/source/v0.15.0.txt
index 06c93541a7783..2322af4752e2e 100644
--- a/doc/source/v0.15.0.txt
+++ b/doc/source/v0.15.0.txt
@@ -212,7 +212,7 @@ Bug Fixes
 
 
 - Bug in ``DataFrame.plot`` with ``subplots=True`` may draw unnecessary minor xticks and yticks (:issue:`7801`)
-
+- Bug in ``StataReader`` which did not read variable labels in 117 files due to difference between Stata documentation and implementation (:issue:`7816`)
 
 
 
diff --git a/pandas/io/stata.py b/pandas/io/stata.py
index 48a5f5ee6c994..3458a95ac096d 100644
--- a/pandas/io/stata.py
+++ b/pandas/io/stata.py
@@ -520,8 +520,15 @@ def _read_header(self):
                 self.byteorder + 'q', self.path_or_buf.read(8))[0] + 9
             seek_value_label_names = struct.unpack(
                 self.byteorder + 'q', self.path_or_buf.read(8))[0] + 19
-            seek_variable_labels = struct.unpack(
-                self.byteorder + 'q', self.path_or_buf.read(8))[0] + 17
+            # Stata 117 data files do not follow the described format.  This is
+            # a work around that uses the previous label, 33 bytes for each
+            # variable, 20 for the closing tag and 17 for the opening tag
+            self.path_or_buf.read(8)  # <variable_lables>, throw away
+            seek_variable_labels = seek_value_label_names + (33*self.nvar) + 20 + 17
+            # Below is the original, correct code (per Stata sta format doc,
+            # although this is not followed in actual 117 dtas)
+            #seek_variable_labels = struct.unpack(
+            #    self.byteorder + 'q', self.path_or_buf.read(8))[0] + 17
             self.path_or_buf.read(8)  # <characteristics>
             self.data_location = struct.unpack(
                 self.byteorder + 'q', self.path_or_buf.read(8))[0] + 6
diff --git a/pandas/io/tests/data/stata7_115.dta b/pandas/io/tests/data/stata7_115.dta
new file mode 100644
index 0000000000000000000000000000000000000000..133713b201ba8d5f3c8ff716bfa4e61638979da5
GIT binary patch
literal 722
zcmXSBVq{=uU}S)RQ(-X<Fg8d8#%6}flSCOADR`CUC>R+Sng9taQ!|FYjQ`3KiwqeI
z3=DCa1XN&zQxYTrRA3A=!2pjUaVC(Ys->PmIv!ORQmU543?ynG(FTTiSyl{dsVk6^
zn3S3W3|?5&!A!q+Ef6JYS7b<lLkU;Zg7kBOG=mKUMhSHdG^V0~U@wB4m&m}-U=IND
CJ0qz8

literal 0
HcmV?d00001

diff --git a/pandas/io/tests/data/stata7_117.dta b/pandas/io/tests/data/stata7_117.dta
new file mode 100644
index 0000000000000000000000000000000000000000..c001478fc902d22fabade64aee76d85afa3f9786
GIT binary patch
literal 1159
zcmb_c%}xR_5H6w*U}DsZ2^X$*ckw6aq#6@Fh`|`|ZVFT(*%i_j663|2&)^$)_eH$<
zD0;(bw{;gd5F$PFo1brIrqkJi(p>Ynqd9}F;2j|uNRbGx1PjAMpk9I2lNNnx2v@fk
zKywuutW1D57&KXl5EPpX65bLCLpc*b_fmnW(g9nKqRQnksYF4zhuhKq0npqF)K`)6
z0mEo^y*b=4@o^nH{BZc`aI}*7{l2-HNp*+U4(}X3JA8Er)MSMbmp$G}M690%Lb0E5
ziF}WNPMHnuk>jx#3Ly!C(5Ib2Bt$dDq5%mtj1UxU6SB>oWdqfhI_W8m?b)&i2Kr}$
z`@T%MR;*gz4IU;!%u%X!MHc*E!3;osaoD<KC}KMsads*a@2t(K@V{=;!|Am7UE5+5
zvw%10CW;Kn`I(V}ubWeEGljf&w{7pst}5@-rHoHf$F~9T_B~-a0c%SPSeR_r1Za1;
q<ZUe^j#;m*7|<C@2j30*zjH!fPJmh+l$MF1J1jKZSf|wd?fC_2<i_Iw

literal 0
HcmV?d00001

diff --git a/pandas/io/tests/test_stata.py b/pandas/io/tests/test_stata.py
index 435226bc4313f..5271604235922 100644
--- a/pandas/io/tests/test_stata.py
+++ b/pandas/io/tests/test_stata.py
@@ -68,6 +68,9 @@ def setUp(self):
         self.dta15_115 = os.path.join(self.dirpath, 'stata6_115.dta')
         self.dta15_117 = os.path.join(self.dirpath, 'stata6_117.dta')
 
+        self.dta16_115 = os.path.join(self.dirpath, 'stata7_115.dta')
+        self.dta16_117 = os.path.join(self.dirpath, 'stata7_117.dta')
+
     def read_dta(self, file):
         return read_stata(file, convert_dates=True)
 
@@ -199,7 +202,7 @@ def test_read_dta4(self):
                      'labeled_with_missings', 'float_labelled'])
 
         # these are all categoricals
-        expected = pd.concat([ Series(pd.Categorical(value)) for col, value in expected.iteritems() ],axis=1)
+        expected = pd.concat([ Series(pd.Categorical(value)) for col, value in compat.iteritems(expected)],axis=1)
 
         tm.assert_frame_equal(parsed_113, expected)
         tm.assert_frame_equal(parsed_114, expected)
@@ -551,6 +554,18 @@ def test_bool_uint(self):
             written_and_read_again = written_and_read_again.set_index('index')
             tm.assert_frame_equal(written_and_read_again, expected)
 
+    def test_variable_labels(self):
+        sr_115 = StataReader(self.dta16_115).variable_labels()
+        sr_117 = StataReader(self.dta16_117).variable_labels()
+        keys = ('var1', 'var2', 'var3')
+        labels = ('label1', 'label2', 'label3')
+        for k,v in compat.iteritems(sr_115):
+            self.assertTrue(k in sr_117)
+            self.assertTrue(v == sr_117[k])
+            self.assertTrue(k in keys)
+            self.assertTrue(v in labels)
+
+
 if __name__ == '__main__':
     nose.runmodule(argv=[__file__, '-vvs', '-x', '--pdb', '--pdb-failure'],
                    exit=False)