ENH: allow user to infer SAS file encoding; use detected encoding; add correct encoding names

Author: YYYasin19

pandas/io/sas/sas7bdat.py

@@ -147,8 +147,10 @@ class SAS7BDATReader(ReaderBase, abc.Iterator):
     chunksize : int, defaults to None
         Return SAS7BDATReader object for iterations, returns chunks
         with given number of lines.
-    encoding : string, defaults to None
-        String encoding.
+    encoding : str, 'infer', defaults to None
+        String encoding acc. to python standard encodings,
+        encoding='infer' tries to detect the encoding from the file header,
+        encoding=None will leave the data in binary format.
     convert_text : bool, defaults to True
         If False, text variables are left as raw bytes.
     convert_header_text : bool, defaults to True
@@ -265,9 +267,11 @@ def _get_properties(self) -> None:
         # Get encoding information
         buf = self._read_bytes(const.encoding_offset, const.encoding_length)[0]
         if buf in const.encoding_names:
-            self.file_encoding = const.encoding_names[buf]
+            self.inferred_encoding = const.encoding_names[buf]
+            if self.encoding == "infer":
+                self.encoding = self.inferred_encoding
         else:
-            self.file_encoding = f"unknown (code={buf})"
+            self.inferred_encoding = f"unknown (code={buf})"
 
         # Get platform information
         buf = self._read_bytes(const.platform_offset, const.platform_length)

pandas/io/sas/sas_constants.py

@@ -107,14 +107,21 @@
 compression_literals: Final = [rle_compression, rdc_compression]
 
 # Incomplete list of encodings, using SAS nomenclature:
-# http://support.sas.com/documentation/cdl/en/nlsref/61893/HTML/default/viewer.htm#a002607278.htm
+# https://support.sas.com/documentation/onlinedoc/dfdmstudio/2.6/dmpdmsug/Content/dfU_Encodings_SAS.html
+# corresponding to the Python documentation of standard encodings
+# https://docs.python.org/3/library/codecs.html#standard-encodings
 encoding_names: Final = {
-    29: "latin1",
     20: "utf-8",
+    29: "latin1",
+    30: "latin2",
+    31: "latin3",
+    32: "latin4",
     33: "cyrillic",
-    60: "wlatin2",
-    61: "wcyrillic",
-    62: "wlatin1",
+    34: "arabic",
+    35: "greek",
+    60: "cp1250",  # wlatin2
+    61: "cp1251",  # wcyrillic
+    62: "cp1252",  # wlatin1
     90: "ebcdic870",
 }
 

pandas/tests/io/sas/test_sas7bdat.py

@@ -136,6 +136,28 @@ def test_encoding_options(datapath):
         assert x == y.decode()
 
 
+def test_encoding_infer(datapath):
+    from pandas.io.sas.sas7bdat import SAS7BDATReader
+
+    fname = datapath("io", "sas", "data", "test1.sas7bdat")
+
+    # check if inferred correctly
+    df1_reader: SAS7BDATReader = pd.read_sas(fname, encoding="infer", iterator=True)
+    assert (
+        df1_reader.inferred_encoding == "cp1252"
+    ), f"""
+        Encoding has been inferred incorrectly:
+        {df1_reader.inferred_encoding} instead of 'cp1252'
+    """
+
+    # check if the reader reads correctly with encoding
+    df1 = df1_reader.read()
+    df2_reader: SAS7BDATReader = pd.read_sas(fname, encoding="cp1252", iterator=True)
+    df2 = df2_reader.read()
+
+    tm.assert_frame_equal(df1, df2)
+
+
 def test_productsales(datapath):
     fname = datapath("io", "sas", "data", "productsales.sas7bdat")
     df = pd.read_sas(fname, encoding="utf-8")