Use proper types so that we work with python3

Author: cpcloud

doc/source/whatsnew/v0.16.1.txt

@@ -171,7 +171,7 @@ Performance Improvements
 
 - Improved csv write performance with mixed dtypes, including datetimes by up to 5x (:issue:`9940`)
 - Improved csv write performance generally by 2x (:issue:`9940`)
-
+- Improved the performance of ``pd.lib.max_len_string_array`` by 5-7x (:issue:`10024`)
 
 
 

pandas/lib.pyx

@@ -1,6 +1,7 @@
 cimport numpy as np
 cimport cython
 import numpy as np
+import sys
 
 from numpy cimport *
 
@@ -10,6 +11,7 @@ cdef extern from "numpy/arrayobject.h":
     cdef enum NPY_TYPES:
         NPY_intp "NPY_INTP"
 
+
 from cpython cimport (PyDict_New, PyDict_GetItem, PyDict_SetItem,
                       PyDict_Contains, PyDict_Keys,
                       Py_INCREF, PyTuple_SET_ITEM,
@@ -19,10 +21,14 @@ from cpython cimport (PyDict_New, PyDict_GetItem, PyDict_SetItem,
                       PyTuple_SetItem,
                       PyTuple_New,
                       PyObject_SetAttrString,
-                      PyString_GET_SIZE,
                       PyBytes_GET_SIZE,
                       PyUnicode_GET_SIZE)
 
+try:
+    from cpython cimport PyString_GET_SIZE
+except ImportError:
+    from cpython cimport PyUnicode_GET_SIZE as PyString_GET_SIZE
+
 cdef extern from "Python.h":
     Py_ssize_t PY_SSIZE_T_MAX
 
@@ -35,7 +41,6 @@ cdef extern from "Python.h":
         Py_ssize_t *slicelength) except -1
 
 
-
 cimport cpython
 
 isnan = np.isnan
@@ -903,6 +908,7 @@ def clean_index_list(list obj):
 ctypedef fused pandas_t:
     str
     unicode
+    bytes
 
 
 @cython.boundscheck(False)

pandas/tests/test_lib.py

@@ -8,22 +8,29 @@
 import pandas.util.testing as tm
 from pandas.compat import u
 
+
 class TestMisc(tm.TestCase):
 
     def test_max_len_string_array(self):
 
-        arr = np.array(['foo','b',np.nan],dtype='object')
-        self.assertTrue(max_len_string_array(arr),3)
+        arr = a = np.array(['foo', 'b', np.nan], dtype='object')
+        self.assertTrue(max_len_string_array(arr), 3)
 
         # unicode
-        arr = arr.astype('U').astype(object)
-        self.assertTrue(max_len_string_array(arr),3)
+        arr = a.astype('U').astype(object)
+        self.assertTrue(max_len_string_array(arr), 3)
+
+        # bytes for python3
+        arr = a.astype('S').astype(object)
+        self.assertTrue(max_len_string_array(arr), 3)
 
         # raises
         tm.assertRaises(TypeError,
                         lambda: max_len_string_array(arr.astype('U')))
 
+
 class TestIsscalar(tm.TestCase):
+
     def test_isscalar_builtin_scalars(self):
         self.assertTrue(isscalar(None))
         self.assertTrue(isscalar(True))