BUG: fix integer overflow possibility in python parser. close #2601

Author: wesm

pandas/io/tests/test_parsers.py

@@ -1492,6 +1492,16 @@ def test_header_names_backward_compat(self):
                                header=0)
         tm.assert_frame_equal(result, expected)
 
+    def test_integer_overflow_bug(self):
+        # #2601
+        data = "65248E10 11\n55555E55 22\n"
+
+        result = self.read_csv(StringIO(data), header=None, sep=' ')
+        self.assertTrue(result[0].dtype == np.float64)
+
+        result = self.read_csv(StringIO(data), header=None, sep='\s+')
+        self.assertTrue(result[0].dtype == np.float64)
+
 
 class TestPythonParser(ParserTests, unittest.TestCase):
 

pandas/lib.pyx

@@ -41,6 +41,8 @@ from util cimport is_array, _checknull, _checknan
 
 cdef extern from "headers/stdint.h":
     enum: UINT8_MAX
+    enum: INT64_MAX
+    enum: INT64_MIN
 
 
 cdef extern from "math.h":

pandas/src/inference.pyx

@@ -298,6 +298,8 @@ def is_period_array(ndarray[object] values):
 cdef extern from "parse_helper.h":
     inline int floatify(object, double *result) except -1
 
+cdef double fINT64_MAX = <double> INT64_MAX
+cdef double fINT64_MIN = <double> INT64_MIN
 
 def maybe_convert_numeric(ndarray[object] values, set na_values,
                           convert_empty=True):
@@ -351,8 +353,10 @@ def maybe_convert_numeric(ndarray[object] values, set na_values,
                     seen_float = 1
                 elif 'inf' in val:  # special case to handle +/-inf
                     seen_float = 1
-                else:
+                elif fval < fINT64_MAX and fval > fINT64_MIN:
                     ints[i] = <int64_t> fval
+                else:
+                    seen_float = 1
 
     if seen_complex:
         return complexes