REF: fix c-parser according to api decisions in master

Author: wesm

pandas/io/parsers.py

@@ -1123,26 +1123,15 @@ def read(self, rows=None):
 
     def _convert_data(self, data):
         # apply converters
-        converted = set()
+        clean_conv = {}
+
         for col, f in self.converters.iteritems():
             if isinstance(col, int) and col not in self.orig_names:
                 col = self.orig_names[col]
-            data[col] = lib.map_infer(data[col], f)
-            converted.add(col)
-
-        # do type conversions
-        result = {}
-        for c, values in data.iteritems():
-            if c in converted:
-                result[c] = values
-
-            col_na_values = _get_na_values(c, self.na_values)
-            cvals, na_count = _convert_types(values, col_na_values)
-            result[c] = cvals
-            if self.verbose and na_count:
-                print 'Filled %d NA values in column %s' % (na_count, str(c))
-
-        return result
+            clean_conv[col] = f
+
+        return _convert_to_ndarrays(data, self.na_values, self.verbose,
+                                    clean_conv)
 
     def _infer_columns(self):
         names = self.names

pandas/io/tests/test_parsers.py

@@ -1120,7 +1120,7 @@ def test_converters_no_implicit_conv(self):
         data = """000102,1.2,A\n001245,2,B"""
         f = lambda x: x.strip()
         converter = {0: f}
-        df = read_csv(StringIO(data), header=None, converters=converter)
+        df = self.read_csv(StringIO(data), header=None, converters=converter)
         self.assert_(df.X0.dtype == object)
 
     def test_converters_euro_decimal_format(self):

pandas/src/parser.pyx

@@ -3,7 +3,7 @@
 
 from libc.stdio cimport fopen, fclose
 from libc.stdlib cimport malloc, free
-from libc.string cimport strncpy, strlen
+from libc.string cimport strncpy, strlen, strcmp
 cimport libc.stdio as stdio
 
 from cpython cimport (PyObject, PyBytes_FromString,
@@ -44,6 +44,9 @@ import sys
 
 cdef bint PY3 = (sys.version_info[0] >= 3)
 
+cdef double INF = <double> np.inf
+cdef double NEGINF = -INF
+
 cdef extern from "stdint.h":
     enum: UINT8_MAX
     enum: UINT16_MAX
@@ -458,7 +461,7 @@ cdef class TextReader:
 
             if self.memory_map:
                 ptr = new_mmap(source)
-                if ptr == NULL:         
+                if ptr == NULL:
                     # fall back
                     ptr = new_file_source(source, self.parser.chunksize)
                     self.parser.cb_io = &buffer_file_bytes
@@ -1152,6 +1155,8 @@ cdef _to_fw_string(parser_t *parser, int col, int line_start,
 
     return result
 
+cdef char* cinf = b'inf'
+cdef char* cneginf = b'-inf'
 
 cdef _try_double(parser_t *parser, int col, int line_start, int line_end,
                  bint na_filter, kh_str_t *na_hashset):
@@ -1182,14 +1187,24 @@ cdef _try_double(parser_t *parser, int col, int line_start, int line_end,
             else:
                 error = to_double(word, data, parser.sci, parser.decimal)
                 if error != 1:
-                    return None, None
+                    if strcmp(word, cinf) == 0:
+                        data[0] = INF
+                    elif strcmp(word, cneginf) == 0:
+                        data[0] = NEGINF
+                    else:
+                        return None, None
             data += 1
     else:
         for i in range(lines):
             word = COLITER_NEXT(it)
             error = to_double(word, data, parser.sci, parser.decimal)
             if error != 1:
-                return None, None
+                if strcmp(word, cinf) == 0:
+                    data[0] = INF
+                elif strcmp(word, cneginf) == 0:
+                    data[0] = NEGINF
+                else:
+                    return None, None
             data += 1
 
     return result, na_count
@@ -1492,18 +1507,18 @@ cdef _apply_converter(object f, parser_t *parser, int col,
                                    c_encoding, errors)
             result[i] = f(val)
 
-    values = lib.maybe_convert_objects(result)
+    return lib.maybe_convert_objects(result)
 
-    if issubclass(values.dtype.type, (np.number, np.bool_)):
-        return values
+    # if issubclass(values.dtype.type, (np.number, np.bool_)):
+    #     return values
 
-    # XXX
-    na_values = set([''])
-    try:
-        return lib.maybe_convert_numeric(values, na_values, False)
-    except Exception:
-        na_count = lib.sanitize_objects(values, na_values, False)
-        return result
+    # # XXX
+    # na_values = set([''])
+    # try:
+    #     return lib.maybe_convert_numeric(values, na_values, False)
+    # except Exception:
+    #     na_count = lib.sanitize_objects(values, na_values, False)
+    #     return result
 
 def _to_structured_array(dict columns, object names):
     cdef: