BUG: do not coerce types in parser if converters specified #2184

Author: changhiskhan

pandas/io/parsers.py

@@ -847,13 +847,15 @@ def get_chunk(self, rows=None):
         if self.parse_dates is not None:
             data, columns = self._process_date_conversion(data)
 
-        # apply converters
+        #converters to column names
+        clean_conv = {}
         for col, f in self.converters.iteritems():
             if isinstance(col, int) and col not in self.orig_columns:
                 col = self.orig_columns[col]
-            data[col] = lib.map_infer(data[col], f)
+            clean_conv[col] = f
 
-        data = _convert_to_ndarrays(data, self.na_values, self.verbose)
+        data = _convert_to_ndarrays(data, self.na_values, self.verbose,
+                                    clean_conv)
 
         if self.index_col is None:
             numrows = len(content)
@@ -1141,18 +1143,23 @@ def _get_na_values(col, na_values):
         return na_values
 
 
-def _convert_to_ndarrays(dct, na_values, verbose=False):
+def _convert_to_ndarrays(dct, na_values, verbose=False, converters=None):
     result = {}
     for c, values in dct.iteritems():
+        conv_f = None if converters is None else converters.get(c, None)
         col_na_values = _get_na_values(c, na_values)
-        cvals, na_count = _convert_types(values, col_na_values)
+        coerce_type = True
+        if conv_f is not None:
+            values = lib.map_infer(values, conv_f)
+            coerce_type = False
+        cvals, na_count = _convert_types(values, col_na_values, coerce_type)
         result[c] = cvals
         if verbose and na_count:
             print 'Filled %d NA values in column %s' % (na_count, str(c))
     return result
 
 
-def _convert_types(values, na_values):
+def _convert_types(values, na_values, try_num_bool=True):
     na_count = 0
     if issubclass(values.dtype.type, (np.number, np.bool_)):
         mask = lib.ismember(values, na_values)
@@ -1163,13 +1170,17 @@ def _convert_types(values, na_values):
             np.putmask(values, mask, np.nan)
         return values, na_count
 
-    try:
-        result = lib.maybe_convert_numeric(values, na_values, False)
-    except Exception:
+    if try_num_bool:
+        try:
+            result = lib.maybe_convert_numeric(values, na_values, False)
+        except Exception:
+            na_count = lib.sanitize_objects(values, na_values, False)
+            result = values
+    else:
         na_count = lib.sanitize_objects(values, na_values, False)
         result = values
 
-    if result.dtype == np.object_:
+    if result.dtype == np.object_ and try_num_bool:
         result = lib.maybe_convert_bool(values)
 
     return result, na_count
@@ -1352,7 +1363,7 @@ def parse(self, sheetname, header=0, skiprows=None, skip_footer=0,
                                      skip_footer=skip_footer)
 
     def _should_parse(self, i, parse_cols):
-        
+
         def _range2cols(areas):
             """
             Convert comma separated list of column names and column ranges to a
@@ -1363,10 +1374,10 @@ def _range2cols(areas):
             >>> _range2cols('A,C,Z:AB')
             [0, 2, 25, 26, 27]
             """
-            def _excel2num(x): 
+            def _excel2num(x):
                 "Convert Excel column name like 'AB' to 0-based column index"
                 return reduce(lambda s,a: s*26+ord(a)-ord('A')+1, x.upper().strip(), 0)-1
-            
+
             cols = []
             for rng in areas.split(','):
                 if ':' in rng:

pandas/io/tests/test_parsers.py

@@ -1259,6 +1259,14 @@ def test_converters(self):
         expected['D'] = expected['D'].map(converter)
         assert_frame_equal(result, expected)
 
+    def test_converters_no_implicit_conv(self):
+        #GH2184
+        data = """000102,1.2,A\n001245,2,B"""
+        f = lambda x: x.strip()
+        converter = {0: f}
+        df = read_csv(StringIO(data), header=None, converters=converter)
+        self.assert_(df.X0.dtype == object)
+
     def test_converters_euro_decimal_format(self):
         data = """Id;Number1;Number2;Text1;Text2;Number3
 1;1521,1541;187101,9543;ABC;poi;4,738797819
@@ -1277,7 +1285,7 @@ def test_converter_return_string_bug(self):
 1;1521,1541;187101,9543;ABC;poi;4,738797819
 2;121,12;14897,76;DEF;uyt;0,377320872
 3;878,158;108013,434;GHI;rez;2,735694704"""
-        f = lambda x : x.replace(",", ".")
+        f = lambda x : float(x.replace(",", "."))
         converter = {'Number1':f,'Number2':f, 'Number3':f}
         df2 = read_csv(StringIO(data), sep=';',converters=converter)
         self.assert_(df2['Number1'].dtype == float)