Merge pull request #8331 from jreback/concat_copy

PERF: add copy=True argument to pd.concat to enable pass-thru concats with complete blocks (GH8252)

Author: jreback

doc/source/v0.15.0.txt

@@ -276,6 +276,7 @@ API changes
      Index(['a','b','c']).difference(Index(['b','c','d']))
 
 - ``DataFrame.info()`` now ends its output with a newline character (:issue:`8114`)
+- add ``copy=True`` argument to ``pd.concat`` to enable pass thrue of complete blocks (:issue:`8252`)
 
 .. _whatsnew_0150.dt:
 

pandas/tools/merge.py

@@ -666,7 +666,7 @@ def _sort_labels(uniques, left, right):
 
 
 def concat(objs, axis=0, join='outer', join_axes=None, ignore_index=False,
-           keys=None, levels=None, names=None, verify_integrity=False):
+           keys=None, levels=None, names=None, verify_integrity=False, copy=True):
     """
     Concatenate pandas objects along a particular axis with optional set logic
     along the other axes. Can also add a layer of hierarchical indexing on the
@@ -704,6 +704,8 @@ def concat(objs, axis=0, join='outer', join_axes=None, ignore_index=False,
         concatenating objects where the concatenation axis does not have
         meaningful indexing information. Note the the index values on the other
         axes are still respected in the join.
+    copy : boolean, default True
+        If False, do not copy data unnecessarily
 
     Notes
     -----
@@ -716,7 +718,8 @@ def concat(objs, axis=0, join='outer', join_axes=None, ignore_index=False,
     op = _Concatenator(objs, axis=axis, join_axes=join_axes,
                        ignore_index=ignore_index, join=join,
                        keys=keys, levels=levels, names=names,
-                       verify_integrity=verify_integrity)
+                       verify_integrity=verify_integrity,
+                       copy=copy)
     return op.get_result()
 
 
@@ -727,7 +730,7 @@ class _Concatenator(object):
 
     def __init__(self, objs, axis=0, join='outer', join_axes=None,
                  keys=None, levels=None, names=None,
-                 ignore_index=False, verify_integrity=False):
+                 ignore_index=False, verify_integrity=False, copy=True):
         if not isinstance(objs, (list,tuple,types.GeneratorType,dict,TextFileReader)):
             raise TypeError('first argument must be a list-like of pandas '
                             'objects, you passed an object of type '
@@ -846,6 +849,7 @@ def __init__(self, objs, axis=0, join='outer', join_axes=None,
 
         self.ignore_index = ignore_index
         self.verify_integrity = verify_integrity
+        self.copy = copy
 
         self.new_axes = self._get_new_axes()
 
@@ -879,7 +883,9 @@ def get_result(self):
                 mgrs_indexers.append((obj._data, indexers))
 
             new_data = concatenate_block_managers(
-                mgrs_indexers, self.new_axes, concat_axis=self.axis, copy=True)
+                mgrs_indexers, self.new_axes, concat_axis=self.axis, copy=self.copy)
+            if not self.copy:
+                new_data._consolidate_inplace()
 
             return self.objs[0]._from_axes(new_data, self.new_axes).__finalize__(self, method='concat')
 

pandas/tools/tests/test_merge.py

@@ -1485,6 +1485,38 @@ def test_append_missing_column_proper_upcast(self):
         self.assertEqual(appended['A'].dtype, 'f8')
         self.assertEqual(appended['B'].dtype, 'O')
 
+    def test_concat_copy(self):
+
+        df = DataFrame(np.random.randn(4, 3))
+        df2 = DataFrame(np.random.randint(0,10,size=4).reshape(4,1))
+        df3 = DataFrame({5 : 'foo'},index=range(4))
+
+        # these are actual copies
+        result = concat([df,df2,df3],axis=1,copy=True)
+        for b in result._data.blocks:
+            self.assertIsNone(b.values.base)
+
+        # these are the same
+        result = concat([df,df2,df3],axis=1,copy=False)
+        for b in result._data.blocks:
+            if b.is_float:
+                self.assertTrue(b.values.base is df._data.blocks[0].values.base)
+            elif b.is_integer:
+                self.assertTrue(b.values.base is df2._data.blocks[0].values.base)
+            elif b.is_object:
+                self.assertIsNotNone(b.values.base)
+
+        # float block was consolidated
+        df4 = DataFrame(np.random.randn(4,1))
+        result = concat([df,df2,df3,df4],axis=1,copy=False)
+        for b in result._data.blocks:
+            if b.is_float:
+                self.assertIsNone(b.values.base)
+            elif b.is_integer:
+                self.assertTrue(b.values.base is df2._data.blocks[0].values.base)
+            elif b.is_object:
+                self.assertIsNotNone(b.values.base)
+
     def test_concat_with_group_keys(self):
         df = DataFrame(np.random.randn(4, 3))
         df2 = DataFrame(np.random.randn(4, 4))