PERF: improved performance of pd.concat, by not forcing C ordering when testing for isnull, #119678

Author: navreetaft

asv_bench/benchmarks/join_merge.py

@@ -91,6 +91,59 @@ def time_concat_small_frames(self):
         concat(([self.df] * 1000))
 
 
+class concat_panels(object):
+    goal_time = 0.2
+
+    def setup(self):
+        dataset = np.zeros((10000, 200, 2), dtype=np.float32)
+        self.panels_f = [pd.Panel(np.copy(dataset, order='F'))
+                         for i in range(20)]
+        self.panels_c = [pd.Panel(np.copy(dataset, order='C'))
+                         for i in range(20)]
+
+    def time_concat_c_ordered_axis0(self):
+        concat(self.panels_c, axis=0, ignore_index=True)
+
+    def time_concat_f_ordered_axis0(self):
+        concat(self.panels_f, axis=0, ignore_index=True)
+
+    def time_concat_c_ordered_axis1(self):
+        concat(self.panels_c, axis=1, ignore_index=True)
+
+    def time_concat_f_ordered_axis1(self):
+        concat(self.panels_f, axis=1, ignore_index=True)
+
+    def time_concat_c_ordered_axis2(self):
+        concat(self.panels_c, axis=2, ignore_index=True)
+
+    def time_concat_f_ordered_axis2(self):
+        concat(self.panels_f, axis=2, ignore_index=True)
+
+
+class concat_dataframes(object):
+    goal_time = 0.2
+
+    def setup(self):
+        dataset = np.zeros((10000, 200), dtype=np.float32)
+
+        self.frames_f = [pd.DataFrame(np.copy(dataset, order='F'))
+                         for i in range(20)]
+        self.frames_c = [pd.DataFrame(np.copy(dataset, order='C'))
+                         for i in range(20)]
+
+    def time_concat_c_ordered_axis0(self):
+        concat(self.frames_c, axis=0, ignore_index=True)
+
+    def time_concat_f_ordered_axis0(self):
+        concat(self.frames_f, axis=0, ignore_index=True)
+
+    def time_concat_c_ordered_axis1(self):
+        concat(self.frames_c, axis=1, ignore_index=True)
+
+    def time_concat_f_ordered_axis1(self):
+        concat(self.frames_f, axis=1, ignore_index=True)
+
+
 class i8merge(object):
     goal_time = 0.2
 
@@ -356,4 +409,4 @@ def time_series_align_left_monotonic(self):
 
     def sample(self, values, k):
         self.sampler = np.random.permutation(len(values))
-        return values.take(self.sampler[:k])
+        return values.take(self.sampler[:k])

asv_bench/benchmarks/plotting.py

@@ -24,8 +24,8 @@ class plot_andrews_curves(object):
     def setup(self):
         self.N = 500
         self.M = 10
-	data_dict = {x: np.random.randn(self.N) for x in range(self.M)}
-	data_dict["Name"] = ["A"] * self.N
+        data_dict = {x: np.random.randn(self.N) for x in range(self.M)}
+        data_dict["Name"] = ["A"] * self.N
         self.df = DataFrame(data_dict)
 
     def time_plot_andrews_curves(self):

doc/source/whatsnew/v0.18.0.txt

@@ -413,6 +413,7 @@ Performance Improvements
 - Improved performance of ``andrews_curves`` (:issue:`11534`)
 
 - Improved huge ``DatetimeIndex``, ``PeriodIndex`` and ``TimedeltaIndex``'s ops performance including ``NaT`` (:issue:`10277`)
+- Improved performance of ``pandas.concat`` (:issue:`11958`)
 
 
 

pandas/core/internals.py

@@ -4769,7 +4769,7 @@ def is_null(self):
         if self.block.is_categorical:
             values_flat = values.categories
         else:
-            values_flat = values.ravel()
+            values_flat = values.ravel(order='K')
         total_len = values_flat.shape[0]
         chunk_len = max(total_len // 40, 1000)
         for i in range(0, total_len, chunk_len):