From 5c56bfe07de2e332316636f2dc41bd1c73068177 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 4 Dec 2017 12:12:50 -0500 Subject: [PATCH 1/4] ENH: Profile VMS as well as RSS --- nipype/interfaces/base/core.py | 3 ++- nipype/pipeline/engine/utils.py | 5 +++-- nipype/utils/profiler.py | 13 +++++++++---- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/nipype/interfaces/base/core.py b/nipype/interfaces/base/core.py index f6f9ba655a..2a664c9f48 100644 --- a/nipype/interfaces/base/core.py +++ b/nipype/interfaces/base/core.py @@ -530,8 +530,9 @@ def run(self, **inputs): runtime.prof_dict = { 'time': vals[:, 0].tolist(), - 'mem_gb': (vals[:, 1] / 1024).tolist(), + 'rss_gb': (vals[:, 1] / 1024).tolist(), 'cpus': vals[:, 2].tolist(), + 'vms_gb': (vals[:, 3] / 1024).tolist(), } return results diff --git a/nipype/pipeline/engine/utils.py b/nipype/pipeline/engine/utils.py index 7a730b817c..c87b01869d 100644 --- a/nipype/pipeline/engine/utils.py +++ b/nipype/pipeline/engine/utils.py @@ -1320,7 +1320,8 @@ def write_workflow_resources(graph, filename=None, append=None): 'time': [], 'name': [], 'interface': [], - 'mem_gb': [], + 'rss_gb': [], + 'vms_gb': [], 'cpus': [], 'mapnode': [], 'params': [], @@ -1361,7 +1362,7 @@ def write_workflow_resources(graph, filename=None, append=None): '(mapflow %d/%d).', nodename, subidx + 1, len(rt_list)) continue - for key in ['time', 'mem_gb', 'cpus']: + for key in ['time', 'rss_gb', 'cpus', 'vms_gb']: big_dict[key] += runtime.prof_dict[key] big_dict['interface'] += [classname] * nsamples diff --git a/nipype/utils/profiler.py b/nipype/utils/profiler.py index 82855db43c..76bd328bd5 100644 --- a/nipype/utils/profiler.py +++ b/nipype/utils/profiler.py @@ -67,11 +67,14 @@ def stop(self): def _sample(self, cpu_interval=None): cpu = 0.0 - mem = 0.0 + rss = 0.0 + vms = 0.0 try: with self._process.oneshot(): cpu += self._process.cpu_percent(interval=cpu_interval) - mem += self._process.memory_info().rss + mem_info = self._process.memory_info() + rss += mem_info.rss + vms += mem_info.vms except psutil.NoSuchProcess: pass @@ -85,11 +88,13 @@ def _sample(self, cpu_interval=None): try: with child.oneshot(): cpu += child.cpu_percent() - mem += child.memory_info().rss + mem_info = child.memory_info() + rss += mem_info.rss + vms += mem_info.vms except psutil.NoSuchProcess: pass - print('%f,%f,%f' % (time(), (mem / _MB), cpu), + print('%f,%f,%f,%f' % (time(), rss / _MB, cpu, vms / _MB), file=self._logfile) self._logfile.flush() From a5f48d752e076ead8ea21d31ebc7b9685e6800f2 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 4 Dec 2017 21:39:59 -0500 Subject: [PATCH 2/4] rss/vms_gb -> rss/vms_GiB --- nipype/interfaces/base/core.py | 4 ++-- nipype/pipeline/engine/utils.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nipype/interfaces/base/core.py b/nipype/interfaces/base/core.py index 2a664c9f48..b23f86f2fc 100644 --- a/nipype/interfaces/base/core.py +++ b/nipype/interfaces/base/core.py @@ -530,9 +530,9 @@ def run(self, **inputs): runtime.prof_dict = { 'time': vals[:, 0].tolist(), - 'rss_gb': (vals[:, 1] / 1024).tolist(), + 'rss_GiB': (vals[:, 1] / 1024).tolist(), 'cpus': vals[:, 2].tolist(), - 'vms_gb': (vals[:, 3] / 1024).tolist(), + 'vms_GiB': (vals[:, 3] / 1024).tolist(), } return results diff --git a/nipype/pipeline/engine/utils.py b/nipype/pipeline/engine/utils.py index c87b01869d..57da5b4b24 100644 --- a/nipype/pipeline/engine/utils.py +++ b/nipype/pipeline/engine/utils.py @@ -1320,8 +1320,8 @@ def write_workflow_resources(graph, filename=None, append=None): 'time': [], 'name': [], 'interface': [], - 'rss_gb': [], - 'vms_gb': [], + 'rss_GiB': [], + 'vms_GiB': [], 'cpus': [], 'mapnode': [], 'params': [], @@ -1362,7 +1362,7 @@ def write_workflow_resources(graph, filename=None, append=None): '(mapflow %d/%d).', nodename, subidx + 1, len(rt_list)) continue - for key in ['time', 'rss_gb', 'cpus', 'vms_gb']: + for key in ['time', 'rss_GiB', 'cpus', 'vms_GiB']: big_dict[key] += runtime.prof_dict[key] big_dict['interface'] += [classname] * nsamples From a15e08600798e56bcd616cd81fc8f918e8b1b0c5 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 5 Dec 2017 11:48:05 -0500 Subject: [PATCH 3/4] ENH: Reorder profiler columns --- nipype/interfaces/base/core.py | 4 ++-- nipype/pipeline/engine/utils.py | 2 +- nipype/utils/profiler.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nipype/interfaces/base/core.py b/nipype/interfaces/base/core.py index b23f86f2fc..bcf2656620 100644 --- a/nipype/interfaces/base/core.py +++ b/nipype/interfaces/base/core.py @@ -530,8 +530,8 @@ def run(self, **inputs): runtime.prof_dict = { 'time': vals[:, 0].tolist(), - 'rss_GiB': (vals[:, 1] / 1024).tolist(), - 'cpus': vals[:, 2].tolist(), + 'cpus': vals[:, 1].tolist(), + 'rss_GiB': (vals[:, 2] / 1024).tolist(), 'vms_GiB': (vals[:, 3] / 1024).tolist(), } diff --git a/nipype/pipeline/engine/utils.py b/nipype/pipeline/engine/utils.py index 57da5b4b24..96ba23cd3d 100644 --- a/nipype/pipeline/engine/utils.py +++ b/nipype/pipeline/engine/utils.py @@ -1362,7 +1362,7 @@ def write_workflow_resources(graph, filename=None, append=None): '(mapflow %d/%d).', nodename, subidx + 1, len(rt_list)) continue - for key in ['time', 'rss_GiB', 'cpus', 'vms_GiB']: + for key in ['time', 'cpus', 'rss_GiB', 'vms_GiB']: big_dict[key] += runtime.prof_dict[key] big_dict['interface'] += [classname] * nsamples diff --git a/nipype/utils/profiler.py b/nipype/utils/profiler.py index 76bd328bd5..6788393cef 100644 --- a/nipype/utils/profiler.py +++ b/nipype/utils/profiler.py @@ -94,7 +94,7 @@ def _sample(self, cpu_interval=None): except psutil.NoSuchProcess: pass - print('%f,%f,%f,%f' % (time(), rss / _MB, cpu, vms / _MB), + print('%f,%f,%f,%f' % (time(), cpu, rss / _MB, vms / _MB), file=self._logfile) self._logfile.flush() From e8b7697df23dc942433035691ccef23ff9c4fe1b Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Tue, 5 Dec 2017 12:42:17 -0500 Subject: [PATCH 4/4] ENH: Profile at constant interval --- nipype/utils/profiler.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/nipype/utils/profiler.py b/nipype/utils/profiler.py index 6788393cef..800b68a95f 100644 --- a/nipype/utils/profiler.py +++ b/nipype/utils/profiler.py @@ -100,9 +100,12 @@ def _sample(self, cpu_interval=None): def run(self): """Core monitoring function, called by start()""" + start_time = time() + wait_til = start_time while not self._event.is_set(): self._sample() - self._event.wait(self._freq) + wait_til += self._freq + self._event.wait(max(0, wait_til - time())) # Log node stats function