|
| 1 | +# -*- coding: utf-8 -*- |
| 2 | +""" |
| 3 | +Introduction to Holistic Trace Analysis |
| 4 | +------------ |
| 5 | +**Author:** `Anupam Bhatnagar <https://github.com/anupambhatnagar>`_ |
| 6 | +
|
| 7 | +.. note:: |
| 8 | + Visualizations have been set to False to keep the notebook size small. When |
| 9 | + running the notebook locally set the visualize variable to True to display |
| 10 | + the plots. |
| 11 | +
|
| 12 | +""" |
| 13 | + |
| 14 | +############################################################## |
| 15 | +# Setup and loading traces |
| 16 | +# ~~~~~~~~~~~~~~~~~~~~~~~~ |
| 17 | +# |
| 18 | +# In this demo we analyze the traces from a distributed training job which used 8 GPUs. To run the code on your laptop: |
| 19 | +# |
| 20 | +# 1) Install Holistic Trace Analysis via pip. `pip install HolisticTraceAnalysis` |
| 21 | +# 2) [Optional and recommended] Setup a conda environment. See here for details. |
| 22 | +# 3) Edit the `hta_install_dir` vairable below to the folder in your local `HolisticTraceAnalysis` installation. |
| 23 | + |
| 24 | +from hta.trace_analysis import TraceAnalysis |
| 25 | +hta_install_dir = "/path/to/HolisticTraceAnalysis" |
| 26 | +trace_dir = hta_install_dir + "/tests/data/vision_transformer/" |
| 27 | +analyzer = TraceAnalysis(trace_dir=trace_dir) |
| 28 | + |
| 29 | + |
| 30 | +############################################################## |
| 31 | +# Temporal Breakdown |
| 32 | +# ~~~~~~~~~~~~~~~~~~ |
| 33 | +# |
| 34 | +# The temporal breakdown feature gives a breakdown of time spent by the GPU as follows: |
| 35 | +# |
| 36 | +# 1) Idle time - GPU idle |
| 37 | +# 2) Compute time - GPU busy with computation events |
| 38 | +# 3) Non compute time - GPU busy with communication or memory events |
| 39 | + |
| 40 | +time_spent_df = analyzer.get_temporal_breakdown(visualize=False) |
| 41 | +print(time_spent_df) |
| 42 | + |
| 43 | + |
| 44 | +############################################################## |
| 45 | +# Kernel Breakdown |
| 46 | +# ~~~~~~~~~~~~~~~~ |
| 47 | +# |
| 48 | +# This feature computes the following: |
| 49 | +# |
| 50 | +# 1) Breakdown of time spent among kernel types (Computation, Communication, Memory) across all ranks. |
| 51 | +# 2) Kernels taking the most time on each rank by kernel type. |
| 52 | +# 3) Distribution of average time across ranks for the kernels taking the most time. |
0 commit comments