Skip to content

Commit

Permalink
Log UVM cache load factor to stdout and manifold (pytorch#1527)
Browse files Browse the repository at this point in the history
Summary:
Pull Request resolved: pytorch#1527

Log cache load factor information in torchrec planner.stats logger.

Reviewed By: henrylhtsang, levythu

Differential Revision: D51452269

fbshipit-source-id: 6fcf06a1ec26fca29f243cd35f8cf75248e08436
  • Loading branch information
Esther Zhou authored and facebook-github-bot committed Nov 22, 2023
1 parent 853440e commit d355caf
Showing 1 changed file with 17 additions and 0 deletions.
17 changes: 17 additions & 0 deletions torchrec/distributed/planner/stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,7 @@ def log(
"Compute Kernel",
"Perf (ms)",
"Storage (HBM, DDR)",
"Cache Load Factor",
"Pooling Factor",
"Num Poolings",
"Output",
Expand All @@ -263,6 +264,7 @@ def log(
"----------------",
"-----------",
"--------------------",
"-------------------",
"----------------",
"--------------",
"--------",
Expand Down Expand Up @@ -327,6 +329,10 @@ def log(
or so.sharding_type == ShardingType.TABLE_COLUMN_WISE.value
else f"{so.tensor.shape[1]}"
)
cache_load_factor = _get_cache_load_factor(sharding_option=so)
cache_load_factor = (
str(cache_load_factor) if cache_load_factor is not None else "None"
)
hash_size = so.tensor.shape[0]
param_table.append(
[
Expand All @@ -335,6 +341,7 @@ def log(
so.compute_kernel,
shard_perfs,
shard_storages,
cache_load_factor,
pooling_factor,
num_poolings,
output,
Expand Down Expand Up @@ -586,6 +593,16 @@ def _generate_max_text(perfs: List[float]) -> str:
return f"{round(max_perf, 3)} ms on {max_perf_ranks}"


def _get_cache_load_factor(
sharding_option: ShardingOption,
) -> Optional[float]:
return (
sharding_option.cache_params.load_factor
if sharding_option.cache_params
else None
)


def _get_sharding_type_abbr(sharding_type: str) -> str:
if sharding_type == ShardingType.DATA_PARALLEL.value:
return "DP"
Expand Down

0 comments on commit d355caf

Please sign in to comment.