Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feat: We added the horizon weighing to the distribution losses, simil… #1233

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
+89 −27
Open

Feat: We added the horizon weighing to the distribution losses, simil… #1233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f2c957e
Feat: Introducing horizon weighting to the distribution losses simila…
mwamsojo Dec 19, 2024
2d39cfa
Merge branch 'main' into Weighted-Horizons-Distribution-Loss
mwamsojo Jan 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 58 additions & 26 deletions nbs/losses.pytorch.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2475,7 +2475,7 @@
"\n",
" \"\"\"\n",
" def __init__(self, distribution, level=[80, 90], quantiles=None,\n",
" num_samples=1000, return_params=False, **distribution_kwargs):\n",
" num_samples=1000, return_params=False, horizon_weight = None, **distribution_kwargs):\n",
" super(DistributionLoss, self).__init__()\n",
"\n",
" qs, self.output_names = level_to_outputs(level)\n",
Expand All @@ -2489,6 +2489,12 @@
" self.quantiles = torch.nn.Parameter(qs, requires_grad=False)\n",
" num_qk = len(self.quantiles)\n",
"\n",
" # Generate a horizon weight tensor from the array\n",
" if horizon_weight is not None:\n",
" horizon_weight = torch.Tensor(horizon_weight.flatten())\n",
" self.horizon_weight = horizon_weight\n",
"\n",
"\n",
" if \"num_pieces\" not in distribution_kwargs:\n",
" num_pieces = 5\n",
" else:\n",
Expand Down Expand Up @@ -2610,36 +2616,62 @@
"\n",
" return samples, sample_mean, quants\n",
"\n",
" def __call__(self,\n",
" y: torch.Tensor,\n",
" distr_args: torch.Tensor,\n",
" mask: Union[torch.Tensor, None] = None):\n",
"\n",
"\n",
" def _compute_weights(self, y, mask):\n",
" \"\"\"\n",
" Compute final weights for each datapoint (based on all weights and all masks)\n",
" Set horizon_weight to a ones[H] tensor if not set.\n",
" If set, check that it has the same length as the horizon in x.\n",
" \"\"\"\n",
" Computes the negative log-likelihood objective function. \n",
" To estimate the following predictive distribution:\n",
" if mask is None:\n",
" mask = torch.ones_like(y, device=y.device)\n",
" else:\n",
" mask = mask.unsqueeze(1) # Add Q dimension.\n",
"\n",
" $$\\mathrm{P}(\\mathbf{y}_{\\\\tau}\\,|\\,\\\\theta) \\\\quad \\mathrm{and} \\\\quad -\\log(\\mathrm{P}(\\mathbf{y}_{\\\\tau}\\,|\\,\\\\theta))$$\n",
"\n",
" where $\\\\theta$ represents the distributions parameters. It aditionally \n",
" summarizes the objective signal using a weighted average using the `mask` tensor. \n",
" # get uniform weights if none\n",
" if self.horizon_weight is None:\n",
" self.horizon_weight = torch.ones(mask.shape[-1])\n",
" else:\n",
" assert mask.shape[-1] == len(self.horizon_weight), \\\n",
" 'horizon_weight must have same length as Y'\n",
" weights = self.horizon_weight.clone()\n",
" weights = torch.ones_like(mask, device=mask.device) * weights.to(mask.device)\n",
" return weights * mask\n",
" \n",
"\n",
" **Parameters**<br>\n",
" `y`: tensor, Actual values.<br>\n",
" `distr_args`: Constructor arguments for the underlying Distribution type.<br>\n",
" `loc`: Optional tensor, of the same shape as the batch_shape + event_shape\n",
" of the resulting distribution.<br>\n",
" `scale`: Optional tensor, of the same shape as the batch_shape+event_shape \n",
" of the resulting distribution.<br>\n",
" `mask`: tensor, Specifies date stamps per serie to consider in loss.<br>\n",
"\n",
" **Returns**<br>\n",
" `loss`: scalar, weighted loss function against which backpropagation will be performed.<br>\n",
" \"\"\"\n",
" # Instantiate Scaled Decoupled Distribution\n",
" distr = self.get_distribution(distr_args=distr_args, **self.distribution_kwargs)\n",
" loss_values = -distr.log_prob(y)\n",
" loss_weights = mask\n",
" return weighted_average(loss_values, weights=loss_weights)"
" def __call__(self,\n",
" y: torch.Tensor,\n",
" distr_args: torch.Tensor,\n",
" mask: Union[torch.Tensor, None] = None):\n",
" \"\"\"\n",
" Computes the negative log-likelihood objective function. \n",
" To estimate the following predictive distribution:\n",
"\n",
" $$\\mathrm{P}(\\mathbf{y}_{\\\\tau}\\,|\\,\\\\theta) \\\\quad \\mathrm{and} \\\\quad -\\log(\\mathrm{P}(\\mathbf{y}_{\\\\tau}\\,|\\,\\\\theta))$$\n",
"\n",
" where $\\\\theta$ represents the distributions parameters. It aditionally \n",
" summarizes the objective signal using a weighted average using the `mask` tensor. \n",
" \n",
" **Parameters**<br>\n",
" `y`: tensor, Actual values.<br>\n",
" `distr_args`: Constructor arguments for the underlying Distribution type.<br>\n",
" `loc`: Optional tensor, of the same shape as the batch_shape + event_shape\n",
" of the resulting distribution.<br>\n",
" `scale`: Optional tensor, of the same shape as the batch_shape+event_shape \n",
" of the resulting distribution.<br>\n",
" `mask`: tensor, Specifies date stamps per serie to consider in loss.<br>\n",
"\n",
" **Returns**<br>\n",
" `loss`: scalar, weighted loss function against which backpropagation will be performed.<br>\n",
" \"\"\"\n",
" # Instantiate Scaled Decoupled Distribution\n",
" distr = self.get_distribution(distr_args=distr_args, **self.distribution_kwargs)\n",
" loss_values = -distr.log_prob(y)\n",
" loss_weights = self._compute_weights(y=y, mask=mask)\n",
" return weighted_average(loss_values, weights=loss_weights)"
]
},
{
Expand Down
2 changes: 2 additions & 0 deletions neuralforecast/_modidx.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,8 @@
'neuralforecast/losses/pytorch.py'),
'neuralforecast.losses.pytorch.DistributionLoss.__init__': ( 'losses.pytorch.html#distributionloss.__init__',
'neuralforecast/losses/pytorch.py'),
'neuralforecast.losses.pytorch.DistributionLoss._compute_weights': ( 'losses.pytorch.html#distributionloss._compute_weights',
'neuralforecast/losses/pytorch.py'),
'neuralforecast.losses.pytorch.DistributionLoss.get_distribution': ( 'losses.pytorch.html#distributionloss.get_distribution',
'neuralforecast/losses/pytorch.py'),
'neuralforecast.losses.pytorch.DistributionLoss.sample': ( 'losses.pytorch.html#distributionloss.sample',
Expand Down
30 changes: 29 additions & 1 deletion neuralforecast/losses/pytorch.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1873,6 +1873,7 @@ def __init__(
quantiles=None,
num_samples=1000,
return_params=False,
horizon_weight=None,
**distribution_kwargs,
):
super(DistributionLoss, self).__init__()
Expand All @@ -1888,6 +1889,11 @@ def __init__(
self.quantiles = torch.nn.Parameter(qs, requires_grad=False)
num_qk = len(self.quantiles)

# Generate a horizon weight tensor from the array
if horizon_weight is not None:
horizon_weight = torch.Tensor(horizon_weight.flatten())
self.horizon_weight = horizon_weight

if "num_pieces" not in distribution_kwargs:
num_pieces = 5
else:
Expand Down Expand Up @@ -2011,6 +2017,28 @@ def sample(self, distr_args: torch.Tensor, num_samples: Optional[int] = None):

return samples, sample_mean, quants

def _compute_weights(self, y, mask):
"""
Compute final weights for each datapoint (based on all weights and all masks)
Set horizon_weight to a ones[H] tensor if not set.
If set, check that it has the same length as the horizon in x.
"""
if mask is None:
mask = torch.ones_like(y, device=y.device)
else:
mask = mask.unsqueeze(1) # Add Q dimension.

# get uniform weights if none
if self.horizon_weight is None:
self.horizon_weight = torch.ones(mask.shape[-1])
else:
assert mask.shape[-1] == len(
self.horizon_weight
), "horizon_weight must have same length as Y"
weights = self.horizon_weight.clone()
weights = torch.ones_like(mask, device=mask.device) * weights.to(mask.device)
return weights * mask

def __call__(
self,
y: torch.Tensor,
Expand Down Expand Up @@ -2041,7 +2069,7 @@ def __call__(
# Instantiate Scaled Decoupled Distribution
distr = self.get_distribution(distr_args=distr_args, **self.distribution_kwargs)
loss_values = -distr.log_prob(y)
loss_weights = mask
loss_weights = self._compute_weights(y=y, mask=mask)
return weighted_average(loss_values, weights=loss_weights)

# %% ../../nbs/losses.pytorch.ipynb 74
Expand Down