core.objective.McAllesterObjective 

 1import numpy as np
 2import torch
 3from torch import Tensor
 4
 5from core.objective import AbstractObjective
 6
 7
 8class McAllesterObjective(AbstractObjective):
 9    """
10    McAllester bound objective (based on McAllester, 1999 and related works),
11    combining empirical loss with a square-root term involving KL and delta,
12    plus additional constants (e.g., 5/2 ln(n)) in the bounding expression.
13    """
14
15    def __init__(self, kl_penalty: float, delta: float):
16        """
17        Args:
18            kl_penalty (float): Multiplier for the KL divergence term.
19            delta (float): Confidence parameter in the PAC-Bayes bound.
20        """
21        self._kl_penalty = kl_penalty
22        self._delta = delta  # confidence value for the training objective
23
24    def calculate(self, loss: Tensor, kl: Tensor, num_samples: float) -> Tensor:
25        """
26        Compute the McAllester objective.
27
28        Derived from a PAC-Bayes bound that includes terms like ln(num_samples)
29        and -ln(delta).
30
31        Args:
32            loss (Tensor): Empirical loss or risk.
33            kl (Tensor): KL divergence.
34            num_samples (float): Number of samples/training size.
35
36        Returns:
37            Tensor: A scalar objective = loss + sqrt( [kl_penalty*KL + 5/2 ln(n) - ln(delta) + 8 ] / [2n - 1] ).
38        """
39        kl = kl * self._kl_penalty
40        kl_ratio = torch.div(
41            kl + 5 / 2 * np.log(num_samples) - np.log(self._delta) + 8,
42            2 * num_samples - 1,
43        )
44        return loss + torch.sqrt(kl_ratio)
class McAllesterObjective(core.objective.AbstractObjective.AbstractObjective): 
 9class McAllesterObjective(AbstractObjective):
10    """
11    McAllester bound objective (based on McAllester, 1999 and related works),
12    combining empirical loss with a square-root term involving KL and delta,
13    plus additional constants (e.g., 5/2 ln(n)) in the bounding expression.
14    """
15
16    def __init__(self, kl_penalty: float, delta: float):
17        """
18        Args:
19            kl_penalty (float): Multiplier for the KL divergence term.
20            delta (float): Confidence parameter in the PAC-Bayes bound.
21        """
22        self._kl_penalty = kl_penalty
23        self._delta = delta  # confidence value for the training objective
24
25    def calculate(self, loss: Tensor, kl: Tensor, num_samples: float) -> Tensor:
26        """
27        Compute the McAllester objective.
28
29        Derived from a PAC-Bayes bound that includes terms like ln(num_samples)
30        and -ln(delta).
31
32        Args:
33            loss (Tensor): Empirical loss or risk.
34            kl (Tensor): KL divergence.
35            num_samples (float): Number of samples/training size.
36
37        Returns:
38            Tensor: A scalar objective = loss + sqrt( [kl_penalty*KL + 5/2 ln(n) - ln(delta) + 8 ] / [2n - 1] ).
39        """
40        kl = kl * self._kl_penalty
41        kl_ratio = torch.div(
42            kl + 5 / 2 * np.log(num_samples) - np.log(self._delta) + 8,
43            2 * num_samples - 1,
44        )
45        return loss + torch.sqrt(kl_ratio)

McAllester bound objective (based on McAllester, 1999 and related works), combining empirical loss with a square-root term involving KL and delta, plus additional constants (e.g., 5/2 ln(n)) in the bounding expression.

McAllesterObjective(kl_penalty: float, delta: float) 
16    def __init__(self, kl_penalty: float, delta: float):
17        """
18        Args:
19            kl_penalty (float): Multiplier for the KL divergence term.
20            delta (float): Confidence parameter in the PAC-Bayes bound.
21        """
22        self._kl_penalty = kl_penalty
23        self._delta = delta  # confidence value for the training objective
Arguments:
  • kl_penalty (float): Multiplier for the KL divergence term.
  • delta (float): Confidence parameter in the PAC-Bayes bound.
def calculate( self, loss: torch.Tensor, kl: torch.Tensor, num_samples: float) -> torch.Tensor: 
25    def calculate(self, loss: Tensor, kl: Tensor, num_samples: float) -> Tensor:
26        """
27        Compute the McAllester objective.
28
29        Derived from a PAC-Bayes bound that includes terms like ln(num_samples)
30        and -ln(delta).
31
32        Args:
33            loss (Tensor): Empirical loss or risk.
34            kl (Tensor): KL divergence.
35            num_samples (float): Number of samples/training size.
36
37        Returns:
38            Tensor: A scalar objective = loss + sqrt( [kl_penalty*KL + 5/2 ln(n) - ln(delta) + 8 ] / [2n - 1] ).
39        """
40        kl = kl * self._kl_penalty
41        kl_ratio = torch.div(
42            kl + 5 / 2 * np.log(num_samples) - np.log(self._delta) + 8,
43            2 * num_samples - 1,
44        )
45        return loss + torch.sqrt(kl_ratio)

Compute the McAllester objective.

Derived from a PAC-Bayes bound that includes terms like ln(num_samples) and -ln(delta).

Arguments:
  • loss (Tensor): Empirical loss or risk.
  • kl (Tensor): KL divergence.
  • num_samples (float): Number of samples/training size.
Returns:

Tensor: A scalar objective = loss + sqrt( [kl_penalty*KL + 5/2 ln(n) - ln(delta) + 8 ] / [2n - 1] ).