Skip to content

FIA

FIA

Bases: Attack

The FIA (Feature Importance-aware) Attack.

From the paper: Feature Importance-aware Transferable Adversarial Attacks.

Parameters:

Name Type Description Default
model Module | AttackModel

The model to attack.

 required
normalize Callable[[Tensor], Tensor] | None

A transform to normalize images.

 None
device device | None

Device to use for tensors. Defaults to cuda if available.

 None
eps float

The maximum perturbation. Defaults to 8/255.

 8 / 255
steps int

Number of steps. Defaults to 10.

 10
alpha float | None

Step size, eps / steps if None. Defaults to None.

 None
decay float

Decay factor for the momentum term. Defaults to 1.0.

 1.0
num_ens int

Number of aggregate gradients. Defaults to 30.

 30
feature_layer_name str

Layer to compute feature importance. Defaults to "layer4".

 'layer4'
drop_rate float

Dropout rate for random pixels. Defaults to 0.3.

 0.3
clip_min float

Minimum value for clipping. Defaults to 0.0.

 0.0
clip_max float

Maximum value for clipping. Defaults to 1.0.

 1.0
Source code in torchattack/fia.py 
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
@register_attack()
class FIA(Attack):
    """The FIA (Feature Importance-aware) Attack.

    > From the paper: [Feature Importance-aware Transferable Adversarial
    Attacks](https://arxiv.org/abs/2107.14185).

    Args:
        model: The model to attack.
        normalize: A transform to normalize images.
        device: Device to use for tensors. Defaults to cuda if available.
        eps: The maximum perturbation. Defaults to 8/255.
        steps: Number of steps. Defaults to 10.
        alpha: Step size, `eps / steps` if None. Defaults to None.
        decay: Decay factor for the momentum term. Defaults to 1.0.
        num_ens: Number of aggregate gradients. Defaults to 30.
        feature_layer_name: Layer to compute feature importance. Defaults to "layer4".
        drop_rate: Dropout rate for random pixels. Defaults to 0.3.
        clip_min: Minimum value for clipping. Defaults to 0.0.
        clip_max: Maximum value for clipping. Defaults to 1.0.
    """

    def __init__(
        self,
        model: nn.Module | AttackModel,
        normalize: Callable[[torch.Tensor], torch.Tensor] | None = None,
        device: torch.device | None = None,
        eps: float = 8 / 255,
        steps: int = 10,
        alpha: float | None = None,
        decay: float = 1.0,
        num_ens: int = 30,
        feature_layer_name: str = 'layer4',
        drop_rate: float = 0.3,
        clip_min: float = 0.0,
        clip_max: float = 1.0,
    ) -> None:
        super().__init__(model, normalize, device)

        self.eps = eps
        self.steps = steps
        self.alpha = alpha
        self.decay = decay
        self.num_ens = num_ens
        self.drop_rate = drop_rate
        self.clip_min = clip_min
        self.clip_max = clip_max

        self.feature_layer_name = feature_layer_name
        self.feature_module = rgetattr(self.model, feature_layer_name)

    def forward(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
        """Perform FIA on a batch of images.

        Args:
            x: A batch of images. Shape: (N, C, H, W).
            y: A batch of labels. Shape: (N).

        Returns:
            The perturbed images if successful. Shape: (N, C, H, W).
        """

        g = torch.zeros_like(x)
        delta = torch.zeros_like(x, requires_grad=True)

        # If alpha is not given, set to eps / steps
        if self.alpha is None:
            self.alpha = self.eps / self.steps

        hf = self.feature_module.register_forward_hook(self._forward_hook)
        hb = self.feature_module.register_full_backward_hook(self._backward_hook)

        # Gradient aggregation on ensembles
        agg_grad: torch.Tensor | float = 0.0
        for _ in range(self.num_ens):
            # Create variants of input with randomly dropped pixels
            x_dropped = self.drop_pixels(x)

            # Get model outputs and compute gradients
            outs_dropped = self.model(self.normalize(x_dropped))
            outs_dropped = torch.softmax(outs_dropped, dim=1)

            loss = sum(outs_dropped[bi][y[bi]] for bi in range(x.shape[0]))
            loss.backward()

            # Accumulate gradients
            agg_grad += self.mid_grad[0].detach()

        # for batch_i in range(x.shape[0]):
        #     agg_grad[batch_i] /= agg_grad[batch_i].norm(p=2)
        agg_grad /= torch.norm(agg_grad, p=2, dim=(1, 2, 3), keepdim=True)
        hb.remove()

        # Perform FIA
        for _ in range(self.steps):
            # Pass through the model
            _ = self.model(self.normalize(x + delta))

            # Hooks are updated during forward pass
            loss = (self.mid_output * agg_grad).sum()
            loss.backward()

            if delta.grad is None:
                continue

            # Apply momentum term
            g = self.decay * g + delta.grad / torch.mean(
                torch.abs(delta.grad), dim=(1, 2, 3), keepdim=True
            )

            # Update delta
            delta.data = delta.data - self.alpha * g.sign()
            delta.data = torch.clamp(delta.data, -self.eps, self.eps)
            delta.data = torch.clamp(x + delta.data, self.clip_min, self.clip_max) - x

            # Zero out gradient
            delta.grad.detach_()
            delta.grad.zero_()

        hf.remove()
        return x + delta

    def drop_pixels(self, x: torch.Tensor) -> torch.Tensor:
        """Randomly drop pixels from the input image.

        Args:
            x: A batch of images. Shape: (N, C, H, W).

        Returns:
            A batch of images with randomly dropped pixels.
        """

        x_dropped = torch.zeros_like(x)
        x_dropped.copy_(x).detach()
        x_dropped.requires_grad = True

        mask = torch.bernoulli(torch.ones_like(x) * (1 - self.drop_rate))
        x_dropped = x_dropped * mask

        return x_dropped

    def _forward_hook(self, m: nn.Module, i: torch.Tensor, o: torch.Tensor) -> None:
        self.mid_output = o

    def _backward_hook(self, m: nn.Module, i: torch.Tensor, o: torch.Tensor) -> None:
        self.mid_grad = o

drop_pixels(x)

Randomly drop pixels from the input image.

Parameters:

Name Type Description Default
x Tensor

A batch of images. Shape: (N, C, H, W).

 required

Returns:

Type Description
Tensor

A batch of images with randomly dropped pixels.
Source code in torchattack/fia.py 
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
def drop_pixels(self, x: torch.Tensor) -> torch.Tensor:
    """Randomly drop pixels from the input image.

    Args:
        x: A batch of images. Shape: (N, C, H, W).

    Returns:
        A batch of images with randomly dropped pixels.
    """

    x_dropped = torch.zeros_like(x)
    x_dropped.copy_(x).detach()
    x_dropped.requires_grad = True

    mask = torch.bernoulli(torch.ones_like(x) * (1 - self.drop_rate))
    x_dropped = x_dropped * mask

    return x_dropped

forward(x, y)

Perform FIA on a batch of images.

Parameters:

Name Type Description Default
x Tensor

A batch of images. Shape: (N, C, H, W).

 required
y Tensor

A batch of labels. Shape: (N).

 required

Returns:

Type Description
Tensor

The perturbed images if successful. Shape: (N, C, H, W).
Source code in torchattack/fia.py 
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
def forward(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
    """Perform FIA on a batch of images.

    Args:
        x: A batch of images. Shape: (N, C, H, W).
        y: A batch of labels. Shape: (N).

    Returns:
        The perturbed images if successful. Shape: (N, C, H, W).
    """

    g = torch.zeros_like(x)
    delta = torch.zeros_like(x, requires_grad=True)

    # If alpha is not given, set to eps / steps
    if self.alpha is None:
        self.alpha = self.eps / self.steps

    hf = self.feature_module.register_forward_hook(self._forward_hook)
    hb = self.feature_module.register_full_backward_hook(self._backward_hook)

    # Gradient aggregation on ensembles
    agg_grad: torch.Tensor | float = 0.0
    for _ in range(self.num_ens):
        # Create variants of input with randomly dropped pixels
        x_dropped = self.drop_pixels(x)

        # Get model outputs and compute gradients
        outs_dropped = self.model(self.normalize(x_dropped))
        outs_dropped = torch.softmax(outs_dropped, dim=1)

        loss = sum(outs_dropped[bi][y[bi]] for bi in range(x.shape[0]))
        loss.backward()

        # Accumulate gradients
        agg_grad += self.mid_grad[0].detach()

    # for batch_i in range(x.shape[0]):
    #     agg_grad[batch_i] /= agg_grad[batch_i].norm(p=2)
    agg_grad /= torch.norm(agg_grad, p=2, dim=(1, 2, 3), keepdim=True)
    hb.remove()

    # Perform FIA
    for _ in range(self.steps):
        # Pass through the model
        _ = self.model(self.normalize(x + delta))

        # Hooks are updated during forward pass
        loss = (self.mid_output * agg_grad).sum()
        loss.backward()

        if delta.grad is None:
            continue

        # Apply momentum term
        g = self.decay * g + delta.grad / torch.mean(
            torch.abs(delta.grad), dim=(1, 2, 3), keepdim=True
        )

        # Update delta
        delta.data = delta.data - self.alpha * g.sign()
        delta.data = torch.clamp(delta.data, -self.eps, self.eps)
        delta.data = torch.clamp(x + delta.data, self.clip_min, self.clip_max) - x

        # Zero out gradient
        delta.grad.detach_()
        delta.grad.zero_()

    hf.remove()
    return x + delta