forked from yjhuangcd/CNNF
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy patheval.py
156 lines (128 loc) · 5.98 KB
/
eval.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
import torch
import torch.nn as nn
import torch.nn.functional as F
import logging
import os
import torch.optim as optim
import numpy as np
import math
import torchvision
from torchvision import datasets, transforms
import matplotlib.pyplot as plt
import shutil
from tensorboardX import SummaryWriter
from advertorch.attacks import GradientSignAttack, LinfPGDAttack, LinfSPSAAttack
from advertorch.context import ctx_noparamgrad_and_eval
class Evaluator:
def __init__(self, device, model):
self.device = device
self.model = model
def clean_accuracy(self, clean_loader, test='last'):
""" Evaluate the model on clean dataset. """
self.model.eval()
correct = 0
with torch.no_grad():
for batch_idx, (data, target) in enumerate(clean_loader):
data, target = data.to(self.device), target.to(self.device)
if (test=='last'):
output = self.model.run_cycles(data)
elif(test=='average'):
output = self.model.run_average(data)
else:
self.model.reset()
output = self.model(data)
pred = output.argmax(dim=1, keepdim=True)
correct += pred.eq(target.view_as(pred)).sum().item()
acc = correct / len(clean_loader.dataset)
print('Clean Test Acc {:.3f}'.format(100. * acc))
return acc
def attack_pgd(self, clean_loader, epsilon=0.1, eps_iter=0.02, test='average', ete=False, nb_iter=7):
""" Use PGD to attack the model. """
self.model.eval()
self.model.reset()
if (ete==False):
adv_func = self.model.forward_adv
else:
adv_func = self.model.run_cycles_adv
adversary = LinfPGDAttack(
adv_func, loss_fn=nn.CrossEntropyLoss(reduction="sum"), eps=epsilon,
nb_iter=nb_iter, eps_iter=eps_iter, rand_init=True, clip_min=-1.0, clip_max=1.0, targeted=False)
correct = 0
for batch_idx, (data, target) in enumerate(clean_loader):
data, target = data.to(self.device), target.to(self.device)
self.model.reset()
with ctx_noparamgrad_and_eval(self.model):
adv_images = adversary.perturb(data, target)
if(test=='last'):
output = self.model.run_cycles(adv_images)
elif(test=='average'):
output = self.model.run_average(adv_images)
else:
self.model.reset()
output = self.model(adv_images)
pred = output.argmax(dim=1, keepdim=True)
correct += pred.eq(target.view_as(pred)).sum().item()
acc = correct / len(clean_loader.dataset)
print('PGD attack Acc {:.3f}'.format(100. * acc))
return acc
def attack_spsa(self, clean_loader, epsilon=0.1, test='average', ete=False, nb_iter=7):
""" Use SPSA to attack the model. """
self.model.eval()
self.model.reset()
if (ete==False):
adv_func = self.model.forward_adv
else:
adv_func = self.model.run_cycles_adv
adversary = LinfSPSAAttack(
adv_func, loss_fn=nn.CrossEntropyLoss(reduction="none"), eps=epsilon,
nb_iter=nb_iter, delta=0.01, nb_sample=128, max_batch_size=64, clip_min=-1.0, clip_max=1.0, targeted=False)
correct = 0
numofdata = 0
for batch_idx, (data, target) in enumerate(clean_loader):
# To speed up the evaluation of attack, evaluate the first 10 batches
if(batch_idx < 10):
data, target = data.to(self.device), target.to(self.device)
self.model.reset()
with ctx_noparamgrad_and_eval(self.model):
adv_images = adversary.perturb(data, target)
if(test=='last'):
output = self.model.run_cycles(adv_images)
elif(test=='average'):
output = self.model.run_average(adv_images)
else:
self.model.reset()
output = self.model(adv_images)
pred = output.argmax(dim=1, keepdim=True)
correct += pred.eq(target.view_as(pred)).sum().item()
numofdata += data.shape[0]
acc = correct / numofdata
print('SPSA attack Acc {:.3f}'.format(100. * acc))
return acc
def attack_pgd_transfer(self, model_attacker, clean_loader, epsilon=0.1, eps_iter=0.02, test='average', nb_iter=7):
""" Use adversarial samples generated against model_attacker to attack the current model. """
self.model.eval()
self.model.reset()
model_attacker.eval()
model_attacker.reset()
adversary = LinfPGDAttack(
model_attacker.forward_adv, loss_fn=nn.CrossEntropyLoss(reduction="sum"), eps=epsilon,
nb_iter=nb_iter, eps_iter=eps_iter, rand_init=True, clip_min=-1.0, clip_max=1.0, targeted=False)
correct = 0
for batch_idx, (data, target) in enumerate(clean_loader):
data, target = data.to(self.device), target.to(self.device)
self.model.reset()
model_attacker.reset()
with ctx_noparamgrad_and_eval(model_attacker):
adv_images = adversary.perturb(data, target)
if(test=='last'):
output = self.model.run_cycles(adv_images)
elif(test=='average'):
output = self.model.run_average(adv_images)
else:
self.model.reset()
output = self.model(adv_images)
pred = output.argmax(dim=1, keepdim=True)
correct += pred.eq(target.view_as(pred)).sum().item()
acc = correct / len(clean_loader.dataset)
print('PGD attack Acc {:.3f}'.format(100. * acc))
return acc