In [1]:
# !pip install aeon
In [2]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
In [ ]:
Recurrent Neural Networks¶
In [3]:
from aeon.datasets import load_classification
Dataset¶
ECG200
This dataset was formatted by R. Olszewski as part of his thesis "Generalized feature extraction for structural pattern recognition in time-series data" at Carnegie Mellon University, 2001. Each series traces the electrical activity recorded during one heartbeat. The two classes are a normal heartbeat and a Myocardial Infarction.
In [4]:
# Import the dataset
X_train, y_train = load_classification("ECG200", split="train")
X_test, y_test = load_classification("ECG200", split="test")
X_train = X_train.astype(np.float32)
X_test = X_test.astype(np.float32)
X_train.shape, y_train.shape, X_test.shape, y_test.shape
Out[4]:
((100, 1, 96), (100,), (100, 1, 96), (100,))
In [5]:
# Encode the labels
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
y_train = le.fit_transform(y_train)
y_test = le.transform(y_test)
le.classes_
Out[5]:
array(['-1', '1'], dtype='<U2')
In [6]:
LABELS = ["infarction", "normal"]
In [7]:
y_train
Out[7]:
array([0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1,
1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0,
0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0,
1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1,
0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1]) In [8]:
i = 0
plt.figure(figsize=(5, 5))
plt.plot(X_train[i].ravel())
plt.title('Class: {}'.format(LABELS[y_train[i]]))
plt.show()
In [9]:
# Dataset visualization
fig, axs = plt.subplots(1, len(le.classes_), figsize=(10, 5), sharey=True)
for i in range(len(le.classes_)):
for x in X_train[y_train == i]:
axs[i].plot(x.ravel(), color='C0', linewidth=0.9)
axs[i].set_title('Class: {}'.format(LABELS[i]), fontsize=16)
plt.tight_layout()
plt.show()
Vanilla RNN¶
In [10]:
import torch
from torch import nn
import torch.nn.functional as F
from skorch import NeuralNetClassifier
from skorch.callbacks import EpochScoring
from skorch.dataset import ValidSplit
from sklearn.metrics import accuracy_score
In [11]:
DEVICE = "mps" if torch.backends.mps.is_available() else "cuda" if torch.cuda.is_available() else "cpu"
In [12]:
# build a vanilla RNN with a classification head
class VanillaRNN(nn.Module):
def __init__(self, input_size, hidden_size, output_size):
super(VanillaRNN, self).__init__()
self.rnn = nn.RNN(input_size, hidden_size, batch_first=True)
self.fc = nn.Linear(hidden_size, output_size)
def forward(self, x):
x = torch.swapaxes(x, 1, 2) # swap the last two dimensions to match RNN input shape
out, _ = self.rnn(x)
out = out[:, -1, :] # take the last output of the sequence
out = self.fc(out)
return out
In [13]:
net = NeuralNetClassifier(
VanillaRNN,
module__input_size=X_train.shape[1],
module__hidden_size=32,
module__output_size=len(le.classes_),
criterion=nn.CrossEntropyLoss,
max_epochs=200,
lr=0.01,
batch_size=32,
callbacks=[
EpochScoring(
scoring='accuracy',
name='train_acc',
on_train=True,
),
],
device=DEVICE,
train_split=ValidSplit(cv=5, stratified=True)
)
net
Out[13]:
<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized]( module=<class '__main__.VanillaRNN'>, module__hidden_size=32, module__input_size=1, module__output_size=2, )In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized]( module=<class '__main__.VanillaRNN'>, module__hidden_size=32, module__input_size=1, module__output_size=2, )
In [14]:
net.fit(X_train, y_train)
epoch train_acc train_loss valid_acc valid_loss dur
------- ----------- ------------ ----------- ------------ ------
1 0.2625 0.7152 0.3500 0.7090 0.4373
2 0.2250 0.7076 0.2500 0.7031 0.1464
3 0.2375 0.7006 0.3000 0.6977 0.1436
4 0.5250 0.6939 0.6000 0.6925 0.1417
5 0.7000 0.6877 0.6500 0.6877 0.1530
6 0.7000 0.6818 0.6500 0.6832 0.1623
7 0.7000 0.6762 0.6500 0.6789 0.1483
8 0.7000 0.6709 0.6500 0.6748 0.1419
9 0.7000 0.6659 0.6500 0.6710 0.1560
10 0.7000 0.6611 0.6500 0.6673 0.1660
11 0.7000 0.6565 0.6500 0.6638 0.1452
12 0.7000 0.6522 0.6500 0.6605 0.1434
13 0.7000 0.6480 0.6500 0.6573 0.1520
14 0.7000 0.6440 0.6500 0.6542 0.1611
15 0.7000 0.6402 0.6500 0.6513 0.1570
16 0.7000 0.6365 0.6500 0.6484 0.1605
17 0.7000 0.6329 0.6500 0.6457 0.1454
18 0.7000 0.6295 0.6500 0.6430 0.1440
19 0.7000 0.6261 0.6500 0.6404 0.1525
20 0.7000 0.6229 0.6500 0.6379 0.1452
21 0.7000 0.6198 0.6500 0.6354 0.1434
22 0.7000 0.6168 0.6500 0.6330 0.1439
23 0.7000 0.6138 0.6500 0.6306 0.1451
24 0.7000 0.6110 0.6500 0.6283 0.1439
25 0.7000 0.6082 0.6500 0.6260 0.1427
26 0.7000 0.6054 0.6500 0.6236 0.1423
27 0.7000 0.6027 0.6500 0.6213 0.1454
28 0.7000 0.6000 0.6500 0.6190 0.1543
29 0.7000 0.5974 0.6500 0.6167 0.1471
30 0.7000 0.5948 0.6500 0.6144 0.1460
31 0.7000 0.5923 0.6500 0.6120 0.1477
32 0.7000 0.5897 0.6500 0.6097 0.1454
33 0.7000 0.5872 0.6500 0.6073 0.1443
34 0.7125 0.5847 0.6500 0.6048 0.1432
35 0.7125 0.5821 0.6500 0.6023 0.1533
36 0.7125 0.5796 0.6500 0.5997 0.1421
37 0.7125 0.5771 0.6500 0.5971 0.1427
38 0.7250 0.5745 0.6500 0.5944 0.1423
39 0.7250 0.5719 0.6500 0.5916 0.1429
40 0.7250 0.5694 0.6500 0.5887 0.1432
41 0.7250 0.5667 0.6500 0.5858 0.1438
42 0.7250 0.5641 0.6500 0.5827 0.1430
43 0.7250 0.5614 0.6500 0.5796 0.1411
44 0.7250 0.5587 0.6500 0.5763 0.1419
45 0.7250 0.5560 0.6500 0.5729 0.1433
46 0.7375 0.5532 0.6500 0.5695 0.1434
47 0.7375 0.5504 0.7000 0.5658 0.1426
48 0.7375 0.5475 0.7000 0.5621 0.1425
49 0.7375 0.5446 0.7000 0.5582 0.1449
50 0.7500 0.5417 0.7000 0.5542 0.1428
51 0.7750 0.5387 0.7000 0.5501 0.1546
52 0.7750 0.5358 0.7000 0.5458 0.1432
53 0.7625 0.5328 0.7000 0.5414 0.1431
54 0.7750 0.5298 0.7000 0.5369 0.1428
55 0.7750 0.5268 0.7000 0.5323 0.1432
56 0.7750 0.5239 0.7000 0.5276 0.1428
57 0.7625 0.5210 0.7000 0.5229 0.1410
58 0.7625 0.5181 0.7000 0.5181 0.1410
59 0.7750 0.5154 0.8000 0.5134 0.1429
60 0.7750 0.5128 0.8000 0.5088 0.1413
61 0.7625 0.5104 0.8000 0.5044 0.1439
62 0.7875 0.5081 0.8000 0.5001 0.1443
63 0.7875 0.5060 0.8000 0.4961 0.1432
64 0.7875 0.5041 0.8000 0.4923 0.1426
65 0.7750 0.5023 0.8000 0.4888 0.2005
66 0.7750 0.5007 0.8000 0.4856 0.1403
67 0.7750 0.4993 0.8000 0.4827 0.1427
68 0.7750 0.4980 0.8000 0.4801 0.1431
69 0.7625 0.4969 0.8000 0.4777 0.1415
70 0.7750 0.4958 0.8000 0.4756 0.1443
71 0.7750 0.4949 0.8000 0.4736 0.1414
72 0.7750 0.4941 0.8000 0.4719 0.1433
73 0.7750 0.4933 0.8000 0.4703 0.1441
74 0.7750 0.4926 0.8000 0.4689 0.1425
75 0.7750 0.4919 0.8500 0.4676 0.1431
76 0.7750 0.4913 0.8000 0.4663 0.1437
77 0.7750 0.4908 0.8000 0.4652 0.1436
78 0.7750 0.4903 0.8000 0.4642 0.1440
79 0.7750 0.4898 0.8000 0.4632 0.1444
80 0.7750 0.4893 0.8000 0.4623 0.1426
81 0.7750 0.4889 0.8000 0.4615 0.1515
82 0.7750 0.4884 0.8000 0.4607 0.1431
83 0.7750 0.4880 0.8000 0.4599 0.1418
84 0.7750 0.4877 0.8000 0.4592 0.1429
85 0.7750 0.4873 0.8000 0.4585 0.1441
86 0.7750 0.4869 0.8000 0.4578 0.1424
87 0.7750 0.4866 0.8000 0.4572 0.1433
88 0.7750 0.4862 0.8000 0.4566 0.1405
89 0.7750 0.4859 0.8000 0.4560 0.1437
90 0.7750 0.4856 0.8500 0.4554 0.1418
91 0.7750 0.4853 0.8500 0.4549 0.1441
92 0.7750 0.4850 0.8500 0.4544 0.1412
93 0.7750 0.4847 0.8500 0.4538 0.1412
94 0.7750 0.4844 0.8500 0.4533 0.1426
95 0.7750 0.4841 0.8500 0.4529 0.1414
96 0.7750 0.4838 0.8500 0.4524 0.1423
97 0.7875 0.4836 0.8500 0.4519 0.1526
98 0.7875 0.4833 0.8500 0.4515 0.1409
99 0.7875 0.4830 0.8500 0.4510 0.1436
100 0.7875 0.4828 0.8500 0.4506 0.1428
101 0.7875 0.4825 0.8500 0.4502 0.1415
102 0.7875 0.4823 0.8500 0.4498 0.1405
103 0.7875 0.4820 0.8500 0.4494 0.1417
104 0.7875 0.4818 0.8500 0.4489 0.1412
105 0.7875 0.4815 0.8500 0.4486 0.1415
106 0.7875 0.4813 0.8500 0.4482 0.1395
107 0.7875 0.4811 0.8500 0.4478 0.1435
108 0.7875 0.4808 0.8500 0.4474 0.1415
109 0.7875 0.4806 0.8500 0.4470 0.1401
110 0.8000 0.4804 0.8500 0.4467 0.1431
111 0.8000 0.4802 0.8500 0.4463 0.1425
112 0.8000 0.4800 0.8500 0.4459 0.1426
113 0.8000 0.4797 0.8500 0.4456 0.1511
114 0.8000 0.4795 0.8500 0.4452 0.1418
115 0.8000 0.4793 0.8500 0.4449 0.1425
116 0.8000 0.4791 0.8500 0.4445 0.1414
117 0.8000 0.4789 0.8500 0.4442 0.1439
118 0.8000 0.4787 0.8500 0.4438 0.1414
119 0.8000 0.4785 0.8500 0.4435 0.1438
120 0.8000 0.4783 0.8500 0.4432 0.1420
121 0.8000 0.4781 0.8500 0.4428 0.1435
122 0.8000 0.4779 0.8500 0.4425 0.1415
123 0.8000 0.4777 0.8500 0.4422 0.1422
124 0.8000 0.4775 0.8500 0.4418 0.1420
125 0.8000 0.4774 0.8500 0.4415 0.1415
126 0.8000 0.4772 0.8500 0.4412 0.1418
127 0.8000 0.4770 0.8500 0.4409 0.1429
128 0.8000 0.4768 0.8500 0.4405 0.1422
129 0.8000 0.4766 0.8500 0.4402 0.1520
130 0.8000 0.4764 0.8500 0.4399 0.1439
131 0.8000 0.4763 0.8500 0.4396 0.1407
132 0.8000 0.4761 0.8500 0.4393 0.1423
133 0.8000 0.4759 0.8500 0.4390 0.1422
134 0.8000 0.4757 0.8500 0.4387 0.8520
135 0.8000 0.4756 0.8500 0.4384 0.3665
136 0.8000 0.4754 0.8500 0.4381 0.4193
137 0.8000 0.4752 0.8500 0.4378 0.1436
138 0.8000 0.4750 0.8500 0.4375 0.1407
139 0.8000 0.4749 0.8500 0.4372 0.1411
140 0.8000 0.4747 0.8500 0.4369 0.1418
141 0.8000 0.4745 0.8500 0.4366 0.1424
142 0.8000 0.4744 0.8500 0.4363 0.1419
143 0.8000 0.4742 0.8500 0.4360 0.1420
144 0.8000 0.4741 0.8500 0.4357 0.1434
145 0.8000 0.4739 0.8500 0.4354 0.1522
146 0.8000 0.4737 0.8500 0.4351 0.1409
147 0.8000 0.4736 0.8500 0.4348 0.1423
148 0.8000 0.4734 0.8500 0.4345 0.1417
149 0.8000 0.4733 0.8500 0.4342 0.1397
150 0.8000 0.4731 0.8500 0.4339 0.1413
151 0.8000 0.4729 0.8500 0.4336 0.1438
152 0.8000 0.4728 0.8500 0.4333 0.1428
153 0.8000 0.4726 0.8500 0.4331 0.1437
154 0.8000 0.4725 0.8500 0.4328 0.1409
155 0.8000 0.4723 0.8500 0.4325 0.1418
156 0.8000 0.4722 0.8500 0.4322 0.1418
157 0.8000 0.4720 0.8500 0.4319 0.1792
158 0.8000 0.4719 0.8500 0.4317 0.1416
159 0.8000 0.4717 0.8500 0.4314 0.1453
160 0.8000 0.4716 0.8500 0.4311 0.1537
161 0.8000 0.4714 0.8500 0.4308 0.1423
162 0.8000 0.4713 0.8500 0.4305 0.1434
163 0.8000 0.4711 0.8500 0.4303 0.1430
164 0.8000 0.4710 0.8500 0.4300 0.1430
165 0.8000 0.4708 0.8500 0.4297 0.1425
166 0.8000 0.4707 0.8500 0.4294 0.1447
167 0.8000 0.4705 0.8500 0.4292 0.1434
168 0.8000 0.4704 0.8500 0.4289 0.1424
169 0.8000 0.4702 0.8500 0.4286 0.1451
170 0.8000 0.4701 0.8500 0.4283 0.1444
171 0.8000 0.4700 0.8500 0.4281 0.1465
172 0.8000 0.4698 0.8500 0.4278 0.1429
173 0.8000 0.4697 0.8500 0.4275 0.1429
174 0.8000 0.4695 0.8500 0.4273 0.1459
175 0.8000 0.4694 0.8500 0.4270 0.1540
176 0.8000 0.4693 0.8500 0.4267 0.1434
177 0.8000 0.4691 0.8500 0.4265 0.1432
178 0.8000 0.4690 0.8500 0.4262 0.1464
179 0.8000 0.4688 0.8500 0.4259 0.1481
180 0.8000 0.4687 0.8500 0.4257 0.1494
181 0.8000 0.4686 0.8500 0.4254 0.1440
182 0.8000 0.4684 0.8500 0.4251 0.1442
183 0.8000 0.4683 0.8500 0.4249 0.1450
184 0.8000 0.4682 0.8500 0.4246 0.1438
185 0.8000 0.4680 0.8500 0.4244 0.1461
186 0.8000 0.4679 0.8500 0.4241 0.1448
187 0.8000 0.4677 0.8500 0.4238 0.1466
188 0.8000 0.4676 0.8500 0.4236 0.1518
189 0.8000 0.4675 0.8500 0.4233 0.1432
190 0.8000 0.4673 0.8500 0.4231 0.1430
191 0.8000 0.4672 0.8500 0.4228 0.1454
192 0.8000 0.4671 0.8500 0.4226 0.1467
193 0.8000 0.4670 0.8500 0.4223 0.1722
194 0.8000 0.4668 0.8500 0.4221 0.1519
195 0.8000 0.4667 0.8500 0.4218 0.1505
196 0.8000 0.4666 0.8500 0.4215 0.1438
197 0.8000 0.4664 0.8500 0.4213 0.1424
198 0.8000 0.4663 0.8500 0.4210 0.1454
199 0.8000 0.4662 0.8500 0.4208 0.2106
200 0.8000 0.4660 0.8500 0.4205 0.1688
Out[14]:
<class 'skorch.classifier.NeuralNetClassifier'>[initialized](
module_=VanillaRNN(
(rnn): RNN(1, 32, batch_first=True)
(fc): Linear(in_features=32, out_features=2, bias=True)
),
)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
<class 'skorch.classifier.NeuralNetClassifier'>[initialized](
module_=VanillaRNN(
(rnn): RNN(1, 32, batch_first=True)
(fc): Linear(in_features=32, out_features=2, bias=True)
),
)In [15]:
y_pred = net.predict(X_test)
accuracy_score(y_test, y_pred)
Out[15]:
0.69
In [16]:
net.criterion, net.optimizer
Out[16]:
(torch.nn.modules.loss.CrossEntropyLoss, torch.optim.sgd.SGD)
In [17]:
history = net.history
plt.plot(history[:, 'train_loss'], label='train_loss')
plt.plot(history[:, 'valid_loss'], label='valid_loss')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.show()
plt.plot(history[:, 'train_acc'], label='train_acc')
plt.plot(history[:, 'valid_acc'], label='valid_acc')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.show()
LSTM¶
In [18]:
from skorch.callbacks import LRScheduler, EarlyStopping, Checkpoint
from torch.optim.lr_scheduler import ReduceLROnPlateau
In [19]:
# build a vanilla RNN with a classification head
class LSTM(nn.Module):
def __init__(self, input_size, hidden_size, output_size):
super(LSTM, self).__init__()
self.rnn = nn.LSTM(input_size, hidden_size, batch_first=True)
self.fc = nn.Linear(hidden_size, output_size)
def forward(self, x):
x = torch.swapaxes(x, 1, 2) # swap the last two dimensions to match RNN input shape
out, _ = self.rnn(x)
out = out[:, -1, :] # take the last output of the sequence
out = self.fc(out)
return out
In [20]:
net = NeuralNetClassifier(
LSTM,
module__input_size=X_train.shape[1],
module__hidden_size=32,
module__output_size=len(le.classes_),
criterion=nn.CrossEntropyLoss,
optimizer=torch.optim.Adam,
max_epochs=500,
batch_size=32,
callbacks=[
EpochScoring(
scoring='accuracy',
name='train_acc',
on_train=True,
),
Checkpoint(
monitor="valid_acc_best",
load_best=True,
f_history=None,
f_optimizer=None,
)
],
device=DEVICE,
train_split=ValidSplit(cv=5, stratified=True)
)
net
Out[20]:
<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized]( module=<class '__main__.LSTM'>, module__hidden_size=32, module__input_size=1, module__output_size=2, )In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized]( module=<class '__main__.LSTM'>, module__hidden_size=32, module__input_size=1, module__output_size=2, )
In [21]:
net.fit(X_train, y_train)
epoch train_acc train_loss valid_acc valid_loss cp dur
------- ----------- ------------ ----------- ------------ ---- ------
1 0.7000 0.6550 0.6500 0.6367 + 0.2448
2 0.7000 0.6079 0.6500 0.5962 0.0470
3 0.7000 0.5641 0.6500 0.5359 0.0470
4 0.7750 0.5256 0.8000 0.5036 + 0.0470
5 0.7750 0.5142 0.8500 0.4533 + 0.0483
6 0.7750 0.5023 0.8000 0.4596 0.0472
7 0.8000 0.4865 0.7500 0.4728 0.0518
8 0.8000 0.4813 0.7500 0.4742 0.0492
9 0.8125 0.4780 0.8000 0.4685 0.0483
10 0.8000 0.4777 0.8000 0.4599 0.0537
11 0.8000 0.4767 0.8000 0.4496 0.0553
12 0.8000 0.4718 0.8500 0.4399 0.0547
13 0.8000 0.4661 0.8500 0.4317 0.0533
14 0.8000 0.4611 0.8500 0.4244 0.0472
15 0.8000 0.4558 0.8500 0.4174 0.0474
16 0.8125 0.4456 0.8500 0.4098 0.0475
17 0.8000 0.4360 0.8500 0.4200 0.0465
18 0.8000 0.4060 0.9000 0.4120 + 0.0488
19 0.8250 0.3714 0.9000 0.3614 0.0487
20 0.7875 0.4907 0.8000 0.4599 0.0479
21 0.8125 0.4050 0.9000 0.3459 0.0509
22 0.8375 0.4111 0.8500 0.3918 0.0490
23 0.8500 0.3757 0.8500 0.4017 0.0528
24 0.8500 0.3682 0.9000 0.3511 0.0481
25 0.8125 0.4113 0.8500 0.4144 0.0470
26 0.8625 0.3544 0.9000 0.3879 0.0469
27 0.8625 0.3430 0.9000 0.3771 0.0481
28 0.9000 0.3002 0.7500 0.4771 0.0467
29 0.9000 0.2888 0.8500 0.3788 0.0480
30 0.8375 0.3745 0.8000 0.4516 0.0477
31 0.8375 0.3755 0.8000 0.5073 0.0488
32 0.8375 0.3776 0.8500 0.3754 0.0484
33 0.8875 0.2903 0.8000 0.4028 0.0485
34 0.8875 0.3054 0.8000 0.3955 0.0475
35 0.8875 0.2910 0.8500 0.3909 0.0490
36 0.8500 0.3429 0.7000 0.5693 0.0482
37 0.6625 0.5911 0.7000 0.5685 0.0481
38 0.7875 0.5016 0.7000 0.4688 0.0478
39 0.8250 0.4276 0.8000 0.4213 0.0485
40 0.8000 0.5141 0.8500 0.4116 0.0473
41 0.8000 0.4554 0.7500 0.4589 0.0483
42 0.8125 0.4360 0.7500 0.4732 0.0499
43 0.8250 0.4197 0.8000 0.4520 0.0548
44 0.8250 0.4248 0.8500 0.4516 0.0533
45 0.8125 0.4415 0.8500 0.4496 0.0510
46 0.8250 0.4268 0.8000 0.4545 0.0461
47 0.8000 0.4131 0.8000 0.4594 0.0554
48 0.8125 0.4064 0.8000 0.4519 0.0474
49 0.8250 0.4059 0.8500 0.4462 0.0473
50 0.8250 0.4127 0.8500 0.4470 0.0473
51 0.8250 0.4114 0.8000 0.4529 0.0483
52 0.8375 0.4037 0.8000 0.4623 0.0484
53 0.8375 0.3981 0.8000 0.4694 0.0519
54 0.8375 0.3979 0.8000 0.4754 0.0481
55 0.8250 0.4005 0.8000 0.4817 0.0474
56 0.8250 0.3980 0.8000 0.4890 0.0486
57 0.8375 0.3936 0.8000 0.4978 0.0486
58 0.8375 0.3916 0.8000 0.5076 0.0486
59 0.8375 0.3922 0.8000 0.5175 0.0474
60 0.8375 0.3908 0.8000 0.5273 0.0485
61 0.8375 0.3874 0.8000 0.5364 0.0525
62 0.8375 0.3855 0.8000 0.5448 0.0484
63 0.8375 0.3852 0.8000 0.5535 0.0487
64 0.8375 0.3838 0.8000 0.5632 0.0480
65 0.8375 0.3812 0.8000 0.5729 0.0923
66 0.8375 0.3795 0.8000 0.5824 0.0496
67 0.8375 0.3787 0.8000 0.5933 0.0483
68 0.8375 0.3769 0.8000 0.6054 0.0472
69 0.8375 0.3746 0.8000 0.6166 0.0461
70 0.8375 0.3730 0.8000 0.6277 0.0476
71 0.8375 0.3715 0.7500 0.6402 0.0488
72 0.8250 0.3693 0.7500 0.6526 0.0486
73 0.8250 0.3670 0.7500 0.6642 0.0481
74 0.8250 0.3647 0.7500 0.6769 0.0489
75 0.8375 0.3621 0.7500 0.6904 0.0480
76 0.8250 0.3591 0.7500 0.7040 0.0508
77 0.8250 0.3563 0.7000 0.7196 0.0485
78 0.8250 0.3531 0.7000 0.7380 0.0477
79 0.8250 0.3495 0.7000 0.7578 0.0486
80 0.8250 0.3454 0.7000 0.7805 0.0477
81 0.8250 0.3398 0.7000 0.8069 0.0480
82 0.8250 0.3313 0.7000 0.8341 0.0485
83 0.8250 0.3182 0.6500 0.8529 0.0506
84 0.8750 0.3078 0.6500 0.8208 0.0484
85 0.8375 0.3042 0.6500 0.8805 0.0479
86 0.8250 0.3752 0.8000 0.6743 0.0487
87 0.8500 0.3491 0.6000 0.8725 0.0481
88 0.8625 0.3190 0.7500 0.7324 0.0478
89 0.8375 0.3458 0.7000 0.8354 0.0484
90 0.8750 0.3043 0.6000 0.9088 0.0479
91 0.9125 0.2834 0.6500 0.8021 0.0480
92 0.8625 0.2936 0.6000 0.7502 0.0469
93 0.8500 0.3316 0.7000 0.8989 0.0478
94 0.8625 0.3162 0.7000 0.8503 0.0492
95 0.8625 0.2872 0.7500 0.8049 0.0484
96 0.8625 0.2927 0.6500 0.9009 0.0467
97 0.9000 0.2608 0.5500 0.9668 0.0463
98 0.9125 0.2660 0.7000 0.8201 0.0471
99 0.9375 0.2518 0.6500 0.8498 0.0488
100 0.9125 0.2171 0.6000 0.8606 0.0484
101 0.9125 0.2487 0.6000 0.8891 0.0480
102 0.9125 0.2211 0.6000 0.8848 0.0481
103 0.8500 0.3420 0.6500 0.9764 0.0478
104 0.8000 0.3593 0.7000 0.7942 0.0555
105 0.8375 0.3411 0.7000 0.7187 0.0476
106 0.8000 0.4129 0.7500 0.7040 0.0517
107 0.8125 0.4092 0.7500 0.6367 0.0483
108 0.8625 0.3590 0.7000 0.6562 0.0477
109 0.8375 0.3453 0.7500 0.6302 0.0496
110 0.8500 0.3398 0.8000 0.6272 0.0522
111 0.8375 0.3501 0.7000 0.6278 0.0482
112 0.8500 0.3326 0.7000 0.6236 0.0483
113 0.8875 0.3051 0.6500 0.6041 0.0475
114 0.8875 0.2830 0.7000 0.6154 0.0516
115 0.9000 0.2656 0.7500 0.6571 0.0479
116 0.9000 0.2880 0.7000 0.6976 0.0509
117 0.9125 0.2730 0.7000 0.6723 0.0481
118 0.8125 0.3488 0.7500 0.6479 0.0511
119 0.8375 0.2944 0.6500 0.7208 0.0480
120 0.8750 0.2794 0.6500 0.7448 0.0487
121 0.8625 0.2809 0.6000 0.7519 0.0490
122 0.9000 0.2815 0.6500 0.7535 0.0515
123 0.9250 0.2430 0.6500 0.8050 0.0486
124 0.8875 0.2376 0.7000 0.7925 0.0518
125 0.9250 0.2221 0.7000 0.7211 0.0488
126 0.8750 0.2591 0.7000 0.7647 0.0476
127 0.9000 0.2388 0.6500 0.8549 0.0472
128 0.9000 0.2110 0.7000 0.7897 0.0482
129 0.9250 0.2209 0.6000 0.8262 0.0481
130 0.9250 0.1975 0.6000 0.8276 0.0509
131 0.9500 0.1795 0.6500 0.8272 0.0483
132 0.9500 0.1570 0.7500 0.8091 0.0485
133 0.9250 0.1490 0.7500 0.7945 0.0489
134 0.9500 0.1413 0.6500 0.8440 0.0481
135 0.9625 0.1350 0.7000 0.8505 0.0490
136 0.9750 0.1263 0.8000 0.8885 0.0487
137 0.9750 0.1228 0.7500 0.9073 0.0485
138 0.9750 0.1154 0.6500 0.9325 0.0489
139 0.9750 0.1087 0.6500 1.0005 0.0487
140 0.9750 0.1028 0.6500 1.0371 0.0483
141 0.9750 0.0992 0.6000 1.0179 0.0496
142 0.9750 0.0980 0.6000 1.0732 0.0484
143 0.9750 0.0908 0.6500 1.1674 0.0489
144 0.9875 0.0918 0.6000 1.0711 0.0482
145 0.9500 0.1936 0.6500 1.1769 0.0481
146 0.9250 0.1667 0.6000 1.4184 0.0482
147 0.9500 0.1216 0.6000 1.2671 0.0486
148 0.9125 0.1431 0.7000 1.0005 0.0480
149 0.7750 0.6649 0.7500 0.8151 0.0471
150 0.7250 0.7617 0.7000 0.6950 0.0464
151 0.7500 0.6796 0.7500 0.4866 0.0473
152 0.8000 0.5247 0.7500 0.5657 0.0469
153 0.8250 0.3867 0.7500 0.6154 0.0480
154 0.8250 0.3795 0.8000 0.6868 0.0480
155 0.8125 0.4334 0.7000 0.6577 0.0475
156 0.8000 0.3663 0.7000 0.6197 0.0486
157 0.8500 0.3350 0.8000 0.6783 0.0478
158 0.8125 0.4372 0.7500 0.7116 0.0485
159 0.8250 0.4182 0.7500 0.6407 0.0475
160 0.7875 0.4886 0.6000 0.7112 0.0472
161 0.7625 0.4264 0.6500 0.6755 0.0467
162 0.8125 0.3918 0.7500 0.5528 0.0736
163 0.8250 0.3957 0.8500 0.4501 0.0344
164 0.8250 0.4287 0.7500 0.4623 0.0370
165 0.8375 0.3698 0.7000 0.6436 0.0401
166 0.7500 0.4505 0.7500 0.5887 0.0460
167 0.7875 0.4296 0.6500 0.5501 0.0453
168 0.7750 0.3653 0.6500 0.6277 0.0450
169 0.8250 0.3369 0.6500 0.6359 0.0456
170 0.8125 0.3333 0.6500 0.6402 0.0451
171 0.8250 0.3370 0.6500 0.7160 0.0454
172 0.8500 0.3226 0.6000 0.6773 0.0447
173 0.8250 0.3222 0.6500 0.5985 0.0482
174 0.8250 0.3377 0.7000 0.5805 0.0489
175 0.8625 0.2883 0.6500 0.6112 0.0479
176 0.8750 0.2725 0.6000 0.5857 0.0479
177 0.9000 0.2687 0.6000 0.5257 0.0476
178 0.8875 0.2756 0.6500 0.5174 0.0475
179 0.9000 0.2483 0.5500 0.5352 0.0477
180 0.9250 0.2417 0.5500 0.5449 0.0469
181 0.9250 0.2372 0.6000 0.5364 0.0507
182 0.9250 0.2300 0.6500 0.5159 0.0485
183 0.9250 0.2231 0.7000 0.4961 0.0472
184 0.9250 0.2179 0.7000 0.4890 0.0474
185 0.9250 0.2094 0.7000 0.4897 0.0476
186 0.9250 0.2028 0.7000 0.4964 0.0471
187 0.9250 0.1962 0.7000 0.5032 0.0470
188 0.9250 0.1877 0.7000 0.5073 0.0475
189 0.9375 0.1801 0.7000 0.5111 0.0469
190 0.9375 0.1726 0.7000 0.5142 0.0486
191 0.9500 0.1652 0.7000 0.5142 0.0479
192 0.9500 0.1581 0.7500 0.5173 0.0478
193 0.9625 0.1512 0.7500 0.5271 0.0473
194 0.9625 0.1436 0.7500 0.5398 0.0476
195 0.9625 0.1367 0.7500 0.5529 0.0482
196 0.9625 0.1294 0.7500 0.5623 0.0479
197 0.9625 0.1227 0.8000 0.5759 0.0471
198 0.9625 0.1153 0.8000 0.5915 0.0479
199 0.9625 0.1090 0.8000 0.6077 0.0475
200 0.9625 0.1134 0.8000 0.6246 0.0495
201 0.9750 0.1056 0.8000 0.6290 0.0470
202 0.9500 0.1128 0.8000 0.6667 0.0481
203 0.9750 0.1172 0.8000 0.6859 0.0483
204 0.9375 0.1677 0.7000 0.7682 0.0481
205 0.8750 0.2865 0.7000 0.7507 0.0487
206 0.8750 0.3235 0.6500 0.6018 0.0488
207 0.8500 0.2999 0.6500 0.6465 0.0490
208 0.8625 0.2775 0.7000 0.6885 0.0476
209 0.8750 0.3052 0.8000 0.6883 0.0481
210 0.8875 0.2854 0.7500 0.7631 0.0479
211 0.9000 0.2623 0.6500 0.8210 0.0582
212 0.8875 0.2646 0.7000 0.8063 0.0707
213 0.9125 0.2291 0.7000 0.7853 0.0557
214 0.9250 0.2264 0.6500 0.7610 0.0493
215 0.9125 0.2167 0.7000 0.7774 0.0482
216 0.9125 0.2001 0.7000 0.8153 0.0548
217 0.9375 0.1813 0.7000 0.7794 0.0522
218 0.9375 0.1766 0.7000 0.8366 0.0536
219 0.9125 0.2596 0.7000 0.8234 0.0529
220 0.8875 0.1965 0.7000 0.7550 0.0528
221 0.8375 0.3444 0.8000 0.6509 0.0488
222 0.8250 0.4719 0.8500 0.6177 0.0483
223 0.8750 0.3946 0.8000 0.5869 0.0468
224 0.8500 0.2908 0.7000 0.6148 0.0489
225 0.8375 0.2930 0.7500 0.6179 0.0488
226 0.8875 0.2560 0.7000 0.6133 0.0538
227 0.8875 0.2205 0.7000 0.5468 0.0552
228 0.9000 0.1888 0.8000 0.4636 0.0519
229 0.9000 0.1966 0.8000 0.5177 0.0505
230 0.9250 0.1716 0.8000 0.5546 0.0459
231 0.9375 0.1648 0.8000 0.5951 0.0469
232 0.9625 0.1568 0.7500 0.6044 0.0494
233 0.9625 0.1475 0.7500 0.5997 0.0488
234 0.9500 0.1402 0.7500 0.5847 0.0482
235 0.9500 0.1385 0.7500 0.5991 0.0490
236 0.9500 0.1297 0.7000 0.6183 0.0476
237 0.9625 0.1324 0.7000 0.6301 0.0479
238 0.9625 0.1239 0.7000 0.6022 0.0493
239 0.9625 0.1186 0.7500 0.5958 0.0524
240 0.9750 0.1136 0.8000 0.5931 0.0490
241 0.9750 0.1080 0.8000 0.5849 0.0477
242 0.9750 0.1055 0.8000 0.5962 0.0481
243 0.9750 0.1022 0.8000 0.6072 0.0465
244 0.9750 0.0981 0.8000 0.6183 0.0462
245 0.9750 0.0944 0.8000 0.6333 0.0474
246 0.9875 0.0910 0.8000 0.6458 0.0476
247 0.9875 0.0878 0.8000 0.6470 0.0475
248 0.9875 0.0851 0.8000 0.6277 0.0496
249 0.9875 0.0815 0.7500 0.5963 0.0476
250 0.9875 0.0785 0.7500 0.5966 0.0479
251 0.9875 0.0759 0.7000 0.5997 0.0481
252 0.9875 0.0734 0.7000 0.6094 0.0482
253 0.9875 0.0730 0.7500 0.5659 0.0487
254 0.8875 0.3195 0.7000 0.8996 0.0482
255 0.7000 1.2222 0.5500 1.5879 0.0481
256 0.6375 1.1513 0.6500 0.6751 0.0483
257 0.6375 0.7293 0.6500 0.5202 0.0483
258 0.7750 0.5563 0.7500 0.4589 0.0484
259 0.8125 0.5205 0.7000 0.5515 0.0472
260 0.7750 0.4955 0.8500 0.3910 0.0482
261 0.8125 0.4604 0.8500 0.4052 0.0485
262 0.7625 0.5233 0.8500 0.3787 0.0478
263 0.7750 0.4811 0.8500 0.3612 0.0760
264 0.8250 0.4416 0.8500 0.3742 0.0467
265 0.8250 0.4236 0.8500 0.3455 0.0470
266 0.8000 0.4119 0.9000 0.3379 0.0477
267 0.8250 0.4197 0.8500 0.3508 0.0476
268 0.8125 0.4070 0.8500 0.3733 0.0494
269 0.8125 0.3934 0.8500 0.3926 0.0483
270 0.8250 0.3855 0.8500 0.3820 0.0489
271 0.8250 0.3818 0.8500 0.3684 0.0489
272 0.8250 0.3852 0.8500 0.3660 0.0484
273 0.8250 0.3835 0.8500 0.3738 0.0490
274 0.8250 0.3789 0.8500 0.3865 0.0486
275 0.8250 0.3760 0.8500 0.3950 0.0491
276 0.8375 0.3756 0.8500 0.3963 0.0474
277 0.8375 0.3764 0.8500 0.3946 0.0475
278 0.8375 0.3760 0.8500 0.3945 0.0481
279 0.8375 0.3742 0.8500 0.3973 0.0485
280 0.8375 0.3725 0.8500 0.4016 0.0482
281 0.8375 0.3717 0.8500 0.4055 0.0492
282 0.8375 0.3718 0.8500 0.4081 0.0482
283 0.8375 0.3717 0.8500 0.4096 0.0481
284 0.8375 0.3705 0.8500 0.4109 0.0483
285 0.8375 0.3690 0.8500 0.4128 0.0489
286 0.8375 0.3678 0.8500 0.4154 0.0482
287 0.8375 0.3672 0.8000 0.4181 0.0485
288 0.8375 0.3667 0.8000 0.4205 0.0484
289 0.8375 0.3659 0.8000 0.4228 0.0495
290 0.8375 0.3646 0.8000 0.4254 0.0482
291 0.8375 0.3633 0.8000 0.4286 0.0474
292 0.8375 0.3623 0.8000 0.4323 0.0482
293 0.8375 0.3614 0.8000 0.4362 0.0483
294 0.8500 0.3604 0.8000 0.4401 0.0486
295 0.8500 0.3591 0.8000 0.4442 0.0479
296 0.8500 0.3577 0.8000 0.4486 0.0483
297 0.8500 0.3563 0.8000 0.4532 0.0485
298 0.8500 0.3550 0.8000 0.4579 0.0488
299 0.8500 0.3536 0.8000 0.4626 0.0481
300 0.8375 0.3519 0.8000 0.4674 0.0478
301 0.8375 0.3501 0.8000 0.4725 0.0479
302 0.8375 0.3482 0.8000 0.4776 0.0488
303 0.8375 0.3461 0.8000 0.4827 0.0521
304 0.8375 0.3437 0.8000 0.4876 0.0479
305 0.8375 0.3409 0.8000 0.4922 0.0492
306 0.8375 0.3375 0.8000 0.4959 0.0478
307 0.8375 0.3335 0.8000 0.4976 0.0484
308 0.8500 0.3279 0.8000 0.4932 0.0486
309 0.8750 0.3188 0.8000 0.4777 0.0486
310 0.8875 0.3046 0.7500 0.4602 0.0488
311 0.8875 0.3006 0.7500 0.4522 0.0493
312 0.9000 0.2832 0.7500 0.4590 0.0485
313 0.9125 0.2904 0.7500 0.5533 0.0494
314 0.8750 0.3709 0.8000 0.4469 0.0491
315 0.8750 0.3020 0.7500 0.4806 0.0480
316 0.8875 0.3069 0.7500 0.4524 0.0477
317 0.8625 0.3153 0.7500 0.4880 0.0490
318 0.8875 0.2790 0.7500 0.5936 0.0486
319 0.8750 0.2751 0.7000 0.6036 0.0483
320 0.8750 0.3012 0.7500 0.5652 0.0483
321 0.8000 0.3601 0.6500 0.5525 0.0486
322 0.7250 0.5724 0.8000 0.3807 0.0483
323 0.7875 0.4640 0.8000 0.3956 0.0482
324 0.8250 0.4499 0.8000 0.4744 0.0478
325 0.8125 0.4279 0.7000 0.4860 0.0478
326 0.8375 0.3379 0.7500 0.3684 0.0494
327 0.8125 0.3276 0.8000 0.3750 0.0486
328 0.8250 0.3318 0.7500 0.3890 0.0476
329 0.8500 0.3149 0.7000 0.4118 0.0480
330 0.8375 0.3130 0.7000 0.4836 0.0479
331 0.8375 0.2903 0.7000 0.5234 0.0478
332 0.8625 0.2829 0.7000 0.5557 0.0486
333 0.8750 0.2777 0.7000 0.5880 0.0490
334 0.8750 0.2780 0.7000 0.6107 0.0472
335 0.8875 0.2726 0.7000 0.6222 0.0494
336 0.8875 0.2641 0.7000 0.6307 0.0478
337 0.8750 0.2573 0.7000 0.6366 0.0473
338 0.9000 0.2533 0.7000 0.6371 0.0488
339 0.9000 0.2506 0.7000 0.6351 0.0485
340 0.9000 0.2465 0.7000 0.6289 0.0475
341 0.9000 0.2409 0.7000 0.6061 0.0476
342 0.9000 0.2359 0.7000 0.6022 0.0472
343 0.9125 0.2324 0.7000 0.6076 0.0482
344 0.9125 0.2287 0.7000 0.5934 0.0479
345 0.9125 0.2233 0.7000 0.5952 0.0473
346 0.9125 0.2175 0.7000 0.6026 0.0489
347 0.9250 0.2125 0.7000 0.6066 0.0491
348 0.9250 0.2079 0.7000 0.6224 0.0474
349 0.9250 0.2022 0.7000 0.6336 0.0474
350 0.9250 0.1950 0.7000 0.6433 0.0476
351 0.9250 0.1883 0.7000 0.6565 0.0499
352 0.9375 0.1823 0.7000 0.6620 0.0489
353 0.9625 0.1768 0.7000 0.6635 0.0478
354 0.9625 0.1710 0.7000 0.6727 0.0492
355 0.9750 0.1656 0.6500 0.6786 0.0493
356 0.9750 0.1574 0.6500 0.6985 0.0478
357 0.9750 0.1468 0.6500 0.7236 0.0483
358 0.9750 0.1360 0.7000 0.7709 0.0476
359 0.9750 0.1265 0.7000 0.8611 0.0480
360 0.9875 0.1181 0.7500 0.8643 0.0489
361 0.9750 0.1230 0.7000 0.8964 0.0491
362 0.9750 0.1164 0.7000 1.0536 0.0492
363 0.9875 0.1293 0.5500 1.0742 0.0480
364 0.8625 0.2321 0.6000 0.9431 0.0492
365 0.9125 0.2354 0.7000 0.9449 0.0869
366 0.8750 0.2783 0.6500 0.9208 0.0471
367 0.8750 0.2745 0.7000 0.8044 0.0477
368 0.8625 0.3007 0.7000 0.7891 0.0489
369 0.8750 0.2874 0.6500 0.8011 0.0484
370 0.8875 0.2400 0.6500 0.8552 0.0491
371 0.9125 0.2085 0.6500 0.9497 0.0485
372 0.9500 0.1849 0.6500 1.0631 0.0476
373 0.9500 0.1626 0.6500 1.1099 0.0496
374 0.9500 0.1596 0.6500 1.1004 0.0478
375 0.9625 0.1425 0.6500 1.0958 0.0480
376 0.9625 0.1358 0.6500 1.0905 0.0483
377 0.9625 0.1272 0.7000 1.0714 0.0484
378 0.9750 0.1186 0.7000 1.0611 0.0487
379 0.9750 0.1122 0.7000 1.0535 0.0491
380 0.9750 0.1073 0.7000 1.0392 0.0482
381 0.9875 0.1034 0.7000 1.0419 0.0483
382 0.9875 0.0983 0.7000 1.0500 0.0492
383 0.9875 0.0918 0.7000 1.0647 0.0480
384 0.9875 0.0865 0.7000 1.0779 0.0479
385 0.9875 0.0830 0.6500 1.0838 0.0484
386 0.9875 0.0837 0.6000 1.1293 0.0538
387 0.9750 0.0980 0.7000 1.1013 0.0552
388 0.9875 0.0849 0.7000 1.0850 0.0523
389 0.9875 0.0710 0.7000 1.0816 0.0522
390 0.9875 0.0781 0.6500 1.1112 0.0496
391 0.9875 0.0780 0.6500 1.1796 0.0496
392 0.9750 0.0906 0.7000 1.0748 0.0527
393 0.9750 0.0822 0.7000 1.0770 0.0497
394 0.9750 0.0675 0.6500 1.1661 0.0490
395 0.9750 0.0894 0.7000 1.0674 0.0477
396 0.9875 0.0665 0.7000 1.0774 0.0479
397 0.9875 0.0665 0.7000 1.1050 0.0491
398 0.9875 0.0562 0.7000 1.1365 0.0545
399 0.9875 0.0568 0.7000 1.1410 0.0526
400 0.9875 0.0439 0.6500 1.1599 0.0489
401 1.0000 0.0484 0.7000 1.0525 0.0463
402 0.9000 0.2785 0.6000 1.5656 0.0477
403 0.8875 0.2715 0.5000 1.4618 0.0495
404 0.7750 0.7322 0.6000 1.5368 0.0530
405 0.7500 0.7112 0.6500 1.4800 0.0513
406 0.8125 0.5322 0.6000 1.1641 0.0522
407 0.7500 0.6612 0.6500 1.2108 0.0540
408 0.7375 0.6434 0.7500 0.6617 0.0534
409 0.7875 0.4863 0.7000 0.6422 0.0483
410 0.8125 0.5088 0.7000 0.6268 0.0524
411 0.8250 0.4936 0.7500 0.5808 0.0478
412 0.8500 0.4683 0.7500 0.5789 0.0492
413 0.8125 0.4889 0.7000 0.5859 0.0543
414 0.8125 0.4907 0.7500 0.5611 0.0521
415 0.8125 0.4643 0.7500 0.5490 0.0515
416 0.8500 0.4509 0.7500 0.5487 0.0476
417 0.8500 0.4414 0.7500 0.5507 0.0470
418 0.8625 0.4347 0.7500 0.5604 0.0469
419 0.8375 0.4343 0.7500 0.5680 0.0487
420 0.8375 0.4291 0.7500 0.5688 0.0490
421 0.8375 0.4170 0.7500 0.5709 0.0546
422 0.8375 0.4057 0.7500 0.5768 0.0687
423 0.8500 0.3975 0.7500 0.5842 0.0542
424 0.8375 0.3918 0.7500 0.5922 0.0505
425 0.8375 0.3880 0.7500 0.5978 0.0464
426 0.8375 0.3832 0.7500 0.5995 0.0482
427 0.8375 0.3761 0.7500 0.5995 0.0478
428 0.8500 0.3689 0.7500 0.5992 0.0480
429 0.8500 0.3629 0.7500 0.5990 0.0487
430 0.8500 0.3582 0.7500 0.5992 0.0525
431 0.8625 0.3542 0.7500 0.5991 0.0541
432 0.8625 0.3494 0.7500 0.5981 0.0533
433 0.8625 0.3433 0.7500 0.5962 0.0527
434 0.8625 0.3366 0.7500 0.5937 0.0530
435 0.8625 0.3301 0.7500 0.5901 0.0470
436 0.8625 0.3243 0.7500 0.5849 0.0477
437 0.8625 0.3191 0.7500 0.5777 0.0485
438 0.8750 0.3142 0.7500 0.5690 0.0494
439 0.8750 0.3097 0.8000 0.5603 0.0534
440 0.8750 0.3062 0.8000 0.5524 0.0562
441 0.8750 0.3029 0.8000 0.5444 0.0521
442 0.8750 0.2994 0.8000 0.5346 0.0533
443 0.8750 0.2954 0.8000 0.5228 0.0490
444 0.8750 0.2912 0.8000 0.5118 0.0471
445 0.8750 0.2876 0.8000 0.5065 0.0474
446 0.8750 0.2840 0.8000 0.5110 0.0476
447 0.8875 0.2788 0.8000 0.5161 0.0495
448 0.8875 0.2743 0.8000 0.5144 0.0479
449 0.9000 0.2694 0.8000 0.5069 0.0477
450 0.9000 0.2643 0.8000 0.4993 0.0482
451 0.9125 0.2597 0.8000 0.4959 0.0479
452 0.9250 0.2556 0.8000 0.4951 0.0489
453 0.9250 0.2515 0.8000 0.4940 0.0479
454 0.9250 0.2472 0.8000 0.4924 0.0492
455 0.9250 0.2431 0.8000 0.4916 0.0484
456 0.9250 0.2395 0.8000 0.4912 0.0488
457 0.9250 0.2362 0.8000 0.4895 0.0492
458 0.9375 0.2329 0.8000 0.4870 0.0481
459 0.9375 0.2298 0.8000 0.4855 0.0489
460 0.9500 0.2268 0.8000 0.4849 0.0483
461 0.9500 0.2239 0.8000 0.4838 0.0482
462 0.9500 0.2210 0.8000 0.4823 0.0494
463 0.9500 0.2182 0.8000 0.4809 0.0475
464 0.9500 0.2154 0.8000 0.4795 0.0853
465 0.9500 0.2128 0.8500 0.4779 0.0475
466 0.9500 0.2103 0.8500 0.4756 0.0480
467 0.9500 0.2079 0.8500 0.4738 0.0480
468 0.9500 0.2056 0.8500 0.4720 0.0486
469 0.9500 0.2035 0.8500 0.4708 0.0486
470 0.9500 0.2016 0.8500 0.4692 0.0491
471 0.9500 0.1997 0.8500 0.4689 0.0485
472 0.9500 0.1978 0.8500 0.4673 0.0476
473 0.9500 0.1962 0.8500 0.4702 0.0481
474 0.9500 0.1939 0.8500 0.4640 0.0480
475 0.9500 0.1936 0.8500 0.4809 0.0481
476 0.9500 0.1895 0.8500 0.4416 0.0506
477 0.9375 0.2021 0.8500 0.4994 0.0492
478 0.9250 0.2148 0.8500 0.4708 0.0482
479 0.9375 0.1849 0.8500 0.4704 0.0482
480 0.9375 0.1890 0.8500 0.5104 0.0487
481 0.9375 0.2047 0.8500 0.4812 0.0490
482 0.9500 0.1817 0.8500 0.4540 0.0488
483 0.9375 0.1904 0.8500 0.5136 0.0471
484 0.9250 0.2068 0.8500 0.5072 0.0481
485 0.9500 0.1791 0.8500 0.4592 0.0491
486 0.9250 0.1972 0.8500 0.5232 0.0480
487 0.9250 0.1928 0.8500 0.5220 0.0487
488 0.9375 0.1768 0.8500 0.4724 0.0480
489 0.9250 0.1986 0.8500 0.5328 0.0487
490 0.9250 0.1961 0.8500 0.5425 0.0487
491 0.9375 0.1808 0.8500 0.4918 0.0480
492 0.9125 0.2058 0.8000 0.5390 0.0490
493 0.9250 0.2228 0.7500 0.5032 0.0489
494 0.9125 0.2471 0.7500 0.5494 0.0495
495 0.9250 0.2149 0.7500 0.6273 0.0473
496 0.9000 0.2181 0.8000 0.6675 0.0477
497 0.9500 0.1808 0.8000 0.6283 0.0471
498 0.9250 0.1788 0.8000 0.5933 0.0486
499 0.9250 0.1754 0.8000 0.5724 0.0486
500 0.9500 0.1736 0.8000 0.5551 0.0486
Out[21]:
<class 'skorch.classifier.NeuralNetClassifier'>[initialized](
module_=LSTM(
(rnn): LSTM(1, 32, batch_first=True)
(fc): Linear(in_features=32, out_features=2, bias=True)
),
)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
<class 'skorch.classifier.NeuralNetClassifier'>[initialized](
module_=LSTM(
(rnn): LSTM(1, 32, batch_first=True)
(fc): Linear(in_features=32, out_features=2, bias=True)
),
)In [22]:
y_pred = net.predict(X_test)
accuracy_score(y_test, y_pred)
Out[22]:
0.69
In [23]:
history = net.history
plt.plot(history[:, 'train_loss'], label='train_loss')
plt.plot(history[:, 'valid_loss'], label='valid_loss')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.show()
plt.plot(history[:, 'train_acc'], label='train_acc')
plt.plot(history[:, 'valid_acc'], label='valid_acc')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.show()
Trying to fix the unstable training¶
In [24]:
net = NeuralNetClassifier(
LSTM,
module__input_size=X_train.shape[1],
module__hidden_size=32,
module__output_size=len(le.classes_),
criterion=nn.CrossEntropyLoss,
optimizer=torch.optim.Adam,
max_epochs=100,
batch_size=32,
lr=1e-3,
callbacks=[
EpochScoring(
scoring='accuracy',
name='train_acc',
on_train=True,
),
Checkpoint(
monitor="valid_acc_best",
load_best=True,
f_history=None,
f_optimizer=None,
),
],
device=DEVICE,
train_split=ValidSplit(cv=0.2, stratified=True)
)
net
Out[24]:
<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized]( module=<class '__main__.LSTM'>, module__hidden_size=32, module__input_size=1, module__output_size=2, )In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized]( module=<class '__main__.LSTM'>, module__hidden_size=32, module__input_size=1, module__output_size=2, )
In [25]:
net.fit(X_train, y_train)
epoch train_acc train_loss valid_acc valid_loss cp dur
------- ----------- ------------ ----------- ------------ ---- ------
1 0.7000 0.6857 0.7000 0.6801 + 0.0453
2 0.6875 0.6786 0.7000 0.6736 0.0432
3 0.6875 0.6718 0.7000 0.6673 0.0432
4 0.6875 0.6650 0.7000 0.6611 0.0431
5 0.6875 0.6582 0.7000 0.6547 0.0454
6 0.6875 0.6510 0.7000 0.6480 0.0476
7 0.6875 0.6435 0.7000 0.6410 0.0486
8 0.6875 0.6352 0.7000 0.6334 0.0481
9 0.6875 0.6259 0.7000 0.6250 0.0476
10 0.6875 0.6152 0.7000 0.6156 0.0472
11 0.6875 0.6030 0.7000 0.6053 0.0506
12 0.6875 0.5892 0.7000 0.5947 0.0540
13 0.6875 0.5752 0.7000 0.5861 0.0518
14 0.6875 0.5637 0.7000 0.5813 0.0492
15 0.6875 0.5544 0.7000 0.5764 0.0476
16 0.7000 0.5437 0.7000 0.5708 0.0480
17 0.7000 0.5340 0.7000 0.5673 0.0480
18 0.7125 0.5276 0.7000 0.5651 0.0477
19 0.7250 0.5227 0.6500 0.5626 0.0473
20 0.7500 0.5176 0.7000 0.5592 0.0535
21 0.7750 0.5121 0.7000 0.5558 0.0555
22 0.7750 0.5065 0.7000 0.5533 0.0489
23 0.7625 0.5011 0.7000 0.5518 0.0463
24 0.7625 0.4961 0.7000 0.5513 0.0466
25 0.7750 0.4914 0.7500 0.5513 + 0.0469
26 0.7500 0.4870 0.7000 0.5514 0.0502
27 0.7875 0.4824 0.7000 0.5514 0.0490
28 0.8000 0.4775 0.7000 0.5514 0.0501
29 0.8125 0.4723 0.7500 0.5516 0.0512
30 0.8125 0.4669 0.7500 0.5519 0.0545
31 0.8125 0.4612 0.7500 0.5525 0.0493
32 0.8250 0.4553 0.7000 0.5531 0.0478
33 0.8250 0.4489 0.7000 0.5532 0.0507
34 0.8375 0.4410 0.7000 0.5524 0.0489
35 0.8375 0.4317 0.7000 0.5526 0.0494
36 0.8500 0.4184 0.6500 0.5580 0.0812
37 0.8250 0.4040 0.6500 0.5608 0.0338
38 0.8000 0.3886 0.7000 0.5540 0.0352
39 0.8125 0.3673 0.7000 0.5561 0.0373
40 0.8250 0.3542 0.7000 0.5490 0.0422
41 0.8500 0.3357 0.7000 0.5608 0.0517
42 0.8500 0.3265 0.7000 0.5508 0.0486
43 0.8625 0.3132 0.7000 0.5546 0.0492
44 0.8625 0.3041 0.7000 0.5561 0.0489
45 0.8875 0.2975 0.7500 0.5550 0.0468
46 0.8875 0.2922 0.7000 0.5604 0.0482
47 0.8875 0.2895 0.7000 0.5627 0.0479
48 0.8875 0.2874 0.7500 0.5637 0.0513
49 0.8875 0.2853 0.7000 0.5684 0.0495
50 0.8875 0.2832 0.7000 0.5740 0.0486
51 0.8750 0.2812 0.7000 0.5770 0.0483
52 0.8875 0.2787 0.7500 0.5801 0.0491
53 0.8875 0.2757 0.7500 0.5851 0.0478
54 0.8875 0.2725 0.7500 0.5892 0.0471
55 0.8875 0.2692 0.7500 0.5919 0.0489
56 0.8875 0.2657 0.7500 0.5963 0.0674
57 0.8875 0.2622 0.7500 0.6013 0.0519
58 0.8875 0.2589 0.7500 0.6053 0.0525
59 0.8875 0.2558 0.7500 0.6105 0.0495
60 0.8875 0.2529 0.7500 0.6157 0.0474
61 0.8875 0.2502 0.7500 0.6197 0.0491
62 0.8875 0.2475 0.7500 0.6240 0.0486
63 0.8875 0.2450 0.7500 0.6271 0.0532
64 0.8875 0.2425 0.7500 0.6291 0.0508
65 0.8875 0.2399 0.7500 0.6316 0.0498
66 0.8875 0.2368 0.7500 0.6343 0.0490
67 0.9000 0.2330 0.7500 0.6378 0.0517
68 0.9000 0.2286 0.7500 0.6391 0.0473
69 0.9000 0.2244 0.7500 0.6342 0.0526
70 0.8875 0.2203 0.8000 0.6273 + 0.0478
71 0.8875 0.2149 0.8000 0.6359 0.0468
72 0.8750 0.2122 0.8000 0.6200 0.0513
73 0.8875 0.2103 0.7500 0.6561 0.0490
74 0.9000 0.2062 0.8000 0.5801 0.0487
75 0.9250 0.1947 0.6500 0.8792 0.0509
76 0.8750 0.2590 0.8000 0.5193 0.0470
77 0.9125 0.1912 0.7500 0.5673 0.0472
78 0.8875 0.2076 0.8500 0.4998 + 0.0468
79 0.9125 0.1836 0.8500 0.5467 0.0484
80 0.9000 0.1880 0.7500 0.6710 0.0475
81 0.9250 0.1841 0.7500 0.5924 0.0534
82 0.9125 0.1806 0.8000 0.5183 0.0481
83 0.9250 0.1739 0.8500 0.5257 0.0479
84 0.9125 0.1693 0.8500 0.5601 0.0474
85 0.9125 0.1708 0.7500 0.6657 0.0460
86 0.9125 0.1681 0.7500 0.6197 0.0462
87 0.9125 0.1653 0.8000 0.5859 0.0471
88 0.9125 0.1618 0.8500 0.5720 0.0471
89 0.9125 0.1589 0.8500 0.5924 0.0469
90 0.9375 0.1571 0.7500 0.6587 0.0462
91 0.9125 0.1666 0.8000 0.7594 0.0440
92 0.9125 0.1626 0.8500 0.5438 0.0463
93 0.9125 0.1568 0.8500 0.5539 0.0463
94 0.9250 0.1476 0.8500 0.6049 0.0487
95 0.9375 0.1515 0.8500 0.6129 0.0503
96 0.9250 0.1556 0.8500 0.6067 0.0456
97 0.9375 0.1473 0.8000 0.6252 0.0502
98 0.9250 0.1456 0.8500 0.5884 0.0474
99 0.9250 0.1494 0.8500 0.6206 0.0508
100 0.9375 0.1374 0.8500 0.6132 0.0463
Out[25]:
<class 'skorch.classifier.NeuralNetClassifier'>[initialized](
module_=LSTM(
(rnn): LSTM(1, 32, batch_first=True)
(fc): Linear(in_features=32, out_features=2, bias=True)
),
)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
<class 'skorch.classifier.NeuralNetClassifier'>[initialized](
module_=LSTM(
(rnn): LSTM(1, 32, batch_first=True)
(fc): Linear(in_features=32, out_features=2, bias=True)
),
)In [26]:
y_pred = net.predict(X_test)
accuracy_score(y_test, y_pred)
Out[26]:
0.77
In [27]:
history = net.history
plt.plot(history[:, 'train_loss'], label='train_loss')
plt.plot(history[:, 'valid_loss'], label='valid_loss')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.show()
plt.plot(history[:, 'train_acc'], label='train_acc')
plt.plot(history[:, 'valid_acc'], label='valid_acc')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.show()
Bidirectional¶
In [28]:
# build a vanilla RNN with a classification head
class LSTM2(nn.Module):
def __init__(
self,
input_size,
hidden_size,
output_size,
bidirectional=False,
num_layers=1,
):
super(LSTM2, self).__init__()
self.rnn = nn.LSTM(
input_size,
hidden_size,
batch_first=True,
bidirectional=bidirectional,
num_layers=num_layers
)
self.fc = nn.Linear(
hidden_size if not bidirectional else hidden_size * 2,
output_size
)
def forward(self, x):
x = torch.swapaxes(x, 1, 2) # swap the last two dimensions to match RNN input shape
out, _ = self.rnn(x)
out = out[:, -1, :] # take the last output of the sequence
out = self.fc(out)
return out
In [29]:
net = NeuralNetClassifier(
LSTM2,
module__input_size=X_train.shape[1],
module__hidden_size=32,
module__output_size=len(le.classes_),
module__bidirectional=True,
criterion=nn.CrossEntropyLoss,
optimizer=torch.optim.Adam,
max_epochs=100,
batch_size=100,
lr=1e-4,
callbacks=[
EpochScoring(
scoring='accuracy',
name='train_acc',
on_train=True,
),
Checkpoint(
monitor="valid_acc_best",
load_best=True,
f_history=None,
f_optimizer=None,
),
],
device=DEVICE,
train_split=ValidSplit(cv=0.2, stratified=True)
)
net
Out[29]:
<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized]( module=<class '__main__.LSTM2'>, module__bidirectional=True, module__hidden_size=32, module__input_size=1, module__output_size=2, )In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized]( module=<class '__main__.LSTM2'>, module__bidirectional=True, module__hidden_size=32, module__input_size=1, module__output_size=2, )
In [30]:
net.fit(X_train, y_train)
epoch train_acc train_loss valid_acc valid_loss cp dur
------- ----------- ------------ ----------- ------------ ---- ------
1 0.3000 0.7084 0.3000 0.7098 + 0.7769
2 0.3000 0.7080 0.3000 0.7094 0.0346
3 0.3000 0.7077 0.3000 0.7091 0.0233
4 0.3000 0.7074 0.3000 0.7087 0.0247
5 0.3000 0.7071 0.3000 0.7084 0.0299
6 0.2875 0.7068 0.3000 0.7080 0.0337
7 0.2875 0.7064 0.3000 0.7077 0.0250
8 0.2875 0.7061 0.3000 0.7074 0.0245
9 0.2875 0.7058 0.3000 0.7070 0.0246
10 0.2875 0.7055 0.3000 0.7067 0.0252
11 0.2750 0.7052 0.3000 0.7063 0.0277
12 0.2750 0.7049 0.3000 0.7060 0.0281
13 0.2750 0.7045 0.3000 0.7056 0.0317
14 0.2625 0.7042 0.3000 0.7053 0.0326
15 0.2625 0.7039 0.3000 0.7049 0.0341
16 0.2500 0.7036 0.3000 0.7046 0.0346
17 0.2375 0.7033 0.3000 0.7043 0.0337
18 0.2125 0.7030 0.3000 0.7039 0.0338
19 0.2125 0.7026 0.3000 0.7036 0.0342
20 0.2125 0.7023 0.3000 0.7032 0.0358
21 0.2125 0.7020 0.2500 0.7029 0.0336
22 0.2125 0.7017 0.2500 0.7025 0.0326
23 0.2125 0.7014 0.2500 0.7022 0.0346
24 0.2125 0.7011 0.2500 0.7019 0.0347
25 0.2125 0.7007 0.2500 0.7015 0.0335
26 0.2125 0.7004 0.2500 0.7012 0.0347
27 0.2125 0.7001 0.2500 0.7008 0.0330
28 0.2125 0.6998 0.3000 0.7005 0.0324
29 0.2000 0.6995 0.3000 0.7001 0.0230
30 0.1875 0.6992 0.2500 0.6998 0.0266
31 0.1625 0.6988 0.2500 0.6994 0.0250
32 0.1625 0.6985 0.2500 0.6991 0.0271
33 0.1625 0.6982 0.2500 0.6988 0.0278
34 0.1875 0.6979 0.2000 0.6984 0.0287
35 0.1750 0.6976 0.2000 0.6981 0.0297
36 0.1875 0.6972 0.2000 0.6977 0.0322
37 0.2375 0.6969 0.2000 0.6974 0.0350
38 0.2625 0.6966 0.2000 0.6970 0.0658
39 0.3250 0.6963 0.1500 0.6967 0.0384
40 0.3625 0.6960 0.2000 0.6963 0.0394
41 0.3875 0.6956 0.2000 0.6960 0.0483
42 0.4375 0.6953 0.2500 0.6956 0.0491
43 0.4625 0.6950 0.3000 0.6953 0.0442
44 0.5250 0.6947 0.3000 0.6949 0.0260
45 0.5500 0.6944 0.4000 0.6946 + 0.0277
46 0.6000 0.6940 0.4500 0.6942 + 0.0264
47 0.6375 0.6937 0.5000 0.6939 + 0.0288
48 0.6625 0.6934 0.7000 0.6935 + 0.0326
49 0.6750 0.6931 0.7000 0.6932 0.0333
50 0.6875 0.6927 0.7000 0.6928 0.0337
51 0.6875 0.6924 0.7000 0.6925 0.0328
52 0.6875 0.6921 0.7000 0.6921 0.0233
53 0.6875 0.6918 0.7000 0.6918 0.0272
54 0.6875 0.6914 0.7000 0.6914 0.0280
55 0.6875 0.6911 0.7000 0.6911 0.0262
56 0.7000 0.6908 0.7000 0.6907 0.0279
57 0.7000 0.6905 0.7000 0.6903 0.0279
58 0.6875 0.6901 0.7000 0.6900 0.0309
59 0.6875 0.6898 0.7000 0.6896 0.0321
60 0.6875 0.6895 0.7000 0.6893 0.0325
61 0.6875 0.6891 0.7000 0.6889 0.0357
62 0.6875 0.6888 0.7000 0.6885 0.0330
63 0.6875 0.6885 0.7000 0.6882 0.0240
64 0.6875 0.6881 0.7000 0.6878 0.0241
65 0.6875 0.6878 0.7000 0.6875 0.0263
66 0.6875 0.6875 0.7000 0.6871 0.0271
67 0.6875 0.6871 0.7000 0.6867 0.0287
68 0.6875 0.6868 0.7000 0.6864 0.0283
69 0.6875 0.6865 0.7000 0.6860 0.0305
70 0.6875 0.6861 0.7000 0.6856 0.0317
71 0.6875 0.6858 0.7000 0.6852 0.0328
72 0.6875 0.6854 0.7000 0.6849 0.0323
73 0.6875 0.6851 0.7000 0.6845 0.0354
74 0.6875 0.6848 0.7000 0.6841 0.0385
75 0.6875 0.6844 0.7000 0.6838 0.0336
76 0.6875 0.6841 0.7000 0.6834 0.0389
77 0.6875 0.6837 0.7000 0.6830 0.0325
78 0.6875 0.6834 0.7000 0.6826 0.0243
79 0.6875 0.6830 0.7000 0.6822 0.0236
80 0.6875 0.6827 0.7000 0.6819 0.0239
81 0.6875 0.6823 0.7000 0.6815 0.0286
82 0.6875 0.6820 0.7000 0.6811 0.0258
83 0.6875 0.6816 0.7000 0.6807 0.0295
84 0.6875 0.6813 0.7000 0.6803 0.0637
85 0.6875 0.6809 0.7000 0.6799 0.0371
86 0.6875 0.6805 0.7000 0.6795 0.0346
87 0.6875 0.6802 0.7000 0.6791 0.0323
88 0.6875 0.6798 0.7000 0.6788 0.0244
89 0.6875 0.6795 0.7000 0.6784 0.0241
90 0.6875 0.6791 0.7000 0.6780 0.0251
91 0.6875 0.6787 0.7000 0.6776 0.0257
92 0.6875 0.6784 0.7000 0.6772 0.0275
93 0.6875 0.6780 0.7000 0.6768 0.0293
94 0.6875 0.6776 0.7000 0.6764 0.0381
95 0.6875 0.6773 0.7000 0.6760 0.0348
96 0.6875 0.6769 0.7000 0.6756 0.0244
97 0.6875 0.6765 0.7000 0.6751 0.0240
98 0.6875 0.6761 0.7000 0.6747 0.0261
99 0.6875 0.6758 0.7000 0.6743 0.0496
100 0.6875 0.6754 0.7000 0.6739 0.0293
Out[30]:
<class 'skorch.classifier.NeuralNetClassifier'>[initialized](
module_=LSTM2(
(rnn): LSTM(1, 32, batch_first=True, bidirectional=True)
(fc): Linear(in_features=64, out_features=2, bias=True)
),
)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
<class 'skorch.classifier.NeuralNetClassifier'>[initialized](
module_=LSTM2(
(rnn): LSTM(1, 32, batch_first=True, bidirectional=True)
(fc): Linear(in_features=64, out_features=2, bias=True)
),
)In [31]:
y_pred = net.predict(X_test)
accuracy_score(y_test, y_pred)
Out[31]:
0.63
In [32]:
history = net.history
plt.plot(history[:, 'train_loss'], label='train_loss')
plt.plot(history[:, 'valid_loss'], label='valid_loss')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.show()
plt.plot(history[:, 'train_acc'], label='train_acc')
plt.plot(history[:, 'valid_acc'], label='valid_acc')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.show()
GRU¶
In [33]:
from skorch.callbacks import Callback
import torch
class GradientClipping(Callback):
def __init__(self, max_norm=1.0, norm_type=2):
self.max_norm = max_norm
self.norm_type = norm_type
def on_grad_computed(self, net, named_parameters, **kwargs):
params = [p for _, p in named_parameters if p.grad is not None]
torch.nn.utils.clip_grad_norm_(
params,
max_norm=self.max_norm,
norm_type=self.norm_type,
)
In [34]:
class GRU(nn.Module):
def __init__(self, input_size, hidden_size, output_size):
super(GRU, self).__init__()
self.rnn = nn.GRU(input_size, hidden_size, batch_first=True)
self.fc = nn.Linear(hidden_size, output_size)
def forward(self, x):
x = torch.swapaxes(x, 1, 2)
out, _ = self.rnn(x)
out = out[:, -1, :]
out = self.fc(out)
return out
In [35]:
net = NeuralNetClassifier(
GRU,
module__input_size=X_train.shape[1],
module__hidden_size=64,
module__output_size=len(le.classes_),
criterion=nn.CrossEntropyLoss,
optimizer=torch.optim.Adam,
max_epochs=100,
batch_size=32,
callbacks=[
EpochScoring(
scoring='accuracy',
name='train_acc',
on_train=True,
),
Checkpoint(
monitor="valid_acc_best",
load_best=True,
f_history=None,
f_optimizer=None,
),
GradientClipping(max_norm=1.0)
],
device="cpu", # "mps" is known to cause issues with GRUs
train_split=ValidSplit(cv=0.2, stratified=True)
)
net
Out[35]:
<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized]( module=<class '__main__.GRU'>, module__hidden_size=64, module__input_size=1, module__output_size=2, )In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
<class 'skorch.classifier.NeuralNetClassifier'>[uninitialized]( module=<class '__main__.GRU'>, module__hidden_size=64, module__input_size=1, module__output_size=2, )
In [36]:
net.fit(X_train, y_train)
epoch train_acc train_loss valid_acc valid_loss cp dur
------- ----------- ------------ ----------- ------------ ---- ------
1 0.6750 0.7053 0.7000 0.5639 + 1.3439
2 0.7000 0.5865 0.9000 0.5094 + 0.2875
3 0.7500 0.5632 0.9500 0.4361 + 0.4382
4 0.7875 0.5380 0.9500 0.3573 0.4425
5 0.7625 0.5289 0.9500 0.3439 0.3049
6 0.7500 0.5090 0.9500 0.3540 0.2860
7 0.7750 0.5005 0.9500 0.3585 0.2644
8 0.7875 0.4935 0.9000 0.3482 0.2384
9 0.7875 0.4831 0.9000 0.3412 0.2543
10 0.8125 0.4765 0.8500 0.3430 0.2793
11 0.8125 0.4726 0.8500 0.3421 0.2071
12 0.8250 0.4717 0.8500 0.3400 0.1949
13 0.8125 0.4691 0.8500 0.3354 0.2575
14 0.8250 0.4645 0.8500 0.3304 0.3352
15 0.8375 0.4608 0.8500 0.3241 0.3911
16 0.8375 0.4574 0.8500 0.3191 0.2106
17 0.8375 0.4545 0.8500 0.3143 0.3259
18 0.8375 0.4517 0.9000 0.3090 0.3147
19 0.8250 0.4497 0.9000 0.3052 0.2140
20 0.8125 0.4467 0.9000 0.3022 0.2960
21 0.8375 0.4414 0.9000 0.2975 0.2310
22 0.8375 0.4364 0.9000 0.2926 0.2160
23 0.8250 0.4290 0.9000 0.2856 0.1752
24 0.8375 0.4197 0.9000 0.2765 0.2271
25 0.8250 0.4148 0.9000 0.2720 0.2107
26 0.8250 0.4043 0.9000 0.2686 0.2139
27 0.8250 0.3982 0.9000 0.2678 0.2264
28 0.8250 0.3881 0.8500 0.2879 0.2545
29 0.8250 0.4014 0.9000 0.3107 0.3051
30 0.7625 0.5192 0.6500 0.6098 0.2743
31 0.7875 0.6219 0.8000 0.4010 0.1899
32 0.7125 0.6753 0.7500 0.5543 0.2818
33 0.5750 0.9795 0.7000 0.6402 0.2976
34 0.6750 0.7556 0.7000 0.7502 0.2641
35 0.6500 0.8862 0.8000 0.4041 0.2589
36 0.7750 0.6850 0.8000 0.4926 0.3294
37 0.7125 0.7073 0.6000 1.1492 0.7061
38 0.4125 1.2818 0.7000 0.5791 0.2474
39 0.5625 0.9430 0.6500 0.6882 0.3276
40 0.6125 0.7265 0.7500 0.4987 0.3689
41 0.7250 0.5653 0.8500 0.3968 0.3355
42 0.7750 0.5109 0.9500 0.3367 0.9275
43 0.8125 0.4893 0.9500 0.3070 0.1874
44 0.7875 0.4683 0.9500 0.3024 0.1739
45 0.8375 0.4450 0.9000 0.2898 0.2120
46 0.8250 0.4465 0.9500 0.2604 0.1874
47 0.8250 0.4083 0.9000 0.3389 0.2272
48 0.8125 0.4496 0.9500 0.2352 0.2402
49 0.8250 0.3925 0.9500 0.2801 0.3782
50 0.8125 0.4406 0.8500 0.4050 0.2153
51 0.8125 0.5357 0.7500 0.4172 0.1492
52 0.7250 0.5544 0.9000 0.3083 0.2726
53 0.8000 0.4376 0.9500 0.2996 0.2134
54 0.7875 0.4527 0.9500 0.3046 0.1776
55 0.8125 0.4365 0.9500 0.2906 0.2328
56 0.8125 0.4290 0.9000 0.3180 0.2814
57 0.8500 0.4381 0.9000 0.3189 0.2162
58 0.8500 0.4344 0.9000 0.3155 0.3249
59 0.8375 0.4192 0.9000 0.3048 0.2483
60 0.8500 0.4020 0.9000 0.2961 0.2823
61 0.8625 0.3845 0.9000 0.2887 0.2576
62 0.8500 0.3815 0.9000 0.2897 0.2442
63 0.8500 0.3690 0.9000 0.3018 0.2291
64 0.8750 0.3701 0.9000 0.2905 0.2775
65 0.8625 0.3583 0.9000 0.2897 0.1238
66 0.8625 0.3555 0.8500 0.3087 0.3612
67 0.8125 0.4040 0.8500 0.3558 0.2995
68 0.8250 0.4305 0.9000 0.3272 0.2323
69 0.8625 0.3501 0.8500 0.3415 0.2238
70 0.8250 0.3860 0.8000 0.3765 0.4500
71 0.7625 0.4988 0.9000 0.3623 0.5104
72 0.8375 0.4074 0.8500 0.3918 0.2390
73 0.7750 0.4673 0.8500 0.4056 0.4167
74 0.7125 0.5530 0.8500 0.3345 0.6865
75 0.7750 0.5107 0.9500 0.3106 0.4686
76 0.8125 0.4363 0.9500 0.3215 0.3152
77 0.8125 0.4226 0.9500 0.3246 0.2422
78 0.8000 0.4374 0.9000 0.3279 0.3898
79 0.8375 0.4429 0.8500 0.3220 0.2097
80 0.8125 0.4273 0.8500 0.3105 0.2531
81 0.8250 0.4024 0.8500 0.3061 0.3264
82 0.8375 0.3863 0.8500 0.3084 0.3306
83 0.8125 0.3768 0.8500 0.3075 0.2521
84 0.8250 0.3711 0.8500 0.3169 0.2937
85 0.8250 0.3619 0.8000 0.3104 0.2731
86 0.8250 0.3608 0.8000 0.3763 0.4367
87 0.8000 0.4437 0.8000 0.3223 0.2132
88 0.8125 0.4102 0.8000 0.2902 0.2763
89 0.8375 0.3450 0.9000 0.2669 0.3235
90 0.8500 0.3388 0.9500 0.2572 0.3383
91 0.8625 0.3270 0.9000 0.2562 0.3024
92 0.8625 0.3250 0.9000 0.2630 0.2532
93 0.8750 0.3183 0.9000 0.2673 0.3419
94 0.8625 0.3205 0.9000 0.2660 0.3150
95 0.8625 0.3177 0.9000 0.2555 0.2714
96 0.8500 0.3140 0.9500 0.2454 0.2420
97 0.8500 0.3062 0.9500 0.2327 0.3355
98 0.8625 0.3035 0.9500 0.2269 0.3701
99 0.8500 0.3028 0.9500 0.2277 0.3076
100 0.8625 0.2913 0.9500 0.2266 0.3614
Out[36]:
<class 'skorch.classifier.NeuralNetClassifier'>[initialized](
module_=GRU(
(rnn): GRU(1, 64, batch_first=True)
(fc): Linear(in_features=64, out_features=2, bias=True)
),
)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
<class 'skorch.classifier.NeuralNetClassifier'>[initialized](
module_=GRU(
(rnn): GRU(1, 64, batch_first=True)
(fc): Linear(in_features=64, out_features=2, bias=True)
),
)In [37]:
y_pred = net.predict(X_test)
accuracy_score(y_test, y_pred)
Out[37]:
0.66
In [38]:
history = net.history
plt.plot(history[:, 'train_loss'], label='train_loss')
plt.plot(history[:, 'valid_loss'], label='valid_loss')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.show()
plt.plot(history[:, 'train_acc'], label='train_acc')
plt.plot(history[:, 'valid_acc'], label='valid_acc')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.show()
Exercise¶
Solve the classfication task and find the best model you can.
Atrial Fibrillation
This is a physionet dataset of two-channel ECG recordings has been created from data used in the Computers in Cardiology Challenge 2004, an open competition with the goal of developing automated methods for predicting spontaneous termination of atrial fibrillation (AF).
The raw instances were 5 second segments of atrial fibrillation, containing two ECG signals, each sampled at 128 samples per second. The class labels are: n, s and t.
- class n is described as a non termination atrial fibrillation(that is, it did not terminate for at least one hour after the original recording of the data).
- class s is described as an atrial fibrillation that self terminates at least one minute after the recording process.
- class t is described as terminating immediately, that is within one second of the recording ending. PhysioNet Reference:
In [39]:
# Import the dataset
X_train, y_train = load_classification("AtrialFibrillation", split="train")
X_test, y_test = load_classification("AtrialFibrillation", split="test")
X_train = X_train.astype(np.float32)
X_test = X_test.astype(np.float32)
X_train.shape, y_train.shape, X_test.shape, y_test.shape
Out[39]:
((15, 2, 640), (15,), (15, 2, 640), (15,))
In [40]:
# Encode the labels
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
y_train = le.fit_transform(y_train)
y_test = le.transform(y_test)
LABELS = le.classes_
le.classes_
Out[40]:
array(['n', 's', 't'], dtype='<U1')
In [41]:
i = 1
fig, axs = plt.subplots(1, 2, figsize=(10, 5))
axs[0].plot(X_train[i, 0].ravel())
axs[0].set_title("Channel #0")
axs[1].plot(X_train[i, 1].ravel())
axs[1].set_title("Channel #1")
plt.suptitle('Class: {}'.format(LABELS[y_train[i]]), fontsize=16)
plt.show()
In [ ]:
In [ ]:
In [ ]: