In [39]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
import torch.nn as nn
import torch
from torch import nn
import torch.nn.functional as F
from skorch import NeuralNetClassifier
from skorch.callbacks import EpochScoring
from skorch.dataset import ValidSplit
from sklearn.metrics import accuracy_score
from skorch.callbacks import LRScheduler, EarlyStopping, Checkpoint
DEVICE = "mps" if torch.backends.mps.is_available() else "cuda" if torch.cuda.is_available() else "cpu"
Images¶
In [40]:
# Walid Al-Dhabyani, Mohammed Gomaa, et al., "Dataset of breast ultrasound images," Data in Brief, vol. 28, pp. 104863, 2020.
files = np.load("Data/breastmnist.npz")
train_images = files["train_images"]
train_labels = files["train_labels"]
validation_images = files["val_images"]
validation_labels = files["val_labels"]
test_images = files["test_images"]
test_labels = files["test_labels"]
train_images.shape, train_labels.shape, validation_images.shape, validation_labels.shape, test_images.shape, test_labels.shape
Out[40]:
((546, 28, 28), (546, 1), (78, 28, 28), (78, 1), (156, 28, 28), (156, 1))
In [41]:
np.unique(train_labels)
Out[41]:
array([0, 1], dtype=uint8)
In [42]:
i = 0
plt.imshow(train_images[i], interpolation="bicubic", cmap="gray")
plt.title(train_labels[i])
Out[42]:
Text(0.5, 1.0, '[1]')
In [43]:
train_images = train_images.astype(np.float32)[:, None] / 255.0
validation_images = validation_images.astype(np.float32)[:, None] / 255.0
test_images = test_images.astype(np.float32)[:, None] / 255.0
train_labels = train_labels.ravel()
validation_labels = validation_labels.ravel()
test_labels = test_labels.ravel()
train_images.shape, train_labels.shape, validation_images.shape, validation_labels.shape, test_images.shape, test_labels.shape
Out[43]:
((546, 1, 28, 28), (546,), (78, 1, 28, 28), (78,), (156, 1, 28, 28), (156,))
In [44]:
# train a dummy classifier
from sklearn.dummy import DummyClassifier
dummy_clf = DummyClassifier(strategy="most_frequent")
dummy_clf.fit(train_images, train_labels)
dummy_clf.score(test_images, test_labels)
Out[44]:
0.7307692307692307
2D Convolutional Autoencoder¶
In [122]:
# output shape formula for convolutional layers: (W - K + 2P) / S + 1
class Encoder(nn.Module):
def __init__(self, num_input_channels: int, base_channel_size: int, latent_dim: int, act_fn: object = nn.GELU):
"""Encoder.
Args:
num_input_channels : Number of input channels of the image. For CIFAR, this parameter is 3
base_channel_size : Number of channels we use in the first convolutional layers. Deeper layers might use a duplicate of it.
latent_dim : Dimensionality of latent representation z
act_fn : Activation function used throughout the encoder network
"""
super().__init__()
c_hid = base_channel_size
self.net = nn.Sequential(
nn.Conv2d(num_input_channels, c_hid, kernel_size=3, padding=1, stride=2), # 28x28 => 14x14
act_fn(),
nn.Conv2d(c_hid, c_hid, kernel_size=3, padding=1),
act_fn(),
nn.Conv2d(c_hid, 2 * c_hid, kernel_size=3, padding=1, stride=2), # 14x14 => 7x7
act_fn(),
nn.Conv2d(2 * c_hid, 2 * c_hid, kernel_size=3, padding=1),
act_fn(),
nn.Conv2d(2 * c_hid, 2 * c_hid, kernel_size=3, padding=1, stride=2), # 7x7 => 4x4
act_fn(), # shape is now (batch_size, 2 * c_hid, 4, 4)
nn.Flatten(), # shape is now (batch_size, 2 * 16 * c_hid)
nn.Linear(2 * 16 * c_hid, latent_dim),
)
def forward(self, x):
return self.net(x)
# output shape formula for transposed convolutional layers: (W - 1) * S - 2P + K + output_padding
class Decoder(nn.Module):
def __init__(self, num_input_channels: int, base_channel_size: int, latent_dim: int, act_fn: object = nn.GELU):
"""Decoder.
Args:
num_input_channels : Number of channels of the image to reconstruct. For CIFAR, this parameter is 3
base_channel_size : Number of channels we use in the last convolutional layers. Early layers might use a duplicate of it.
latent_dim : Dimensionality of latent representation z
act_fn : Activation function used throughout the decoder network
"""
super().__init__()
c_hid = base_channel_size
self.linear = nn.Sequential(nn.Linear(latent_dim, 2 * 16 * c_hid), act_fn())
self.net = nn.Sequential(
nn.ConvTranspose2d(
2 * c_hid, 2 * c_hid, kernel_size=3, output_padding=0, padding=1, stride=2
), # 4x4 => 7x7
act_fn(),
nn.Conv2d(2 * c_hid, 2 * c_hid, kernel_size=3, padding=1),
act_fn(),
nn.ConvTranspose2d(2 * c_hid, c_hid, kernel_size=3, output_padding=1, padding=1, stride=2), # 7x7 => 14x14
act_fn(),
nn.Conv2d(c_hid, c_hid, kernel_size=3, padding=1),
act_fn(),
nn.ConvTranspose2d(
c_hid, num_input_channels, kernel_size=3, output_padding=1, padding=1, stride=2
), # 14x14 => 28x28
nn.Sigmoid(), # The input images is scaled between 0 and 1, hence the output has to be bounded as well
)
def forward(self, x):
x = self.linear(x)
x = x.reshape(x.shape[0], -1, 4, 4)
x = self.net(x)
return x
class Autoencoder(nn.Module):
def __init__(self, num_input_channels: int, base_channel_size: int, latent_dim: int, act_fn: object = nn.GELU):
"""Autoencoder.
Args:
num_input_channels : Number of channels of the image to reconstruct. For CIFAR, this parameter is 3
base_channel_size : Number of channels we use in the convolutional layers. Deeper layers might use a duplicate of it.
latent_dim : Dimensionality of latent representation z
act_fn : Activation function used throughout the autoencoder network
"""
super().__init__()
self.encoder = Encoder(num_input_channels, base_channel_size, latent_dim, act_fn)
self.decoder = Decoder(num_input_channels, base_channel_size, latent_dim, act_fn)
def forward(self, x):
z = self.encoder(x)
x_rec = self.decoder(z)
return x_rec
In [123]:
validation_images.shape, validation_labels.shape
Out[123]:
((78, 1, 28, 28), (78,))
In [124]:
from skorch.helper import predefined_split
from torch.utils.data import TensorDataset
valid_ds = TensorDataset(torch.Tensor(validation_images), torch.Tensor(validation_images))
In [125]:
# wrap the autoencoder in a skorch NeuralNet
from skorch.net import NeuralNet
In [138]:
net = NeuralNet(
Autoencoder,
module__num_input_channels=train_images.shape[1],
module__base_channel_size=256,
module__latent_dim=16,
criterion=nn.MSELoss,
optimizer=torch.optim.Adam,
max_epochs=50,
batch_size=32,
lr=1e-4,
device=DEVICE,
train_split=predefined_split(valid_ds)
)
net
Out[138]:
<class 'skorch.net.NeuralNet'>[uninitialized]( module=<class '__main__.Autoencoder'>, module__base_channel_size=256, module__latent_dim=16, module__num_input_channels=1, )In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
<class 'skorch.net.NeuralNet'>[uninitialized]( module=<class '__main__.Autoencoder'>, module__base_channel_size=256, module__latent_dim=16, module__num_input_channels=1, )
In [139]:
train_images.shape
Out[139]:
(546, 1, 28, 28)
In [140]:
net.fit(train_images, train_images)
epoch train_loss valid_loss dur
------- ------------ ------------ ------
1 0.0698 0.0399 2.6494
2 0.0320 0.0298 1.9264
3 0.0269 0.0232 1.8867
4 0.0192 0.0158 1.8862
5 0.0145 0.0133 1.8909
6 0.0124 0.0119 1.8957
7 0.0111 0.0107 1.9148
8 0.0102 0.0099 1.8865
9 0.0093 0.0090 1.8993
10 0.0087 0.0085 1.8849
11 0.0083 0.0082 1.8974
12 0.0078 0.0078 1.9028
13 0.0075 0.0076 1.9072
14 0.0072 0.0074 1.8837
15 0.0070 0.0073 1.9040
16 0.0068 0.0072 1.8826
17 0.0067 0.0070 1.9099
18 0.0065 0.0069 1.8780
19 0.0063 0.0068 1.9219
20 0.0061 0.0066 1.9012
21 0.0059 0.0065 1.9049
22 0.0058 0.0063 1.8989
23 0.0057 0.0063 1.9172
24 0.0056 0.0064 1.9496
25 0.0057 0.0062 1.8971
26 0.0056 0.0062 1.8874
27 0.0054 0.0059 1.8884
28 0.0052 0.0058 1.9067
29 0.0051 0.0059 1.8780
30 0.0050 0.0058 1.9063
31 0.0049 0.0059 1.9011
32 0.0049 0.0057 1.8792
33 0.0048 0.0057 1.9076
34 0.0047 0.0055 1.8844
35 0.0046 0.0055 1.8861
36 0.0046 0.0057 1.9029
37 0.0048 0.0062 1.8817
38 0.0054 0.0057 1.9154
39 0.0049 0.0054 1.9414
40 0.0044 0.0053 1.9124
41 0.0043 0.0054 1.8836
42 0.0042 0.0053 1.9319
43 0.0041 0.0055 1.8945
44 0.0041 0.0058 1.8959
45 0.0043 0.0058 1.8863
46 0.0045 0.0053 1.8850
47 0.0044 0.0065 1.8920
48 0.0045 0.0053 1.8931
49 0.0039 0.0051 1.9193
50 0.0037 0.0051 1.8906
Out[140]:
<class 'skorch.net.NeuralNet'>[initialized](
module_=Autoencoder(
(encoder): Encoder(
(net): Sequential(
(0): Conv2d(1, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(1): GELU(approximate='none')
(2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(3): GELU(approximate='none')
(4): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(5): GELU(approximate='none')
(6): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(7): GELU(approximate='none')
(8): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(9): GELU(approximate='none')
(10): Flatten(start_dim=1, end_dim=-1)
(11): Linear(in_features=8192, out_features=16, bias=True)
)
)
(decoder): Decoder(
(linear): Sequential(
(0): Linear(in_features=16, out_features=8192, bias=True)
(1): GELU(approximate='none')
)
(net): Sequential(
(0): ConvTranspose2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(1): GELU(approximate='none')
(2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(3): GELU(approximate='none')
(4): ConvTranspose2d(512, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), output_padding=(1, 1))
(5): GELU(approximate='none')
(6): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(7): GELU(approximate='none')
(8): ConvTranspose2d(256, 1, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), output_padding=(1, 1))
(9): Sigmoid()
)
)
),
)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
<class 'skorch.net.NeuralNet'>[initialized](
module_=Autoencoder(
(encoder): Encoder(
(net): Sequential(
(0): Conv2d(1, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(1): GELU(approximate='none')
(2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(3): GELU(approximate='none')
(4): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(5): GELU(approximate='none')
(6): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(7): GELU(approximate='none')
(8): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(9): GELU(approximate='none')
(10): Flatten(start_dim=1, end_dim=-1)
(11): Linear(in_features=8192, out_features=16, bias=True)
)
)
(decoder): Decoder(
(linear): Sequential(
(0): Linear(in_features=16, out_features=8192, bias=True)
(1): GELU(approximate='none')
)
(net): Sequential(
(0): ConvTranspose2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(1): GELU(approximate='none')
(2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(3): GELU(approximate='none')
(4): ConvTranspose2d(512, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), output_padding=(1, 1))
(5): GELU(approximate='none')
(6): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(7): GELU(approximate='none')
(8): ConvTranspose2d(256, 1, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), output_padding=(1, 1))
(9): Sigmoid()
)
)
),
)In [141]:
history = net.history
plt.plot(history[:, 'train_loss'], label='train_loss')
plt.plot(history[:, 'valid_loss'], label='valid_loss')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.show()
In [142]:
test_images_recon = net.predict(test_images)
test_images_recon.shape
Out[142]:
(156, 1, 28, 28)
In [144]:
i = 0
fig, axs = plt.subplots(1, 2, figsize=(10, 5))
axs[0].imshow(test_images[i, 0], interpolation="bicubic", cmap="gray")
axs[0].set_title("Original")
axs[1].imshow(test_images_recon[i, 0], interpolation="bicubic", cmap="gray")
axs[1].set_title("Reconstructed")
Out[144]:
Text(0.5, 1.0, 'Reconstructed')
Downstream Classification¶
In [145]:
encoder = net.module_.encoder
encoder.eval()
Out[145]:
Encoder(
(net): Sequential(
(0): Conv2d(1, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(1): GELU(approximate='none')
(2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(3): GELU(approximate='none')
(4): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(5): GELU(approximate='none')
(6): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(7): GELU(approximate='none')
(8): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1))
(9): GELU(approximate='none')
(10): Flatten(start_dim=1, end_dim=-1)
(11): Linear(in_features=8192, out_features=16, bias=True)
)
) In [147]:
Z_train = encoder(torch.Tensor(train_images).to(DEVICE)).cpu().detach().numpy()
Z_test = encoder(torch.Tensor(test_images).to(DEVICE)).cpu().detach().numpy()
Z_train.shape, Z_test.shape
Out[147]:
((546, 16), (156, 16))
In [149]:
# Use PCA to visualize the latent space in 2D
from sklearn.decomposition import PCA
pca = PCA(n_components=2)
Z_train_2d = pca.fit_transform(Z_train)
Z_test_2d = pca.transform(Z_test)
In [151]:
plt.figure(figsize=(8, 6))
scatter = plt.scatter(Z_train_2d[:, 0], Z_train_2d[:, 1], c=train_labels, alpha=0.7)
plt.legend(*scatter.legend_elements(), title="Classes")
plt.xlabel("Principal Component 1")
plt.ylabel("Principal Component 2")
plt.title("PCA of Latent Space")
plt.show()
In [148]:
# Training a simple classifier on the latent representation
from sklearn.linear_model import LogisticRegression
clf = LogisticRegression(max_iter=1000)
clf.fit(Z_train, train_labels)
clf.score(Z_test, test_labels)
Out[148]:
0.7948717948717948
Time Series¶
Dataset¶
ECG200
In [322]:
# Import the dataset
from aeon.datasets import load_classification
X_train, y_train = load_classification("ECG200", split="train")
X_test, y_test = load_classification("ECG200", split="test")
X_train = X_train.astype(np.float32)
X_test = X_test.astype(np.float32)
X_train.shape, y_train.shape, X_test.shape, y_test.shape
Out[322]:
((100, 1, 96), (100,), (100, 1, 96), (100,))
In [323]:
# Encode the labels
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
y_train = le.fit_transform(y_train)
y_test = le.transform(y_test)
LABELS = le.classes_
le.classes_
Out[323]:
array(['-1', '1'], dtype='<U2')
In [324]:
i = 0
plt.plot(X_train[i].ravel())
plt.title(f"Class: {LABELS[y_train[i]]}")
plt.show()
In [325]:
from sklearn.dummy import DummyClassifier
dummy_clf = DummyClassifier(strategy="most_frequent")
dummy_clf.fit(X_train, y_train)
dummy_clf.score(X_test, y_test)
Out[325]:
0.64
1D Convolutional Autoencoder¶
In [335]:
class Encoder1D(nn.Module):
def __init__(self, num_input_channels: int, base_channel_size: int, latent_dim: int, act_fn: object = nn.GELU):
"""Encoder.
Args:
num_input_channels : Number of input channels of the time series. For univariate time series, this parameter is 1
base_channel_size : Number of channels we use in the first convolutional layers. Deeper layers might use a duplicate of it.
latent_dim : Dimensionality of latent representation z
act_fn : Activation function used throughout the encoder network
"""
super().__init__()
c_hid = base_channel_size
self.net = nn.Sequential(
nn.Conv1d(num_input_channels, c_hid, kernel_size=3, padding=1, stride=2), # 96 => 48
act_fn(),
nn.Conv1d(c_hid, c_hid, kernel_size=3, padding=1),
act_fn(),
nn.Conv1d(c_hid, 2 * c_hid, kernel_size=3, padding=1, stride=2), # 48 => 24
act_fn(),
nn.Conv1d(2 * c_hid, 2 * c_hid, kernel_size=3, padding=1),
act_fn(),
nn.Conv1d(2 * c_hid, 2 * c_hid, kernel_size=3, padding=1, stride=2), # 24 => 12
act_fn(), # shape is now (batch_size, 2 * c_hid, 12)
nn.Flatten(), # shape is now (batch_size, 2 * 12 * c_hid)
nn.Linear(2 * 12 * c_hid, latent_dim),
)
def forward(self, x):
return self.net(x)
class Decoder1D(nn.Module):
def __init__(self, num_input_channels: int, base_channel_size: int, latent_dim: int, act_fn: object = nn.GELU):
"""Decoder.
Args:
num_input_channels : Number of channels of the time series to reconstruct. For univariate time series, this parameter is 1
base_channel_size : Number of channels we use in the last convolutional layers. Early layers might use a duplicate of it.
latent_dim : Dimensionality of latent representation z
act_fn : Activation function used throughout the decoder network
"""
super().__init__()
c_hid = base_channel_size
self.linear = nn.Sequential(nn.Linear(latent_dim, 2 * 12 * c_hid), act_fn())
self.net = nn.Sequential(
nn.ConvTranspose1d(
2 * c_hid, 2 * c_hid, kernel_size=3, output_padding=1, padding=1, stride=2
), # 12 => 24
act_fn(),
nn.Conv1d(2 * c_hid, 2 * c_hid, kernel_size=3, padding=1),
act_fn(),
nn.ConvTranspose1d(2 * c_hid, c_hid, kernel_size=3, output_padding=1, padding=1, stride=2), # 24 => 48
act_fn(),
nn.Conv1d(c_hid, c_hid, kernel_size=3, padding=1),
act_fn(),
nn.ConvTranspose1d(
c_hid, num_input_channels, kernel_size=3, output_padding=1, padding=1, stride=2
), # 48 => 96
)
def forward(self, x):
x = self.linear(x)
x = x.reshape(x.shape[0], -1, 12)
x = self.net(x)
return x
class Autoencoder1D(nn.Module):
def __init__(self, num_input_channels: int, base_channel_size: int, latent_dim: int, act_fn: object = nn.GELU):
"""Autoencoder.
Args:
num_input_channels : Number of channels of the time series to reconstruct. For univariate time series, this parameter is 1
base_channel_size : Number of channels we use in the convolutional layers. Deeper layers might use a duplicate of it.
latent_dim : Dimensionality of latent representation z
act_fn : Activation function used throughout the autoencoder network
"""
super().__init__()
self.encoder = Encoder1D(num_input_channels, base_channel_size, latent_dim, act_fn)
self.decoder = Decoder1D(num_input_channels, base_channel_size, latent_dim, act_fn)
def forward(self, x):
z = self.encoder(x)
x_rec = self.decoder(z)
return x_rec
In [176]:
net = NeuralNet(
Autoencoder1D,
module__num_input_channels=X_train.shape[1],
module__base_channel_size=128,
module__latent_dim=2,
criterion=nn.MSELoss,
optimizer=torch.optim.Adam,
max_epochs=50,
batch_size=32,
lr=1e-4,
device=DEVICE,
train_split=predefined_split(TensorDataset(torch.Tensor(X_test), torch.Tensor(X_test)))
)
In [177]:
net.fit(X_train, X_train)
epoch train_loss valid_loss dur
------- ------------ ------------ ------
1 1.0997 1.0734 0.1275
2 1.0628 1.0379 0.0740
3 1.0257 0.9988 0.0803
4 0.9823 0.9488 0.0864
5 0.9237 0.8773 0.0925
6 0.8366 0.7734 0.0719
7 0.7122 0.6630 0.0758
8 0.5857 0.5413 0.0907
9 0.4475 0.4623 0.0939
10 0.3837 0.4793 0.0827
11 0.3902 0.4461 0.0958
12 0.3655 0.4287 0.0989
13 0.3530 0.4101 0.0727
14 0.3354 0.3825 0.0758
15 0.3098 0.3376 0.0809
16 0.2714 0.2803 0.0849
17 0.2256 0.2405 0.0712
18 0.2065 0.2418 0.0746
19 0.1992 0.2226 0.0814
20 0.1807 0.2169 0.0795
21 0.1743 0.2110 0.0715
22 0.1673 0.2017 0.0736
23 0.1597 0.1958 0.0805
24 0.1559 0.1913 0.0882
25 0.1521 0.1884 0.0741
26 0.1493 0.1869 0.0771
27 0.1472 0.1848 0.0930
28 0.1449 0.1831 0.0875
29 0.1432 0.1817 0.0720
30 0.1413 0.1805 0.0761
31 0.1394 0.1794 0.0883
32 0.1377 0.1777 0.0765
33 0.1360 0.1760 0.0721
34 0.1344 0.1747 0.0759
35 0.1328 0.1735 0.0823
36 0.1312 0.1722 0.0974
37 0.1296 0.1709 0.0865
38 0.1280 0.1698 0.0718
39 0.1264 0.1688 0.0740
40 0.1247 0.1678 0.0799
41 0.1231 0.1668 0.0904
42 0.1214 0.1660 0.0696
43 0.1198 0.1654 0.0723
44 0.1181 0.1649 0.0868
45 0.1165 0.1644 0.0795
46 0.1149 0.1641 0.0731
47 0.1132 0.1639 0.0751
48 0.1117 0.1638 0.0869
49 0.1101 0.1637 0.0820
50 0.1087 0.1636 0.0708
Out[177]:
<class 'skorch.net.NeuralNet'>[initialized](
module_=Autoencoder1D(
(encoder): Encoder1D(
(net): Sequential(
(0): Conv1d(1, 128, kernel_size=(3,), stride=(2,), padding=(1,))
(1): GELU(approximate='none')
(2): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))
(3): GELU(approximate='none')
(4): Conv1d(128, 256, kernel_size=(3,), stride=(2,), padding=(1,))
(5): GELU(approximate='none')
(6): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))
(7): GELU(approximate='none')
(8): Conv1d(256, 256, kernel_size=(3,), stride=(2,), padding=(1,))
(9): GELU(approximate='none')
(10): Flatten(start_dim=1, end_dim=-1)
(11): Linear(in_features=3072, out_features=2, bias=True)
)
)
(decoder): Decoder1D(
(linear): Sequential(
(0): Linear(in_features=2, out_features=3072, bias=True)
(1): GELU(approximate='none')
)
(net): Sequential(
(0): ConvTranspose1d(256, 256, kernel_size=(3,), stride=(2,), padding=(1,), output_padding=(1,))
(1): GELU(approximate='none')
(2): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))
(3): GELU(approximate='none')
(4): ConvTranspose1d(256, 128, kernel_size=(3,), stride=(2,), padding=(1,), output_padding=(1,))
(5): GELU(approximate='none')
(6): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))
(7): GELU(approximate='none')
(8): ConvTranspose1d(128, 1, kernel_size=(3,), stride=(2,), padding=(1,), output_padding=(1,))
)
)
),
)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
<class 'skorch.net.NeuralNet'>[initialized](
module_=Autoencoder1D(
(encoder): Encoder1D(
(net): Sequential(
(0): Conv1d(1, 128, kernel_size=(3,), stride=(2,), padding=(1,))
(1): GELU(approximate='none')
(2): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))
(3): GELU(approximate='none')
(4): Conv1d(128, 256, kernel_size=(3,), stride=(2,), padding=(1,))
(5): GELU(approximate='none')
(6): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))
(7): GELU(approximate='none')
(8): Conv1d(256, 256, kernel_size=(3,), stride=(2,), padding=(1,))
(9): GELU(approximate='none')
(10): Flatten(start_dim=1, end_dim=-1)
(11): Linear(in_features=3072, out_features=2, bias=True)
)
)
(decoder): Decoder1D(
(linear): Sequential(
(0): Linear(in_features=2, out_features=3072, bias=True)
(1): GELU(approximate='none')
)
(net): Sequential(
(0): ConvTranspose1d(256, 256, kernel_size=(3,), stride=(2,), padding=(1,), output_padding=(1,))
(1): GELU(approximate='none')
(2): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))
(3): GELU(approximate='none')
(4): ConvTranspose1d(256, 128, kernel_size=(3,), stride=(2,), padding=(1,), output_padding=(1,))
(5): GELU(approximate='none')
(6): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))
(7): GELU(approximate='none')
(8): ConvTranspose1d(128, 1, kernel_size=(3,), stride=(2,), padding=(1,), output_padding=(1,))
)
)
),
)In [178]:
X_train_recon = net.predict(X_train)
X_test_recon = net.predict(X_test)
In [182]:
i = 29
plt.figure(figsize=(8, 6))
plt.plot(X_train[i].ravel(), label="Original")
plt.plot(X_train_recon[i].ravel(), label="Reconstructed")
plt.legend()
plt.title(f"Class: {LABELS[y_train[i]]}")
plt.show()
In [183]:
encoder = net.module_.encoder
encoder
Out[183]:
Encoder1D(
(net): Sequential(
(0): Conv1d(1, 128, kernel_size=(3,), stride=(2,), padding=(1,))
(1): GELU(approximate='none')
(2): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))
(3): GELU(approximate='none')
(4): Conv1d(128, 256, kernel_size=(3,), stride=(2,), padding=(1,))
(5): GELU(approximate='none')
(6): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))
(7): GELU(approximate='none')
(8): Conv1d(256, 256, kernel_size=(3,), stride=(2,), padding=(1,))
(9): GELU(approximate='none')
(10): Flatten(start_dim=1, end_dim=-1)
(11): Linear(in_features=3072, out_features=2, bias=True)
)
) In [185]:
Z_train = encoder(torch.Tensor(X_train).to(DEVICE)).cpu().detach().numpy()
Z_test = encoder(torch.Tensor(X_test).to(DEVICE)).cpu().detach().numpy()
Z_train.shape, Z_test.shape
Out[185]:
((100, 2), (100, 2))
In [194]:
# visualize the latent space
import seaborn as sns
sns.scatterplot(x=Z_train[:, 0], y=Z_train[:, 1], hue=le.classes_[y_train], alpha=0.7)
plt.xlabel("Latent Dimension 1")
plt.ylabel("Latent Dimension 2")
plt.title("Latent Space Visualization")
plt.show()
In [195]:
# training a downstream classifier on the latent representation
from sklearn.linear_model import LogisticRegression
clf = LogisticRegression()
clf.fit(Z_train, y_train)
clf.score(Z_test, y_test)
Out[195]:
0.75
Variational Autoencoder¶
In [405]:
import torch
import torch.nn as nn
import torch.nn.functional as F
class VariationalEncoder1D(nn.Module):
def __init__(
self,
num_input_channels: int,
base_channel_size: int,
latent_dim: int,
act_fn: object = nn.GELU,
):
super().__init__()
c_hid = base_channel_size
self.net = nn.Sequential(
nn.Conv1d(num_input_channels, c_hid, kernel_size=3, padding=1, stride=2), # 96 => 48
act_fn(),
nn.Conv1d(c_hid, c_hid, kernel_size=3, padding=1),
act_fn(),
nn.Conv1d(c_hid, 2 * c_hid, kernel_size=3, padding=1, stride=2), # 48 => 24
act_fn(),
nn.Conv1d(2 * c_hid, 2 * c_hid, kernel_size=3, padding=1),
act_fn(),
nn.Conv1d(2 * c_hid, 2 * c_hid, kernel_size=3, padding=1, stride=2), # 24 => 12
act_fn(),
nn.Flatten(),
)
self.fc_mu = nn.Linear(2 * 12 * c_hid, latent_dim)
self.fc_logvar = nn.Linear(2 * 12 * c_hid, latent_dim)
def forward(self, x):
h = self.net(x)
mu = self.fc_mu(h)
logvar = self.fc_logvar(h)
return mu, logvar
class VAE1D(nn.Module):
def __init__(
self,
num_input_channels: int,
base_channel_size: int,
latent_dim: int,
act_fn: object = nn.GELU,
):
super().__init__()
self.encoder = VariationalEncoder1D(
num_input_channels=num_input_channels,
base_channel_size=base_channel_size,
latent_dim=latent_dim,
act_fn=act_fn,
)
self.decoder = Decoder1D(
num_input_channels=num_input_channels,
base_channel_size=base_channel_size,
latent_dim=latent_dim,
act_fn=act_fn,
)
def reparameterize(self, mu, logvar):
std = torch.exp(0.5 * logvar)
eps = torch.randn_like(std)
return mu + eps * std
def forward(self, x):
mu, logvar = self.encoder(x)
z = self.reparameterize(mu, logvar)
x_rec = self.decoder(z)
return x_rec, mu, logvar, z
class VAELoss(nn.Module):
def __init__(self, beta: float = 1.0, reduction: str = "mean"):
super().__init__()
self.beta = beta
self.reduction = reduction
def forward(self, output, target):
x_rec, mu, logvar, z = output
rec_loss = F.mse_loss(x_rec, target, reduction=self.reduction)
kl_loss = -0.5 * torch.sum(
1 + logvar - mu.pow(2) - logvar.exp(),
dim=1,
)
if self.reduction == "mean":
kl_loss = kl_loss.mean()
elif self.reduction == "sum":
kl_loss = kl_loss.sum()
return rec_loss + self.beta * kl_loss
In [ ]:
from skorch import NeuralNet
from skorch.helper import predefined_split
import torch
net = NeuralNet(
VAE1D,
module__num_input_channels=X_train.shape[1],
module__base_channel_size=128,
module__latent_dim=2,
criterion=VAELoss,
criterion__beta=0.001,
optimizer=torch.optim.Adam,
max_epochs=100,
batch_size=32,
lr=1e-3,
device=DEVICE,
)
In [414]:
net.fit(X_train, X_train)
epoch train_loss valid_loss dur
------- ------------ ------------ ------
1 1.0199 0.8457 0.0679
2 0.7492 0.5270 0.0517
3 0.4887 0.5744 0.0536
4 0.5361 0.4734 0.0551
5 0.4440 0.4460 0.0556
6 0.4064 0.4836 0.0603
7 0.3878 0.4572 0.0716
8 0.3613 0.4316 0.0504
9 0.3583 0.4053 0.0516
10 0.3540 0.4094 0.0519
11 0.3451 0.3968 0.0581
12 0.3210 0.3745 0.0493
13 0.2733 0.2747 0.0519
14 0.2283 0.2197 0.0545
15 0.1892 0.1643 0.0617
16 0.1629 0.1510 0.0619
17 0.1555 0.1469 0.0607
18 0.1402 0.1452 0.0595
19 0.1326 0.1349 0.0624
20 0.1308 0.1355 0.0687
21 0.1272 0.1308 0.0600
22 0.1210 0.1326 0.0486
23 0.1170 0.1194 0.0509
24 0.1122 0.1196 0.0539
25 0.1086 0.1133 0.0562
26 0.1047 0.1139 0.0655
27 0.1031 0.1139 0.0607
28 0.0993 0.1153 0.0581
29 0.0989 0.1125 0.1212
30 0.0956 0.1126 0.0541
31 0.0949 0.1133 0.0551
32 0.0937 0.1151 0.0528
33 0.0918 0.1188 0.0689
34 0.0914 0.1171 0.0611
35 0.0913 0.1148 0.0485
36 0.0914 0.1188 0.0507
37 0.0898 0.1187 0.0523
38 0.0903 0.1140 0.0584
39 0.0889 0.1149 0.0774
40 0.0853 0.1162 0.0627
41 0.0861 0.1145 0.0544
42 0.0853 0.1161 0.0557
43 0.0830 0.1151 0.0563
44 0.0828 0.1146 0.0587
45 0.0819 0.1160 0.0665
46 0.0811 0.1176 0.0579
47 0.0804 0.1155 0.0508
48 0.0786 0.1168 0.0529
49 0.0783 0.1176 0.0553
50 0.0774 0.1164 0.0630
51 0.0770 0.1157 0.0567
52 0.0753 0.1160 0.0509
53 0.0746 0.1197 0.0526
54 0.0756 0.1162 0.0550
55 0.0733 0.1176 0.0572
56 0.0735 0.1189 0.0625
57 0.0734 0.1249 0.0699
58 0.0779 0.1166 0.0597
59 0.0723 0.1212 0.0505
60 0.0729 0.1192 0.0530
61 0.0734 0.1175 0.0545
62 0.0726 0.1225 0.0576
63 0.0710 0.1240 0.0702
64 0.0703 0.1178 0.0510
65 0.0712 0.1200 0.0519
66 0.0719 0.1232 0.0539
67 0.0748 0.1298 0.0553
68 0.0719 0.1200 0.0611
69 0.0707 0.1213 0.0708
70 0.0684 0.1193 0.0726
71 0.0685 0.1198 0.0571
72 0.0670 0.1225 0.0482
73 0.0661 0.1232 0.0514
74 0.0678 0.1219 0.0544
75 0.0654 0.1239 0.0582
76 0.0660 0.1195 0.0637
77 0.0655 0.1261 0.0737
78 0.0673 0.1190 0.0657
79 0.0652 0.1193 0.0558
80 0.0648 0.1203 0.0549
81 0.0645 0.1199 0.0622
82 0.0638 0.1216 0.0675
83 0.0620 0.1186 0.0607
84 0.0613 0.1203 0.0533
85 0.0593 0.1204 0.0535
86 0.0602 0.1180 0.0597
87 0.0591 0.1209 0.0772
88 0.0589 0.1230 0.0517
89 0.0587 0.1219 0.0522
90 0.0594 0.1210 0.0526
91 0.0587 0.1219 0.0552
92 0.0587 0.1260 0.0594
93 0.0610 0.1268 0.0680
94 0.0615 0.1187 0.0578
95 0.0605 0.1209 0.0515
96 0.0610 0.1249 0.0542
97 0.0623 0.1255 0.0557
98 0.0615 0.1271 0.0612
99 0.0610 0.1231 0.0725
100 0.0588 0.1245 0.0522
101 0.0572 0.1207 0.0514
102 0.0575 0.1240 0.0532
103 0.0572 0.1239 0.0550
104 0.0578 0.1255 0.0597
105 0.0581 0.1202 0.1015
106 0.0563 0.1193 0.0570
107 0.0566 0.1202 0.0500
108 0.0554 0.1186 0.0524
109 0.0550 0.1218 0.0546
110 0.0564 0.1217 0.0634
111 0.0558 0.1217 0.0597
112 0.0557 0.1212 0.0503
113 0.0557 0.1217 0.0531
114 0.0564 0.1261 0.0559
115 0.0573 0.1233 0.0677
116 0.0552 0.1271 0.0664
117 0.0557 0.1229 0.0503
118 0.0541 0.1252 0.0537
119 0.0552 0.1197 0.0571
120 0.0566 0.1320 0.0694
121 0.0572 0.1181 0.0720
122 0.0558 0.1209 0.0596
123 0.0550 0.1221 0.0500
124 0.0558 0.1201 0.0532
125 0.0551 0.1266 0.0528
126 0.0556 0.1202 0.0569
127 0.0548 0.1263 0.0692
128 0.0549 0.1193 0.0513
129 0.0555 0.1207 0.0512
130 0.0575 0.1249 0.0522
131 0.0549 0.1273 0.0565
132 0.0553 0.1162 0.0682
133 0.0545 0.1207 0.0720
134 0.0561 0.1186 0.0579
135 0.0539 0.1207 0.0501
136 0.0563 0.1198 0.0510
137 0.0547 0.1201 0.0527
138 0.0528 0.1167 0.0567
139 0.0507 0.1189 0.0634
140 0.0498 0.1185 0.0718
141 0.0495 0.1196 0.0557
142 0.0494 0.1199 0.0514
143 0.0497 0.1187 0.0591
144 0.0500 0.1222 0.0573
145 0.0487 0.1193 0.0631
146 0.0490 0.1185 0.0702
147 0.0483 0.1217 0.0574
148 0.0478 0.1246 0.0500
149 0.0484 0.1203 0.0515
150 0.0475 0.1186 0.0535
151 0.0477 0.1238 0.0563
152 0.0485 0.1185 0.0606
153 0.0499 0.1244 0.0620
154 0.0510 0.1308 0.0499
155 0.0536 0.1313 0.0506
156 0.0554 0.1281 0.0528
157 0.0544 0.1176 0.0647
158 0.0550 0.1215 0.0637
159 0.0541 0.1285 0.0496
160 0.0571 0.1237 0.0512
161 0.0556 0.1205 0.0524
162 0.0528 0.1204 0.0544
163 0.0495 0.1200 0.0685
164 0.0490 0.1191 0.0495
165 0.0481 0.1174 0.0514
166 0.0476 0.1201 0.0518
167 0.0470 0.1204 0.0669
168 0.0467 0.1220 0.0614
169 0.0471 0.1215 0.0497
170 0.0467 0.1168 0.0523
171 0.0468 0.1253 0.0528
172 0.0477 0.1261 0.0649
173 0.0482 0.1212 0.0640
174 0.0474 0.1221 0.0493
175 0.0476 0.1201 0.0506
176 0.0485 0.1253 0.0530
177 0.0486 0.1175 0.0544
178 0.0458 0.1223 0.0679
179 0.0443 0.1219 0.0620
180 0.0442 0.1204 0.0481
181 0.0428 0.1215 0.0501
182 0.0435 0.1184 0.0526
183 0.0452 0.1184 0.0560
184 0.0434 0.1161 0.0631
185 0.0438 0.1184 0.0602
186 0.0432 0.1243 0.0486
187 0.0425 0.1171 0.1007
188 0.0436 0.1209 0.0570
189 0.0435 0.1207 0.0513
190 0.0443 0.1176 0.0512
191 0.0433 0.1211 0.0538
192 0.0441 0.1193 0.0613
193 0.0445 0.1252 0.0729
194 0.0436 0.1158 0.0670
195 0.0435 0.1209 0.0498
196 0.0434 0.1144 0.0594
197 0.0426 0.1189 0.0549
198 0.0421 0.1234 0.0584
199 0.0408 0.1167 0.0617
200 0.0410 0.1239 0.0506
201 0.0398 0.1210 0.0513
202 0.0391 0.1209 0.0524
203 0.0396 0.1198 0.0555
204 0.0397 0.1213 0.0610
205 0.0398 0.1174 0.0700
206 0.0396 0.1235 0.0552
207 0.0399 0.1197 0.0495
208 0.0395 0.1270 0.0517
209 0.0406 0.1257 0.0544
210 0.0404 0.1236 0.0579
211 0.0406 0.1246 0.0624
212 0.0417 0.1245 0.0653
213 0.0407 0.1207 0.0493
214 0.0416 0.1255 0.0511
215 0.0411 0.1211 0.0525
216 0.0426 0.1204 0.0548
217 0.0433 0.1310 0.0597
218 0.0435 0.1235 0.0499
219 0.0422 0.1239 0.0580
220 0.0414 0.1194 0.0528
221 0.0402 0.1254 0.0685
222 0.0390 0.1226 0.0502
223 0.0384 0.1203 0.0496
224 0.0380 0.1292 0.0526
225 0.0376 0.1263 0.0546
226 0.0378 0.1221 0.0573
227 0.0373 0.1261 0.0642
228 0.0372 0.1286 0.0649
229 0.0368 0.1252 0.0486
230 0.0373 0.1269 0.0499
231 0.0385 0.1251 0.0507
232 0.0380 0.1305 0.0541
233 0.0378 0.1234 0.0567
234 0.0381 0.1287 0.0637
235 0.0379 0.1342 0.0572
236 0.0377 0.1292 0.0495
237 0.0375 0.1288 0.0508
238 0.0367 0.1275 0.0536
239 0.0361 0.1318 0.0574
240 0.0374 0.1309 0.0705
241 0.0384 0.1246 0.0533
242 0.0376 0.1334 0.0522
243 0.0376 0.1344 0.0665
244 0.0396 0.1282 0.0642
245 0.0416 0.1342 0.0651
246 0.0389 0.1251 0.0498
247 0.0366 0.1294 0.0517
248 0.0355 0.1270 0.1059
249 0.0348 0.1269 0.0665
250 0.0342 0.1289 0.0501
251 0.0343 0.1259 0.0508
252 0.0358 0.1333 0.0533
253 0.0365 0.1358 0.0562
254 0.0362 0.1277 0.0725
255 0.0371 0.1381 0.0538
256 0.0363 0.1286 0.0562
257 0.0352 0.1337 0.0601
258 0.0347 0.1320 0.0579
259 0.0331 0.1335 0.0643
260 0.0349 0.1365 0.0669
261 0.0340 0.1371 0.0498
262 0.0356 0.1349 0.0510
263 0.0353 0.1342 0.0527
264 0.0339 0.1335 0.0544
265 0.0345 0.1313 0.0610
266 0.0344 0.1403 0.0510
267 0.0365 0.1336 0.0512
268 0.0345 0.1335 0.0540
269 0.0340 0.1390 0.0644
270 0.0335 0.1271 0.0677
271 0.0332 0.1383 0.0577
272 0.0325 0.1337 0.0503
273 0.0330 0.1298 0.0518
274 0.0340 0.1391 0.0555
275 0.0339 0.1303 0.0616
276 0.0342 0.1347 0.0728
277 0.0338 0.1344 0.0659
278 0.0347 0.1362 0.0482
279 0.0334 0.1358 0.0501
280 0.0334 0.1334 0.0514
281 0.0344 0.1379 0.0552
282 0.0341 0.1357 0.0586
283 0.0321 0.1369 0.0693
284 0.0317 0.1352 0.0505
285 0.0312 0.1424 0.0507
286 0.0318 0.1397 0.0523
287 0.0314 0.1342 0.0555
288 0.0322 0.1361 0.0645
289 0.0331 0.1373 0.0678
290 0.0341 0.1390 0.0589
291 0.0326 0.1338 0.0549
292 0.0337 0.1404 0.1141
293 0.0325 0.1360 0.0744
294 0.0336 0.1356 0.0613
295 0.0333 0.1435 0.0526
296 0.0343 0.1364 0.0537
297 0.0345 0.1412 0.0550
298 0.0338 0.1304 0.0584
299 0.0326 0.1378 0.0658
300 0.0313 0.1337 0.0601
301 0.0315 0.1379 0.0478
302 0.0317 0.1316 0.0510
303 0.0315 0.1392 0.0542
304 0.0308 0.1361 0.0565
305 0.0316 0.1357 0.0616
306 0.0326 0.1322 0.0518
307 0.0310 0.1387 0.0530
308 0.0318 0.1311 0.0536
309 0.0299 0.1313 0.0605
310 0.0302 0.1349 0.0731
311 0.0296 0.1367 0.0663
312 0.0294 0.1380 0.0494
313 0.0295 0.1380 0.0514
314 0.0294 0.1380 0.0545
315 0.0303 0.1392 0.0572
316 0.0303 0.1374 0.0694
317 0.0298 0.1347 0.0731
318 0.0302 0.1336 0.0522
319 0.0297 0.1456 0.0551
320 0.0290 0.1395 0.0578
321 0.0291 0.1363 0.0600
322 0.0301 0.1299 0.0686
323 0.0294 0.1315 0.0598
324 0.0297 0.1416 0.0502
325 0.0290 0.1458 0.0531
326 0.0295 0.1324 0.0545
327 0.0295 0.1363 0.0590
328 0.0288 0.1386 0.0501
329 0.0290 0.1435 0.0507
330 0.0299 0.1369 0.0531
331 0.0297 0.1368 0.0565
332 0.0306 0.1387 0.0687
333 0.0314 0.1404 0.0490
334 0.0321 0.1437 0.0498
335 0.0333 0.1347 0.0533
336 0.0336 0.1393 0.0554
337 0.0314 0.1317 0.0625
338 0.0302 0.1345 0.0646
339 0.0295 0.1344 0.0500
340 0.0298 0.1373 0.0513
341 0.0293 0.1357 0.0660
342 0.0301 0.1424 0.0607
343 0.0294 0.1413 0.0506
344 0.0289 0.1419 0.0514
345 0.0285 0.1392 0.0539
346 0.0296 0.1411 0.0567
347 0.0289 0.1338 0.0611
348 0.0286 0.1377 0.0703
349 0.0284 0.1419 0.0496
350 0.0288 0.1366 0.0507
351 0.0293 0.1385 0.0529
352 0.0283 0.1413 0.0550
353 0.0274 0.1409 0.0570
354 0.0270 0.1377 0.0653
355 0.0267 0.1399 0.0598
356 0.0266 0.1419 0.0478
357 0.0271 0.1361 0.0506
358 0.0267 0.1383 0.0541
359 0.0272 0.1354 0.0572
360 0.0266 0.1426 0.0604
361 0.0260 0.1392 0.0510
362 0.0258 0.1402 0.0510
363 0.0262 0.1406 0.0528
364 0.0264 0.1413 0.0646
365 0.0265 0.1450 0.0751
366 0.0261 0.1418 0.0628
367 0.0267 0.1417 0.0512
368 0.0271 0.1323 0.0552
369 0.0270 0.1428 0.0557
370 0.0268 0.1323 0.1156
371 0.0265 0.1425 0.0602
372 0.0256 0.1354 0.0522
373 0.0256 0.1439 0.0530
374 0.0259 0.1410 0.0539
375 0.0259 0.1385 0.0580
376 0.0257 0.1365 0.0611
377 0.0267 0.1379 0.0510
378 0.0270 0.1419 0.0512
379 0.0282 0.1421 0.0526
380 0.0283 0.1336 0.0646
381 0.0294 0.1441 0.0636
382 0.0284 0.1382 0.0505
383 0.0273 0.1401 0.0503
384 0.0265 0.1386 0.0533
385 0.0269 0.1410 0.0552
386 0.0266 0.1399 0.0597
387 0.0266 0.1372 0.0677
388 0.0267 0.1447 0.0563
389 0.0259 0.1425 0.0494
390 0.0257 0.1390 0.0519
391 0.0262 0.1403 0.0536
392 0.0258 0.1463 0.0654
393 0.0266 0.1372 0.0729
394 0.0261 0.1431 0.0553
395 0.0261 0.1392 0.0502
396 0.0264 0.1324 0.0512
397 0.0263 0.1423 0.0543
398 0.0259 0.1422 0.0581
399 0.0265 0.1439 0.0745
400 0.0267 0.1375 0.0563
401 0.0265 0.1406 0.0513
402 0.0266 0.1444 0.0518
403 0.0270 0.1372 0.0551
404 0.0265 0.1456 0.0617
405 0.0266 0.1373 0.1096
406 0.0254 0.1442 0.0517
407 0.0253 0.1410 0.0536
408 0.0262 0.1390 0.0558
409 0.0263 0.1416 0.0683
410 0.0258 0.1313 0.0525
411 0.0257 0.1424 0.0537
412 0.0253 0.1376 0.0543
413 0.0255 0.1343 0.0621
414 0.0253 0.1428 0.0581
415 0.0248 0.1390 0.0508
416 0.0244 0.1363 0.0515
417 0.0245 0.1408 0.0544
418 0.0242 0.1395 0.0566
419 0.0240 0.1397 0.0624
420 0.0242 0.1387 0.0702
421 0.0244 0.1391 0.0488
422 0.0253 0.1351 0.0496
423 0.0246 0.1384 0.0531
424 0.0248 0.1429 0.0543
425 0.0267 0.1432 0.0590
426 0.0280 0.1426 0.0583
427 0.0262 0.1353 0.0506
428 0.0250 0.1480 0.0518
429 0.0267 0.1447 0.0570
430 0.0274 0.1431 0.0634
431 0.0272 0.1434 0.0597
432 0.0259 0.1378 0.0482
433 0.0255 0.1467 0.0504
434 0.0254 0.1366 0.0521
435 0.0251 0.1413 0.0556
436 0.0244 0.1393 0.0626
437 0.0243 0.1390 0.0600
438 0.0243 0.1389 0.0478
439 0.0247 0.1384 0.0498
440 0.0243 0.1409 0.0524
441 0.0251 0.1402 0.0603
442 0.0260 0.1502 0.0723
443 0.0259 0.1346 0.0552
444 0.0257 0.1425 0.0502
445 0.0257 0.1405 0.0516
446 0.0260 0.1384 0.0544
447 0.0253 0.1498 0.0574
448 0.0253 0.1403 0.0617
449 0.0247 0.1406 0.0694
450 0.0245 0.1462 0.0594
451 0.0243 0.1345 0.0471
452 0.0243 0.1492 0.1057
453 0.0235 0.1480 0.0504
454 0.0236 0.1370 0.0514
455 0.0228 0.1364 0.0531
456 0.0230 0.1405 0.0557
457 0.0231 0.1447 0.0653
458 0.0234 0.1389 0.0710
459 0.0247 0.1425 0.0727
460 0.0255 0.1375 0.0650
461 0.0260 0.1434 0.0503
462 0.0255 0.1389 0.0509
463 0.0256 0.1404 0.0531
464 0.0238 0.1478 0.0551
465 0.0237 0.1479 0.0593
466 0.0241 0.1311 0.0704
467 0.0239 0.1429 0.0661
468 0.0234 0.1424 0.0486
469 0.0244 0.1382 0.0494
470 0.0249 0.1402 0.0510
471 0.0245 0.1465 0.0554
472 0.0240 0.1398 0.0579
473 0.0244 0.1379 0.0686
474 0.0259 0.1405 0.0657
475 0.0249 0.1440 0.0491
476 0.0252 0.1400 0.0508
477 0.0240 0.1329 0.0534
478 0.0250 0.1470 0.0557
479 0.0255 0.1477 0.0628
480 0.0252 0.1447 0.0730
481 0.0258 0.1466 0.0551
482 0.0242 0.1444 0.0583
483 0.0241 0.1377 0.0511
484 0.0235 0.1407 0.0558
485 0.0226 0.1432 0.0597
486 0.0228 0.1389 0.0695
487 0.0232 0.1459 0.0594
488 0.0240 0.1436 0.0485
489 0.0238 0.1378 0.0504
490 0.0248 0.1435 0.0507
491 0.0240 0.1461 0.0544
492 0.0239 0.1388 0.0575
493 0.0246 0.1422 0.0661
494 0.0238 0.1385 0.0574
495 0.0234 0.1345 0.0496
496 0.0237 0.1441 0.0509
497 0.0237 0.1430 0.0531
498 0.0232 0.1399 0.0559
499 0.0222 0.1447 0.0644
500 0.0228 0.1346 0.0574
Out[414]:
<class 'skorch.net.NeuralNet'>[initialized](
module_=VAE1D(
(encoder): VariationalEncoder1D(
(net): Sequential(
(0): Conv1d(1, 128, kernel_size=(3,), stride=(2,), padding=(1,))
(1): GELU(approximate='none')
(2): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))
(3): GELU(approximate='none')
(4): Conv1d(128, 256, kernel_size=(3,), stride=(2,), padding=(1,))
(5): GELU(approximate='none')
(6): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))
(7): GELU(approximate='none')
(8): Conv1d(256, 256, kernel_size=(3,), stride=(2,), padding=(1,))
(9): GELU(approximate='none')
(10): Flatten(start_dim=1, end_dim=-1)
)
(fc_mu): Linear(in_features=3072, out_features=2, bias=True)
(fc_logvar): Linear(in_features=3072, out_features=2, bias=True)
)
(decoder): Decoder1D(
(linear): Sequential(
(0): Linear(in_features=2, out_features=3072, bias=True)
(1): GELU(approximate='none')
)
(net): Sequential(
(0): ConvTranspose1d(256, 256, kernel_size=(3,), stride=(2,), padding=(1,), output_padding=(1,))
(1): GELU(approximate='none')
(2): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))
(3): GELU(approximate='none')
(4): ConvTranspose1d(256, 128, kernel_size=(3,), stride=(2,), padding=(1,), output_padding=(1,))
(5): GELU(approximate='none')
(6): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))
(7): GELU(approximate='none')
(8): ConvTranspose1d(128, 1, kernel_size=(3,), stride=(2,), padding=(1,), output_padding=(1,))
)
)
),
)In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
<class 'skorch.net.NeuralNet'>[initialized](
module_=VAE1D(
(encoder): VariationalEncoder1D(
(net): Sequential(
(0): Conv1d(1, 128, kernel_size=(3,), stride=(2,), padding=(1,))
(1): GELU(approximate='none')
(2): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))
(3): GELU(approximate='none')
(4): Conv1d(128, 256, kernel_size=(3,), stride=(2,), padding=(1,))
(5): GELU(approximate='none')
(6): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))
(7): GELU(approximate='none')
(8): Conv1d(256, 256, kernel_size=(3,), stride=(2,), padding=(1,))
(9): GELU(approximate='none')
(10): Flatten(start_dim=1, end_dim=-1)
)
(fc_mu): Linear(in_features=3072, out_features=2, bias=True)
(fc_logvar): Linear(in_features=3072, out_features=2, bias=True)
)
(decoder): Decoder1D(
(linear): Sequential(
(0): Linear(in_features=2, out_features=3072, bias=True)
(1): GELU(approximate='none')
)
(net): Sequential(
(0): ConvTranspose1d(256, 256, kernel_size=(3,), stride=(2,), padding=(1,), output_padding=(1,))
(1): GELU(approximate='none')
(2): Conv1d(256, 256, kernel_size=(3,), stride=(1,), padding=(1,))
(3): GELU(approximate='none')
(4): ConvTranspose1d(256, 128, kernel_size=(3,), stride=(2,), padding=(1,), output_padding=(1,))
(5): GELU(approximate='none')
(6): Conv1d(128, 128, kernel_size=(3,), stride=(1,), padding=(1,))
(7): GELU(approximate='none')
(8): ConvTranspose1d(128, 1, kernel_size=(3,), stride=(2,), padding=(1,), output_padding=(1,))
)
)
),
)In [415]:
history = net.history
plt.plot(history[:, 'train_loss'], label='train_loss')
plt.plot(history[:, 'valid_loss'], label='valid_loss')
plt.legend()
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.show()
In [416]:
X_train_recon = net.predict(X_train)
X_test_recon = net.predict(X_test)
In [424]:
i = 20
plt.figure(figsize=(8, 6))
plt.plot(X_train[i].ravel(), label="Original")
plt.plot(X_train_recon[i].ravel(), label="Reconstructed")
plt.legend()
plt.title(f"Class: {LABELS[y_train[i]]}")
plt.show()
In [418]:
vae = net.module_
vae.eval()
with torch.no_grad():
mu_train, logvar_train = vae.encoder(torch.tensor(X_train, dtype=torch.float32).to(DEVICE))
with torch.no_grad():
mu_test, logvar_test = vae.encoder(torch.tensor(X_test, dtype=torch.float32).to(DEVICE))
Z_train = mu_train.cpu().numpy()
Z_test = mu_test.cpu().numpy()
In [419]:
# downstream classification on the latent representation
from sklearn.tree import DecisionTreeClassifier
clf = DecisionTreeClassifier()
clf.fit(Z_train, y_train)
clf.score(Z_test, y_test)
Out[419]:
0.79
In [420]:
sns.scatterplot(x=Z_train[:, 0], y=Z_train[:, 1], hue=le.classes_[y_train], alpha=0.7)
plt.xlabel("Latent Dimension 1")
plt.ylabel("Latent Dimension 2")
plt.title("Latent Space Visualization")
plt.show()
In [452]:
import numpy as np
import torch
import matplotlib.pyplot as plt
from scipy.stats import norm
from sklearn.tree import DecisionTreeClassifier
# Fit classifier on latent representation
clf = DecisionTreeClassifier(random_state=0)
clf.fit(Z_train, y_train)
print("test score:", clf.score(Z_test, y_test))
vae = net.module_
vae.eval()
latent_dim = 2
n_grid = 10
# Gaussian quantile grid
q = np.linspace(0.001, 0.999, n_grid)
z_vals = norm.ppf(q)
zz1, zz2 = np.meshgrid(z_vals, z_vals)
Z_grid = np.stack(
[zz1.ravel(), zz2.ravel()],
axis=1,
)
# Predict class at each latent grid point
y_grid_pred = clf.predict(Z_grid)
Z_grid_tensor = torch.tensor(Z_grid, dtype=torch.float32).to(DEVICE)
with torch.no_grad():
X_grid = vae.decoder(Z_grid_tensor)
X_grid = X_grid.cpu().numpy()
test score: 0.79
In [453]:
import numpy as np
import matplotlib.pyplot as plt
from matplotlib.gridspec import GridSpec, GridSpecFromSubplotSpec
classes = np.unique(y_train)
cmap = plt.get_cmap("coolwarm", len(classes))
class_to_color_idx = {cls: k for k, cls in enumerate(classes)}
fig = plt.figure(figsize=(16, 8), constrained_layout=True)
outer = GridSpec(
1,
2,
figure=fig,
width_ratios=[1, 1.25],
)
# Left: latent scatter
ax_scatter = fig.add_subplot(outer[0, 0])
scatter_colors = [class_to_color_idx[y] for y in y_train]
# sc = ax_scatter.scatter(
# Z_train[:, 0],
# Z_train[:, 1],
# c=scatter_colors,
# cmap=cmap,
# s=25,
# alpha=0.8,
# )
ax_scatter.scatter(
Z_grid[:, 0],
Z_grid[:, 1],
c=[class_to_color_idx[y] for y in y_grid_pred],
cmap=cmap,
marker="x",
s=45,
)
ax_scatter.set_xlabel("z1")
ax_scatter.set_ylabel("z2")
ax_scatter.set_title("Latent space")
# Right: decoded grid
inner = GridSpecFromSubplotSpec(
n_grid,
n_grid,
subplot_spec=outer[0, 1],
wspace=0.05,
hspace=0.05,
)
for i in range(n_grid):
for j in range(n_grid):
idx = i * n_grid + j
# This orientation matches the scatter:
# low z2 at the bottom, high z2 at the top.
ax = fig.add_subplot(inner[n_grid - 1 - i, j])
pred_class = y_grid_pred[idx]
color_idx = class_to_color_idx[pred_class]
color = cmap(color_idx)
ax.plot(X_grid[idx, 0], color=color, linewidth=1.4)
ax.set_xticks([])
ax.set_yticks([])
for spine in ax.spines.values():
spine.set_visible(False)
ax.set_facecolor("none")
fig.suptitle("Latent space and decoded Gaussian quantile grid")
plt.show()
In [ ]:
Exercises¶
- Build a standard autoencoder for the Heartbeat dataset
- Try to use it for classification
- Build a variational autoencoder for the same dataset
In [ ]:
# Import the dataset
X_train, y_train = load_classification("Heartbeat", split="train")
X_test, y_test = load_classification("Heartbeat", split="test")
X_train = X_train.astype(np.float32)
X_test = X_test.astype(np.float32)
X_train.shape, y_train.shape, X_test.shape, y_test.shape
((204, 61, 405), (204,), (205, 61, 405), (205,))
In [ ]:
# Encode the labels
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
y_train = le.fit_transform(y_train)
y_test = le.transform(y_test)
LABELS = le.classes_
le.classes_
array(['abnormal', 'normal'], dtype='<U8')
In [ ]:
i = 100
plt.imshow(X_train[i], aspect='auto', cmap='viridis')
plt.suptitle('Class: {}'.format(LABELS[y_train[i]]), fontsize=16)
plt.show()
In [ ]:
from sklearn.dummy import DummyClassifier
dummy_clf = DummyClassifier(strategy="most_frequent")
dummy_clf.fit(X_train, y_train)
dummy_clf.score(X_test, y_test)
0.7219512195121951
In [ ]: