Assignment - 1 (Machine Learning 2)
Autoencoders
Author: Jimut Bahan Pal
Try to improve the performance (measured on the validation dataset) of the model by the following approaches:
- a) Varying the number of layers and units in the layers
- b) Employing regularization
- c) Corrupting the inputs with some noise
An autoencoder is a special type of neural network that is trained to copy its input to its output. For example, given an image of a handwritten digit, an autoencoder first encodes the image into a lower dimensional latent representation, then decodes the latent representation back to an image. An autoencoder learns to compress the data while minimizing the reconstruction error. [1]
In [149]:
#Source: https://www.tensorflow.org/tutorials/generative/autoencoder
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import tensorflow as tf
from sklearn.metrics import accuracy_score, precision_score, recall_score
from sklearn.model_selection import train_test_split
from tensorflow.keras import layers, losses
from tensorflow.keras.datasets import fashion_mnist
from tensorflow.keras.models import Model
import matplotlib.pyplot as plt
from mpl_toolkits.axes_grid1 import ImageGrid
plt.style.use("dark_background")
In [150]:
(x_train, _), (x_test, _) = fashion_mnist.load_data()
x_train = x_train.astype('float32') / 255.
x_test = x_test.astype('float32') / 255.
print (x_train.shape)
print (x_test.shape)
In [172]:
latent_dim = 64
class Autoencoder(Model):
def __init__(self, latent_dim):
super(Autoencoder, self).__init__()
self.latent_dim = latent_dim
self.encoder = tf.keras.Sequential([
layers.Flatten(),
layers.Dense(latent_dim, activation='tanh'),
layers.BatchNormalization(),
])
self.decoder = tf.keras.Sequential([
layers.Dense(784, activation='sigmoid'),
layers.Reshape((28, 28))
])
def call(self, x):
encoded = self.encoder(x)
decoded = self.decoder(encoded)
return decoded
autoencoder = Autoencoder(latent_dim)
autoencoder.compile(optimizer='Adam', loss=losses.MeanSquaredError(),metrics=['accuracy'])
history = autoencoder.fit(x_train, x_train,
epochs=10,
shuffle=True,
validation_data=(x_test, x_test))
encoded_imgs = autoencoder.encoder(x_test).numpy()
decoded_imgs = autoencoder.decoder(encoded_imgs).numpy()
In [173]:
plt.figure(figsize=(12,7))
legend_acc = []
for item in history.history.keys():
if item.split('_')[-1:] == ['accuracy']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(history.history[item])
#plt.plot( history.history[item], lw=3, c="springgreen")
plt.ylabel('Accuracy ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper left')
plt.show()
plt.figure(figsize=(12,7))
legend_acc = []
for item in history.history.keys():
if item.split('_')[-1:] == ['loss']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(history.history[item])
plt.ylabel('Loss ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper right')
plt.show()
In [174]:
n = 10
plt.figure(figsize=(20, 4))
for i in range(n):
# display original
ax = plt.subplot(2, n, i + 1)
plt.imshow(x_test[i])
plt.title("original")
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
# display reconstruction
ax = plt.subplot(2, n, i + 1 + n)
plt.imshow(decoded_imgs[i])
plt.title("reconstructed")
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
plt.show()
In [ ]:
In [169]:
latent_dim = 64
class Autoencoder(Model):
def __init__(self, latent_dim):
super(Autoencoder, self).__init__()
self.latent_dim = latent_dim
self.encoder = tf.keras.Sequential([
layers.Flatten(),
layers.Dense(latent_dim*8, activation='relu',kernel_regularizer='l1'),
layers.BatchNormalization(),
layers.Dense(latent_dim*8, activation='relu',kernel_regularizer='l1'),
layers.BatchNormalization(),
layers.Dense(latent_dim*4, activation='relu',kernel_regularizer='l1'),
layers.BatchNormalization(),
layers.Dense(latent_dim*4, activation='relu',kernel_regularizer='l1'),
layers.BatchNormalization(),
layers.Dense(latent_dim*2, activation='relu',kernel_regularizer='l1'),
layers.BatchNormalization(),
layers.Dense(latent_dim*2, activation='relu',kernel_regularizer='l1'),
layers.BatchNormalization(),
])
self.decoder = tf.keras.Sequential([
layers.Dense(784, activation='sigmoid'),
layers.Reshape((28, 28))
])
def call(self, x):
encoded = self.encoder(x)
decoded = self.decoder(encoded)
return decoded
autoencoder = Autoencoder(latent_dim)
autoencoder.compile(optimizer='Adam', loss=losses.MeanSquaredError(),metrics=['accuracy'])
history = autoencoder.fit(x_train, x_train,
epochs=10,
shuffle=True,
validation_data=(x_test, x_test))
encoded_imgs = autoencoder.encoder(x_test).numpy()
decoded_imgs = autoencoder.decoder(encoded_imgs).numpy()
In [170]:
plt.figure(figsize=(12,7))
legend_acc = []
for item in history.history.keys():
if item.split('_')[-1:] == ['accuracy']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(history.history[item])
#plt.plot( history.history[item], lw=3, c="springgreen")
plt.ylabel('Accuracy ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper left')
plt.show()
plt.figure(figsize=(12,7))
legend_acc = []
for item in history.history.keys():
if item.split('_')[-1:] == ['loss']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(history.history[item])
plt.ylabel('Loss ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper right')
plt.show()
Image denoising¶
In [121]:
(x_train, _), (x_test, _) = fashion_mnist.load_data()
In [122]:
x_train = x_train.astype('float32') / 255.
x_test = x_test.astype('float32') / 255.
x_train = x_train[..., tf.newaxis]
x_test = x_test[..., tf.newaxis]
print(x_train.shape)
Add random noise to the images¶
In [123]:
noise_factor = 0.2
x_train_noisy = x_train + noise_factor * tf.random.normal(shape=x_train.shape)
x_test_noisy = x_test + noise_factor * tf.random.normal(shape=x_test.shape)
x_train_noisy = tf.clip_by_value(x_train_noisy, clip_value_min=0., clip_value_max=1.)
x_test_noisy = tf.clip_by_value(x_test_noisy, clip_value_min=0., clip_value_max=1.)
Plotting the noisy images¶
In [124]:
n = 10
plt.figure(figsize=(20, 2))
for i in range(n):
ax = plt.subplot(1, n, i + 1)
plt.title("original + noise")
plt.imshow(tf.squeeze(x_test_noisy[i]))
plt.gray()
plt.show()
Task c)¶
Training a CVAE (Convolutional Autoencoders)¶
In [125]:
class Denoise(Model):
def __init__(self):
super(Denoise, self).__init__()
self.encoder = tf.keras.Sequential([
layers.Input(shape=(28, 28, 1)),
layers.Conv2D(16, (3,3), activation='relu', padding='same', strides=2),
layers.Conv2D(8, (3,3), activation='relu', padding='same', strides=2),
])
self.decoder = tf.keras.Sequential([
layers.Conv2DTranspose(8, kernel_size=3, strides=2, activation='relu', padding='same'),
layers.Conv2DTranspose(16, kernel_size=3, strides=2, activation='relu', padding='same'),
layers.Conv2D(1, kernel_size=(3,3), activation='sigmoid', padding='same')])
def call(self, x):
encoded = self.encoder(x)
decoded = self.decoder(encoded)
return decoded
denoising_autoencoder = Denoise()
In [126]:
denoising_autoencoder.compile(optimizer='nadam', loss=losses.MeanSquaredError(),metrics=['accuracy'])
history = denoising_autoencoder.fit(x_train_noisy, x_train,
epochs=10,
shuffle=True,
validation_data=(x_test_noisy, x_test))
encoded_imgs = denoising_autoencoder.encoder(x_test).numpy()
decoded_imgs = denoising_autoencoder.decoder(encoded_imgs).numpy()
In [127]:
plt.figure(figsize=(12,7))
legend_acc = []
for item in history.history.keys():
if item.split('_')[-1:] == ['accuracy']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(history.history[item])
#plt.plot( history.history[item], lw=3, c="springgreen")
plt.ylabel('Accuracy ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper left')
plt.show()
plt.figure(figsize=(12,7))
legend_acc = []
for item in history.history.keys():
if item.split('_')[-1:] == ['loss']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(history.history[item])
plt.ylabel('Loss ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper right')
plt.show()
In [128]:
autoencoder.encoder.summary()
In [129]:
autoencoder.decoder.summary()
In [130]:
encoded_imgs = autoencoder.encoder(x_test).numpy()
decoded_imgs = autoencoder.decoder(encoded_imgs).numpy()
In [57]:
n = 10
plt.figure(figsize=(20, 4))
for i in range(n):
# display original + noise
ax = plt.subplot(2, n, i + 1)
plt.title("original + noise")
plt.imshow(tf.squeeze(x_test_noisy[i]))
plt.gray()
ax.get_xaxis().set_visible(False)
ax.get_yaxis().set_visible(False)
# display reconstruction
bx = plt.subplot(2, n, i + n + 1)
plt.title("reconstructed")
plt.imshow(tf.squeeze(decoded_imgs[i]))
plt.gray()
bx.get_xaxis().set_visible(False)
bx.get_yaxis().set_visible(False)
plt.show()
In [132]:
from keras.layers import *
from keras.models import Model, load_model
from keras.datasets import mnist
from keras.callbacks import ReduceLROnPlateau
from keras import regularizers
from keras import optimizers
from keras import backend as K
import scipy
import matplotlib.pyplot as plt
from matplotlib.offsetbox import OffsetImage, AnnotationBbox, TextArea
import pandas as pd
import numpy as np
Bonus Experiments #Clustering using AE¶
In [137]:
# https://www.kaggle.com/zalando-research/fashionmnist
! wget "https://storage.googleapis.com/kaggle-data-sets/2243/9243/bundle/archive.zip?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gcp-kaggle-com%40kaggle-161607.iam.gserviceaccount.com%2F20201011%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20201011T104838Z&X-Goog-Expires=259199&X-Goog-SignedHeaders=host&X-Goog-Signature=81a721a2b46bbb019329a0dafe9a2f8f4fb91c07f8880392a2ef5cdc5cd46fd8520c20dcf6a12170974cf4667bace6c6d1a3dfda81103e35906781e438bff3eed8a422cd9e37f8bab7c5f482b0edec6899a05b8112f4e9d30e29f291ccca48c858a5d1337afd0b2022e10f3aae33be6b956e43b7b20e7c3970dd9357e9702fa385adaf75ef6af6233c3b5695a76165a7eaeb694314a55287d889fe2bb82bc7a5e05421bd38a3bdea2d9a5ff03373ca234be5b522686913a1cd121555f6d151867954326a26928a46f8cee8af0189f22af0d6f5a8301ba76799dee69536b2eef4913b3877af11131693d99066a436a15d9dc30e52a453e1d0866e7df098734ff5" -O fashion.zip
In [138]:
! unzip -qq fashion.zip
In [140]:
x_train = pd.read_csv('fashion-mnist_train.csv')
x_test = pd.read_csv('fashion-mnist_test.csv')
y_train = x_train.pop('label')
y_test = x_test.pop('label')
x_train = np.array(x_train).reshape(-1, 28, 28) / 255.0
x_test = np.array(x_test).reshape(-1, 28, 28) / 255.0
In [141]:
labeldict = {
0: 'T-shirt/top',
1: 'Trouser',
2: 'Pullover',
3: 'Dress',
4: 'Coat',
5: 'Sandal',
6: 'Shirt',
7: 'Sneaker',
8: 'Bag',
9: 'Ankle boot'
}
fig, ax = plt.subplots(1, 10, figsize=(20, 2))
for i in range(10):
ax[i].imshow(x_train[i], cmap='gray')
ax[i].set_title(labeldict[y_train[i]])
In [142]:
x_train = x_train.reshape(-1, 28, 28, 1)
x_test = x_test.reshape(-1, 28, 28, 1)
def make_and_fit():
inputs = Input(shape=(28, 28, 1))
x = Conv2D(32, (3, 3), activation='relu', padding='same')(inputs)
x = Conv2D(32, (3, 3), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2))(x)
x = BatchNormalization()(x)
x = Conv2D(32, (3, 3), activation='relu', padding='same')(x)
x = Conv2D(32, (3, 3), activation='relu', padding='same')(x)
x = Conv2D(32, (2, 2), activation='relu', padding='same')(x)
x = MaxPooling2D((2, 2))(x)
x = BatchNormalization()(x)
x = Conv2D(16, (2, 2), activation='relu', padding='same')(x)
x = Conv2D(4, (2, 2), activation='relu', padding='same')(x)
x = Conv2D(1, (2, 2), activation='relu', padding='same')(x)
x = Flatten()(x)
encoded = Dense(2, activation='relu')(x)
encoder = Model(inputs=inputs, outputs=encoded)
encoded_inputs = Input(shape=(2,))
x = Dense(4, activation='relu')(encoded_inputs)
x = Reshape((2, 2, 1))(x)
x = Conv2D(4, (2, 2), activation='relu', padding='same')(x)
x = Conv2D(16, (2, 2), activation='relu', padding='same')(x)
x = BatchNormalization()(x)
x = UpSampling2D((7, 7))(x)
x = Conv2D(32, (3, 3), activation='relu', padding='same')(x)
x = Conv2D(32, (3, 3), activation='relu', padding='same')(x)
x = Conv2D(32, (3, 3), activation='relu', padding='same')(x)
x = BatchNormalization()(x)
x = UpSampling2D((2, 2))(x)
x = Conv2D(32, (3, 3), activation='relu', padding='same')(x)
x = Conv2D(32, (3, 3), activation='relu', padding='same')(x)
decoded = Conv2D(1, (3, 3), activation='sigmoid', padding='same')(x)
decoder = Model(inputs=encoded_inputs, outputs=decoded)
x = encoder(inputs)
x = decoder(x)
model = Model(inputs=inputs, outputs=x)
model.compile(optimizer=optimizers.Adam(1e-3), loss='binary_crossentropy', metrics=['accuracy', 'mse'])
print(model.summary())
clr = ReduceLROnPlateau(
monitor='loss',
factor=0.5,
patience=3,
min_delta=0.01,
cooldown=0,
min_lr=1e-7,
verbose=1)
model.fit(
x_train,
x_train,
batch_size=256,
epochs=50,
shuffle=True,
validation_data=(x_test, x_test),
callbacks=[clr])
return model, encoder, decoder
model, encoder, decoder = make_and_fit()
In [148]:
from sklearn.manifold import TSNE
from sklearn.decomposition import PCA
def plot_latent(mode, count, technique):
idx = np.random.choice(len(x_test), count)
inputs = x_test[idx]
fig, ax = plt.subplots(figsize=(10, 7))
ax.set_title(technique)
if technique == 'autoencoder':
coords = encoder.predict(inputs)
elif technique == 'pca':
coords = PCA(n_components=2).fit_transform(inputs.reshape(count, -1))
elif technique == 'tsne':
coords = TSNE(n_components=2).fit_transform(inputs.reshape(count, -1))
if mode == 'imgs':
for image, (x, y) in zip(inputs, coords):
im = OffsetImage(image.reshape(28, 28), zoom=1, cmap='gray')
ab = AnnotationBbox(im, (x, y), xycoords='data', frameon=False)
ax.add_artist(ab)
ax.update_datalim(coords)
ax.autoscale()
elif mode == 'dots':
classes = y_test[idx]
plt.scatter(coords[:, 0], coords[:, 1], c=classes)
plt.colorbar()
for i in range(10):
class_center = np.mean(coords[classes == i], axis=0)
text = TextArea('{} ({})'.format(labeldict[i], i))
ab = AnnotationBbox(text, class_center, xycoords='data', frameon=True)
ax.add_artist(ab)
plt.show()
plot_latent('dots', 10000, 'autoencoder')
plot_latent('dots', 10000, 'pca')
plot_latent('dots', 2000, 'tsne')
plot_latent('imgs', 300, 'autoencoder')
plot_latent('imgs', 300, 'pca')
plot_latent('imgs', 300, 'tsne')
References¶
Acknowledgements¶
- Dripta Maharaj