Assignment - 5 (ANN)
Neural network using Keras
Author: Jimut Bahan Pal
Question¶
Import the Iris dataset and divide the data into training and test datasets.
a) Train a 2-layered neural network (1 hidden layer) model on the data. Employ Batch normalization and initialize the weights using the Xaviers method with normal distribution. Make predictions on the test data and compute the accuracy.
b) Vary the number of neurons in the hidden layer and show the impact on accuracy.
c) Take (one of) the case which has best accuracy and employ dropout with different values of dropout rate (say from 0.1 to 0.7). Plot the variation of accuracy versus dropout rate.
d) Now increase the number of hidden layers (with same no. of neurons) and show the variation of accuracy with respect to the number of hidden layers. You can select the number of neurons as determined best in part b)
e) Using the setting as determined best in part b), train the model with a few different optimization algorithms (with default settings) and print the accuracy from each case. You may try adam, adagrad, adadelta.
Hint (python): from keras.models import Sequential from keras.layers import Dense
model = Sequential() model.add(Dense(5, input_dim=2, activation='relu')) model.add(Dense(4, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
Loading the necessary libraries¶
In [1]:
import pandas as pd
import numpy as np
from sklearn import datasets
import seaborn as sns
import matplotlib.pyplot as plt
from matplotlib.colors import ListedColormap
from sklearn.datasets import load_iris
from keras.models import Sequential
from keras.layers import Dense,Dropout
from keras.optimizers import SGD,Adam
from keras import optimizers
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from tensorflow.python.keras.utils import np_utils
from keras.layers.normalization import BatchNormalization
from keras.utils.vis_utils import plot_model #plotting the model structure
Loading the iris data¶
In [2]:
iris = datasets.load_iris()
X_data = iris.data[:,0:4].astype(float) # 4-features
Y_data = iris.target # labels (3 classes)
In [3]:
encoder = LabelEncoder()
In [4]:
encoder.fit(Y_data)
Out[4]:
In [5]:
encoded_Y = encoder.transform(Y_data)
In [6]:
dummy_y = np_utils.to_categorical(encoded_Y)
Dividing the data into training and test dataset¶
In [7]:
#dummy_y
# set aside 20% of train and test data for evaluation
X_train, X_test, y_train, y_test = train_test_split(X_data, dummy_y,
test_size=0.2, shuffle = True, random_state = 123)
print("X_train shape: {}".format(X_train.shape))
print("X_test shape: {}".format(X_test.shape))
print("y_train shape: {}".format(y_train.shape))
print("y_test shape: {}".format(y_test.shape))
In [8]:
print(X_train[:10])
In [9]:
# get reproducible results:: 99.17%
# Seed value
# Apparently you may use different seed values at each stage
seed_value= 0
import tensorflow as tf
# 1. Set the `PYTHONHASHSEED` environment variable at a fixed value
import os
os.environ['PYTHONHASHSEED']=str(seed_value)
# 2. Set the `python` built-in pseudo-random generator at a fixed value
import random
random.seed(seed_value)
# 3. Set the `numpy` pseudo-random generator at a fixed value
import numpy as np
np.random.seed(seed_value)
# 4. Set the `tensorflow` pseudo-random generator at a fixed value
import tensorflow as tf
tf.random.set_seed(seed_value)
# for later versions:
# tf.compat.v1.set_random_seed(seed_value)
# for later versions:
# session_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
# sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph(), config=session_conf)
# tf.compat.v1.keras.backend.set_session(sess)
In [10]:
#Defining the model
# Seq approach
model = Sequential()
# input dimension is 4 since there are 4 features (input layer)
model.add(Dense(4, input_dim=4, activation='softmax'))
# 1st hidden layer
# initialising the weights using Xavier's Method
model.add(Dense(10, activation='softmax',kernel_initializer = 'glorot_normal'))
model.add(BatchNormalization())
# 2nd layer is output layer
model.add(Dense(3, activation='softmax'))
# output layer
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
model.summary()
In [11]:
plot_model(model, to_file='model_plot.png', show_shapes=True, show_layer_names=True,dpi=60)
Out[11]:
In [12]:
#fitting the model and predicting
hist = model.fit(X_train,y_train,epochs=100)
In [17]:
print(hist.history.keys())
In [18]:
plt.figure(figsize=(7,5))
legend_acc = []
for item in hist.history.keys():
if item.split('_')[-1:] == ['accuracy']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(hist.history[item])
plt.ylabel('Accuracy ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper left')
plt.show()
plt.figure(figsize=(7,5))
legend_acc = []
for item in hist.history.keys():
if item.split('_')[-1:] == ['loss']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(hist.history[item])
plt.ylabel('Loss ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper right')
plt.show()
In [19]:
y_pred = model.predict(X_test)
y_test_class = np.argmax(y_test,axis=1)
y_pred_class = np.argmax(y_pred,axis=1)
In [20]:
print(y_pred)
print(y_test_class)
print(y_pred_class)
Lets brute force crazily, starting from Number of Neurons in hidden layer = 4¶
In [22]:
deep_neurons = 50
accuracy_neuron_list = []
for item in range(4,deep_neurons):
model = Sequential()
# input dimension is 4 since there are 4 features (input layer)
model.add(Dense(4, input_dim=4, activation='softmax'))
# 1st hidden layer
# initialising the weights using Xavier's Method
model.add(Dense(item, activation='softmax',kernel_initializer = 'glorot_normal'))
model.add(BatchNormalization())
# 2nd layer is output layer
model.add(Dense(3, activation='softmax'))
# output layer
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
hist = model.fit(X_train,y_train,epochs=100,verbose=0)
#model.summary()
print("Hidden Neurons => ",item)
print("Accuracy => ",hist.history.get('accuracy')[-1])
accuracy_neuron_list.append([item,hist.history.get('accuracy')[-1]])
plt.figure(figsize=(7,5))
legend_acc = []
for item in hist.history.keys():
if item.split('_')[-1:] == ['accuracy']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(hist.history[item])
plt.ylabel('Accuracy ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper left')
plt.show()
plt.figure(figsize=(7,5))
legend_acc = []
for item in hist.history.keys():
if item.split('_')[-1:] == ['loss']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(hist.history[item])
plt.ylabel('Loss ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper right')
plt.show()
In [27]:
print(accuracy_neuron_list)
In [28]:
#accuracy_neuron_list
print("No. of Hidden Neurons selected => ", 18)
In [29]:
print("Max accuracy got =>",max(second))
In [30]:
first = [i[0] for i in accuracy_neuron_list]
second = [i[1] for i in accuracy_neuron_list]
fig, ax = plt.subplots()
ax.plot(first, second,color='red')
ax.set(xlabel='No. of Hidden Neurons', ylabel='Accuracy',
title='Variation of Accuracy with No. of Hidden Neurons')
ax.grid()
plt.show()
So we have brute forced and got number of neuron = 18¶
In [31]:
dropout_acc_list = []
for dropout_ in range(1,8):
dropout = dropout_/10
#print(dropout)
#Defining the model
# Seq approach
model = Sequential()
# input dimension is 4 since there are 4 features (input layer)
model.add(Dense(4, input_dim=4, activation='softmax'))
# 1st hidden layer
# initialising the weights using Xavier's Method
model.add(Dense(18, activation='softmax',kernel_initializer = 'glorot_normal'))
model.add(Dropout(dropout))
model.add(BatchNormalization())
# 2nd layer is output layer
model.add(Dense(3, activation='softmax'))
# output layer
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
#model.summary()
hist = model.fit(X_train,y_train,epochs=100,verbose=0)
print("Dropout => ",dropout)
print("Accuracy => ",hist.history.get('accuracy')[-1])
dropout_acc_list.append([dropout,hist.history.get('accuracy')[-1]])
plt.figure(figsize=(7,5))
legend_acc = []
for item in hist.history.keys():
if item.split('_')[-1:] == ['accuracy']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(hist.history[item])
plt.ylabel('Accuracy ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper left')
plt.show()
plt.figure(figsize=(7,5))
legend_acc = []
for item in hist.history.keys():
if item.split('_')[-1:] == ['loss']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(hist.history[item])
plt.ylabel('Loss ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper right')
plt.show()
In [32]:
dropout_acc_list
Out[32]:
In [34]:
first_d = [i[0] for i in dropout_acc_list]
second_d = [i[1] for i in dropout_acc_list]
fig, ax = plt.subplots()
ax.plot(first_d, second_d,color='green')
ax.set(xlabel='No. of Hidden Neurons', ylabel='Accuracy',
title='Variation of Accuracy with Dropout')
#ax.grid()
plt.show()
Conclusion: Don't add Dropout to such a simple model, it is degrading¶
In [35]:
hiddenlayer_acc_list = []
for hidden_layer in range(1,8):
#dropout = dropout_/10
#print(dropout)
#Defining the model
# Seq approach
model = Sequential()
# input dimension is 4 since there are 4 features (input layer)
model.add(Dense(4, input_dim=4, activation='softmax'))
# add hidden layers
for layer_s in range(1,hidden_layer):
# 42 is the no. of neurons determined in part (b)
# -> the best (though the accuracy curve is not so smooth)
model.add(Dense(18, activation='softmax',kernel_initializer = 'glorot_normal'))
#model.add(Dropout(dropout))
model.add(BatchNormalization())
# 2nd layer is output layer
model.add(Dense(3, activation='softmax'))
# output layer
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
#model.summary()
hist = model.fit(X_train,y_train,epochs=100,verbose=0)
print("Number of hidden layers => ",hidden_layer)
print("Accuracy => ",hist.history.get('accuracy')[-1])
hiddenlayer_acc_list.append([hidden_layer,hist.history.get('accuracy')[-1]])
plt.figure(figsize=(7,5))
legend_acc = []
for item in hist.history.keys():
if item.split('_')[-1:] == ['accuracy']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(hist.history[item])
plt.ylabel('Accuracy ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper left')
plt.show()
plt.figure(figsize=(7,5))
legend_acc = []
for item in hist.history.keys():
if item.split('_')[-1:] == ['loss']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(hist.history[item])
plt.ylabel('Loss ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper right')
plt.show()
In [39]:
hiddenlayer_acc_list
Out[39]:
In [42]:
max([i[1] for i in hiddenlayer_acc_list])
Out[42]:
In [43]:
#accuracy_neuron_list
print("No. of hidden Layers selected => ", 2, " Since, we should select the simple one from Occam's Rasor ")
In [44]:
first_d = [i[0] for i in hiddenlayer_acc_list]
second_d = [i[1] for i in hiddenlayer_acc_list]
fig, ax = plt.subplots()
ax.plot(first_d, second_d,color='blue')
ax.set(xlabel='No. of Hidden Layers with 42 Neurons', ylabel='Accuracy',
title='Variation of Accuracy with No. of Hidden Neurons')
#ax.grid()
plt.show()
With Adam¶
In [45]:
model = Sequential()
# input dimension is 4 since there are 4 features (input layer)
model.add(Dense(4, input_dim=4, activation='softmax'))
# 1st hidden layer
# initialising the weights using Xavier's Method
model.add(Dense(18, activation='softmax',kernel_initializer = 'glorot_normal'))
# model.add(Dropout(dropout))
model.add(BatchNormalization())
# 2nd layer is output layer
model.add(Dense(3, activation='softmax'))
# output layer
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])
#model.summary()
hist = model.fit(X_train,y_train,epochs=100,verbose=1)
#print("Dropout => ",dropout)
In [46]:
print("Accuracy => ",hist.history.get('accuracy')[-1])
#dropout_acc_list.append([dropout,hist.history.get('accuracy')[-1]])
plt.figure(figsize=(7,5))
legend_acc = []
for item in hist.history.keys():
if item.split('_')[-1:] == ['accuracy']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(hist.history[item])
plt.ylabel('Accuracy ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper left')
plt.show()
plt.figure(figsize=(7,5))
legend_acc = []
for item in hist.history.keys():
if item.split('_')[-1:] == ['loss']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(hist.history[item])
plt.ylabel('Loss ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper right')
plt.show()
With Adagrad¶
In [47]:
model = Sequential()
# input dimension is 4 since there are 4 features (input layer)
model.add(Dense(4, input_dim=4, activation='softmax'))
# 1st hidden layer
# initialising the weights using Xavier's Method
model.add(Dense(18, activation='softmax',kernel_initializer = 'glorot_normal'))
# model.add(Dropout(dropout))
model.add(BatchNormalization())
# 2nd layer is output layer
model.add(Dense(3, activation='softmax'))
# output layer
model.compile(loss='categorical_crossentropy', optimizer='adagrad', metrics=['accuracy'])
#model.summary()
hist = model.fit(X_train,y_train,epochs=100,verbose=1)
#print("Dropout => ",dropout)
In [48]:
print("Accuracy => ",hist.history.get('accuracy')[-1])
#dropout_acc_list.append([dropout,hist.history.get('accuracy')[-1]])
plt.figure(figsize=(7,5))
legend_acc = []
for item in hist.history.keys():
if item.split('_')[-1:] == ['accuracy']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(hist.history[item])
plt.ylabel('Accuracy ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper left')
plt.show()
plt.figure(figsize=(7,5))
legend_acc = []
for item in hist.history.keys():
if item.split('_')[-1:] == ['loss']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(hist.history[item])
plt.ylabel('Loss ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper right')
plt.show()
With Adadelta¶
In [50]:
model = Sequential()
# input dimension is 4 since there are 4 features (input layer)
model.add(Dense(4, input_dim=4, activation='softmax'))
# 1st hidden layer
# initialising the weights using Xavier's Method
model.add(Dense(18, activation='softmax',kernel_initializer = 'glorot_normal'))
# model.add(Dropout(dropout))
model.add(BatchNormalization())
# 2nd layer is output layer
model.add(Dense(3, activation='softmax'))
# output layer
model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy'])
#model.summary()
hist = model.fit(X_train,y_train,epochs=100,verbose=1)
#print("Dropout => ",dropout)
In [51]:
print("Accuracy => ",hist.history.get('accuracy')[-1])
#dropout_acc_list.append([dropout,hist.history.get('accuracy')[-1]])
plt.figure(figsize=(7,5))
legend_acc = []
for item in hist.history.keys():
if item.split('_')[-1:] == ['accuracy']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(hist.history[item])
plt.ylabel('Accuracy ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper left')
plt.show()
plt.figure(figsize=(7,5))
legend_acc = []
for item in hist.history.keys():
if item.split('_')[-1:] == ['loss']:
#print("Accuracy = ",item)
legend_acc.append(item)
plt.plot(hist.history[item])
plt.ylabel('Loss ->')
plt.xlabel('Epoch ->')
plt.legend(legend_acc, loc='upper right')
plt.show()
