-
Notifications
You must be signed in to change notification settings - Fork 0
/
proj3_2_ANN_classification_main.py
161 lines (129 loc) · 5.98 KB
/
proj3_2_ANN_classification_main.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
# -*- coding: utf-8 -*-
"""
Created on Thu Apr 15 22:42:24 2021
@author: changai
"""
from proj1_1_load_data import *
import matplotlib.pyplot as plt
import numpy as np
from scipy.io import loadmat
import torch
from sklearn import model_selection
from toolbox_02450 import train_neural_net, draw_neural_net
from scipy import stats
from proj3_2_ANN_classification_validation import ANN_validate
# y = X[:,9].astype('float')
y = y.squeeze()
y = np.reshape(y,(244,1))
print(y)
X = X.squeeze()
# X = X[:,range(0,8)].astype(float)
X = X.astype(float)
N, M = X.shape
#normalizing matrix
X = X - np.ones((N,1)) * X.mean(axis=0)
X = X*(1/np.std(X,axis=0))
print(X.shape)
print(X)
# attributeNames = attributeNames1[range(0,8)].tolist()
attributeNames = attributeNames1.tolist()
classNames = classNames
C = len(classNames)
# Normalize data
# X = stats.zscore(X);
# Parameters for neural network classifier
# n_hidden_units = n_hidd # number of hidden units
n_replicates = 2 # number of networks trained in each k-fold
max_iter = 10000 # stop criterion 2 (max epochs in training)
# K-fold crossvalidation
K = 10 # only five folds to speed up this example
CV = model_selection.KFold(K, shuffle=True)
# Make figure for holding summaries (errors and learning curves)
summaries, summaries_axes = plt.subplots(1,2, figsize=(10,5))
# Make a list for storing assigned color of learning curve for up to K=10
color_list = ['tab:orange', 'tab:green', 'tab:purple', 'tab:brown', 'tab:pink',
'tab:gray', 'tab:olive', 'tab:cyan', 'tab:red', 'tab:blue']
opt_val_errs = []
opt_hidden_units = []
errors = [] # make a list for storing generalizaition error in each loop
for k, (train_index, test_index) in enumerate(CV.split(X,y)):
print('\nCrossvalidation fold: {0}/{1}'.format(k+1,K))
X_train = X[train_index]
y_train = y[train_index]
X_test = X[test_index]
y_test = y[test_index]
n_hidden_units = range(1, 11)
internal_cross_validation = 10
opt_val_err, opt_hidden_unit = ANN_validate(X_train, y_train, n_hidden_units, internal_cross_validation)
opt_val_errs.append(opt_val_err)
opt_hidden_units.append(opt_hidden_unit)
# Extract training and test set for current CV fold, convert to tensors
X_train = torch.Tensor(X[train_index,:])
y_train = torch.Tensor(y[train_index])
X_test = torch.Tensor(X[test_index,:])
y_test = torch.Tensor(y[test_index])
# Define the model, see also Exercise 8.2.2-script for more information.
model = lambda: torch.nn.Sequential(
torch.nn.Linear(M, opt_hidden_unit), #M features to H hiden units
torch.nn.Tanh(), # 1st transfer function,
torch.nn.Linear(opt_hidden_unit, 1), # H hidden units to 1 output neuron
torch.nn.Sigmoid() # final tranfer function
)
loss_fn = torch.nn.BCELoss()
print('Training model of type:\n\n{}\n'.format(str(model())))
# Train the net on training data
net, final_loss, learning_curve = train_neural_net(model,
loss_fn,
X=X_train,
y=y_train,
n_replicates=n_replicates,
max_iter=max_iter)
print('\n\tBest loss: {}\n'.format(final_loss))
# Determine estimated class labels for test set
y_sigmoid = net(X_test)
y_test_est = (y_sigmoid>.5).type(dtype=torch.uint8)
# Determine errors and errors
y_test = y_test.type(dtype=torch.uint8)
e = y_test_est != y_test
error_rate = (sum(e).type(torch.float)/len(y_test)).data.numpy()
errors.append(error_rate) # store error rate for current CV fold
# Display the learning curve for the best net in the current fold
h, = summaries_axes[0].plot(learning_curve, color=color_list[k])
h.set_label('CV fold {0}'.format(k+1))
summaries_axes[0].set_xlabel('Iterations')
summaries_axes[0].set_xlim((0, max_iter))
summaries_axes[0].set_ylabel('Loss')
summaries_axes[0].set_title('optimized hidden unit(s): {} '.format(opt_hidden_units))
# Display the MSE across folds
summaries_axes[1].bar(np.arange(1, K+1), np.squeeze(np.asarray(errors)), color=color_list)
summaries_axes[1].set_xlabel('Fold')
summaries_axes[1].set_xticks(np.arange(1, K+1))
summaries_axes[1].set_ylabel('MSE')
summaries_axes[1].set_title('Test mean-squared-error')
print('Diagram of best neural net in last fold:')
weights = [net[i].weight.data.numpy().T for i in [0,2]]
biases = [net[i].bias.data.numpy() for i in [0,2]]
tf = [str(net[i]) for i in [1,2]]
draw_neural_net(weights, biases, tf, attribute_names=attributeNames)
# Print the average classification error rate
print('\nEstimated generalization error, RMSE: {0}'.format(round(np.sqrt(np.mean(errors)), 4)))
# When dealing with regression outputs, a simple way of looking at the quality
# of predictions visually is by plotting the estimated value as a function of
# the true/known value - these values should all be along a straight line "y=x",
# and if the points are above the line, the model overestimates, whereas if the
# points are below the y=x line, then the model underestimates the value
plt.figure(figsize=(10,10))
y_est = y_test_est.data.numpy(); y_true = y_test.data.numpy()
axis_range = [np.min([y_est, y_true])-1,np.max([y_est, y_true])+1]
plt.plot(axis_range,axis_range,'k--')
plt.plot(y_true, y_est,'ob',alpha=.25)
plt.legend(['Perfect estimation','Model estimations'])
plt.title('Alcohol content: estimated versus true value (for last CV-fold)')
plt.ylim(axis_range); plt.xlim(axis_range)
plt.xlabel('True value')
plt.ylabel('Estimated value')
plt.grid()
plt.show()
print('\n +++++++ ANN regression for classification output ++++++++')
print('Optimized hidden units: {}'.format(opt_hidden_units))
print('test errors: {}'.format(errors))