Analysis(IEMOCAP).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "t=keras.models.load_model(\"weights.best.hdf5\")  #loading the saved model for analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import keras.backend as K\n",
    "out_layer=t.layers[14].output\n",
    "output_fn= K.function([t.input],out_layer)\n",
    "o=output_fn([M_train,x_train])\n",
    "o.shape                                       # to get output from intermediate layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler=StandardScaler()\n",
    "scaler.fit(o)\n",
    "o=scaler.transform(o)\n",
    "print(o.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.decomposition import PCA\n",
    "pca=PCA(n_components=3)\n",
    "pca.fit(o)\n",
    "x_pca=pca.transform(o)\n",
    "print(x_pca.shape)\n",
    "X=x_pca[:,0]\n",
    "Y=x_pca[:,1]\n",
    "Z=x_pca[:,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from mpl_toolkits.mplot3d import Axes3D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig= plt.figure(figsize=(11,11))\n",
    "ax=plt.axes(projection='3d')   #3D plot of PCA coordinates tranformed from the output of intermediate layers\n",
    "ax.scatter(X,Y,Z,c=y_train,marker='^')\n",
    "ax.set_xlabel('x axis')\n",
    "ax.set_ylabel('y axis')\n",
    "ax.set_zlabel('z axis')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.lines as mlines\n",
    "import matplotlib.pyplot as plt\n",
    "#2D plot of PCA coordinates tranformed from the output of intermediate layers\n",
    "fig, (ax1) = plt.subplots(1,1,figsize=(8,8))\n",
    "ax1 = plt.axes()\n",
    "ax1.scatter(X[y_train==1],Y[y_train==1], marker='<',color='blue',s=100)\n",
    "ax1.scatter(X[y_train==2],Y[y_train==2], marker='s',color='orange',s=100)\n",
    "ax1.scatter(X[y_train==3],Y[y_train==3], marker='o',color='green',s=100)\n",
    "ax1.scatter(X[y_train==0],Y[y_train==0], marker='D',color='red',s=100)\n",
    "ax1.set_xlabel('x axis',fontsize=20)\n",
    "ax1.set_ylabel('y axis',fontsize=20)\n",
    "\n",
    "a = mlines.Line2D([], [], color='red', marker='D', linestyle='None',\n",
    "                          markersize=10, label='Angry')\n",
    "b = mlines.Line2D([], [], color='blue', marker='<', linestyle='None',\n",
    "                          markersize=10, label='Happy')\n",
    "c = mlines.Line2D([], [], color='orange', marker='s', linestyle='None',\n",
    "                          markersize=10, label='Sad')\n",
    "d = mlines.Line2D([], [], color='green', marker='o', linestyle='None',\n",
    "                          markersize=10, label='Neutral')\n",
    "\n",
    "plt.legend(handles=[a,b,c,d],fontsize=14)\n",
    "plt.rcParams[\"axes.labelsize\"] = 22\n",
    "fig.text(0.5, -0.005, \"Output Layer\", ha='center',fontsize=18)\n",
    "plt.savefig('PCALastttt(size100).pdf',bbox_inches =\"tight\", \n",
    "            pad_inches = 1, \n",
    "            orientation ='landscape',dpi=1200)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "########### CODE RELATED TO INTERSECT MATRIX #############\n",
    "##########################################################\n",
    "\n",
    "angry,happy,sad,neutral=[],[],[],[]\n",
    "for i in range (x_pca.shape[0]):\n",
    "    if(y_train[i]==3):\n",
    "        neutral.append(x_pca[i])\n",
    "    if(y_train[i]==0):\n",
    "        angry.append(x_pca[i])\n",
    "    if(y_train[i]==1):\n",
    "        happy.append(x_pca[i])  \n",
    "    if(y_train[i]==2):\n",
    "        sad.append(x_pca[i])    \n",
    "angry=np.array(angry) \n",
    "sad=np.array(sad) \n",
    "happy=np.array(happy) \n",
    "neutral=np.array(neutral)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "angstd=np.std(angry,axis=0)         #calculating standard devation of the spread for each class\n",
    "sadstd=np.std(sad,axis=0)\n",
    "happystd=np.std(happy,axis=0)\n",
    "neutralstd=np.std(neutral,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "angm=np.mean(angry,axis=0)         #calculating mean of the spread for each class\n",
    "sadm=np.mean(sad,axis=0)\n",
    "happym=np.mean(happy,axis=0)\n",
    "neutralm=np.mean(neutral,axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "angl=angm-2*angstd                 #approximating the left extreme of the spread for each class\n",
    "sadl=sadm-2*sadstd\n",
    "happyl=happym-2*happystd\n",
    "neutrall=neutralm-2*neutralstd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "angr=angm+2*angstd                 #approximating the right extreme of the spread for each class\n",
    "sadr=sadm+2*sadstd\n",
    "happyr=happym+2*happystd\n",
    "neutralr=neutralm+2*neutralstd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "left=np.zeros((4,3))              #storing values of extremes for each class in left and right matrices\n",
    "left[0]=angl\n",
    "left[1]=happyl\n",
    "left[2]=sadl\n",
    "left[3]=neutrall\n",
    "right=np.zeros((4,3))\n",
    "right[0]=angr\n",
    "right[1]=happyr\n",
    "right[2]=sadr\n",
    "right[3]=neutralr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xx=np.zeros((4,4))\n",
    "Yy=np.zeros((4,4))\n",
    "Zz=np.zeros((4,4))\n",
    "      \n",
    "                                #calculating matrices for each of the coordinates\n",
    "for i in range(0,4):\n",
    "    for j in range(0,4):\n",
    "        Xx[i][j]= max((min(right[j][0],right[i][0])-\n",
    "                       max(left[j][0],left[i][0])),0)/(max(right[j][0],right[i][0])-min(left[j][0],left[i][0]))\n",
    "\n",
    "for i in range(0,4):\n",
    "    for j in range(0,4):\n",
    "        Yy[i][j]= max((min(right[j][1],right[i][1])-\n",
    "                       max(left[j][1],left[i][1])),0)/(max(right[j][1],right[i][1])-min(left[j][1],left[i][1]))\n",
    "\n",
    "for i in range(0,4):\n",
    "    for j in range(0,4):\n",
    "        Zz[i][j]= max((min(right[j][2],right[i][2])-\n",
    "                       max(left[j][2],left[i][2])),0)/(max(right[j][2],right[i][2])-min(left[j][2],left[i][2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "I=np.zeros((4,4))\n",
    "I=np.multiply(Xx,np.multiply(Yy,Zz))  #final intersection matrix I \"element wise product of Xx,Yy,Zz\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sb\n",
    "\n",
    "# plotting intersection matrices using heatmaps\n",
    "\n",
    "fig, (ax1, ax2,ax3) = plt.subplots(1,3,figsize=(33,8))\n",
    "sb.set(font_scale=2)\n",
    "x_axis_labels = [\"Angry\",\"Happy\",\"Sad\",\"Neutral\"] # labels for x-axis\n",
    "y_axis_labels = [\"Angry\",\"Happy\",\"Sad\",\"Neutral\"]\n",
    "#akws = {\"ha\": 'left',\"va\": 'top'}\n",
    "cbar_kws = {\"orientation\":\"vertical\", \n",
    "            \"ticks\":[0,0.1,0.2,0.3,0.4,0.5,0.6,0.7,0.8,0.9,1], \n",
    "           }\n",
    "#plt.subplot(1,3,1)\n",
    "sb.heatmap(Ithird,annot=True,cmap=\"YlGnBu\",fmt='.3f',cbar_kws=cbar_kws,xticklabels=x_axis_labels,linewidths=1, yticklabels=y_axis_labels,ax=ax1)\n",
    "#plt.subplot(1,3,2)\n",
    "sb.heatmap(Isecond,annot=True,cmap=\"YlGnBu\",fmt='.3f',cbar_kws=cbar_kws,xticklabels=x_axis_labels,linewidths=1,yticklabels=y_axis_labels,ax=ax2)\n",
    "#plt.subplot(1,3,3)\n",
    "sb.heatmap(Ifirst,annot=True,cmap=\"YlGnBu\",fmt='.3f',cbar_kws=cbar_kws,xticklabels=x_axis_labels,linewidths=1, yticklabels=y_axis_labels,ax=ax3)\n",
    "#h3.set_yticklabels(h3.get_yticklabels(), rotation=0)\n",
    "ax1.tick_params(rotation=0)\n",
    "ax1.text(1.3, 4.8, \"Third Last Layer\",fontsize=22)\n",
    "ax2.text(1.3, 4.8, \"Second Last Layer\",fontsize=22)\n",
    "ax3.text(1.5, 4.8, \"Output Layer\",fontsize=22)\n",
    "ax2.tick_params(labelrotation=0)\n",
    "ax3.tick_params(labelrotation=0)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "###tensorboard visualization###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "try:\n",
    "  # %tensorflow_version only exists in Colab.\n",
    "  %tensorflow_version 2.x\n",
    "except Exception:\n",
    "  pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os,cv2\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle # if your feature vector is stored in pickle file\n",
    "import tensorflow_datasets as tfds\n",
    "from tensorboard.plugins import projector\n",
    "import tensorflow as tf\n",
    "tf.compat.v1.disable_eager_execution()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "emotion = [\"Neutral\",\"Angry\",\"Happy\",\"Sad\"]\n",
    "\n",
    "PATH = os.getcwd()\n",
    "\n",
    "LOG_DIR = PATH + '/log-1/'\n",
    "metadata = os.path.join(LOG_DIR, 'metadata.tsv')\n",
    "\n",
    "Voices = tf.Variable(o.reshape((len(o), -1)), name='Voices')\n",
    "\n",
    "#def save_metadata(file):\n",
    "with open(metadata, 'w') as metadata_file:\n",
    "    for row in range(912):\n",
    "        c = emotion[y_train[row]]\n",
    "        metadata_file.write('{}\\n'.format(c))\n",
    "\n",
    "        \n",
    "with tf.compat.v1.Session() as sess:\n",
    "    saver = tf.compat.v1.train.Saver([Voices])\n",
    "\n",
    "    sess.run(Voices.initializer)\n",
    "    saver.save(sess, os.path.join(LOG_DIR, 'model.ckpt'))\n",
    "\n",
    "    config = projector.ProjectorConfig()\n",
    "    # One can add multiple embeddings.\n",
    "    embedding = config.embeddings.add()\n",
    "    embedding.tensor_name = Voices.name\n",
    "    # Link this tensor to its metadata file (e.g. labels).\n",
    "    embedding.metadata_path = metadata\n",
    "    # Saves a config file that TensorBoard will read during startup.\n",
    "    projector.visualize_embeddings(tf.compat.v1.summary.FileWriter(LOG_DIR), config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!tensorboard --logdir=./log-1/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}