neuralNet.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 神经网络模型\n",
    "目前有二层神经网络，多层神经网络"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from netParts import *\n",
    "from optim import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def TwoLayerNet(x,y,H,L,lr,iterations):\n",
    "    '''\n",
    "    两层神经网络\n",
    "    H，隐藏层的神经元\n",
    "    L,输出层的神经元\n",
    "    lr:学习率\n",
    "    iterations:display代次数\n",
    "    \n",
    "    '''\n",
    "    N = x.reshape(x.shape[0],-1).shape[1]\n",
    "    w1 = np.random.randn(N,H)\n",
    "    b1 = np.zeros((H,))\n",
    "    w2 = np.random.randn(H,L)\n",
    "    b2 = np.zeros((L,))\n",
    "    for i in range(iterations):\n",
    "        out1,cache1 = affine_forward(x,w1,b1)\n",
    "        out2,cache2 = affine_forward(out1,w2,b2)\n",
    "        loss,dout = svm_loss(out2,y)\n",
    "        dout1,dw2,db2 = affine_backward(dout,cache2)\n",
    "        dx,dw1,db1 = affine_backward(dout1,cache1)\n",
    "        print(loss)\n",
    "        w1 = w1 - lr * dw1\n",
    "        b1 = b1 - lr * db1\n",
    "        w2 = w2 - lr * dw2\n",
    "        b2 = b2 - lr * db2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     73,
     82,
     123
    ]
   },
   "outputs": [],
   "source": [
    "class FullyConnectedNets(object):\n",
    "    '''\n",
    "    全连接神经网络\n",
    "    '''\n",
    "    def __init__(self,input_dims,hidden_dims,num_classes,\n",
    "                 loss_function = svm_loss,activation_function = 'relu',\n",
    "                weight_scale = 1e-2,reg = 0,\n",
    "                 use_norm = None,dropout = 1,\n",
    "                 eps = 1e-8,bn_momentum = 0.9,\n",
    "                config = {'lr':1e-5},grad_function = sgd):\n",
    "        '''\n",
    "        Inputs:\n",
    "        -input_dims:数据特征的维数\n",
    "        -hidden_dims:一个list，里面包含各个隐藏层的神经元数目\n",
    "        -num_classes:预测标签的数目\n",
    "        -loss_function:loss函数,有svm_loss,softmax_loss\n",
    "        -activation_function:激活函数，有sigmoid,relu,tanh\n",
    "        -reg:正则化参数,默认0\n",
    "        -norm:批标准化，None为不标准化,默认None\n",
    "        -dropout:dropout操作，为消除神经元的百分比，1表示不dropout,默认1\n",
    "        -weight_scale:初始化W矩阵的权重\n",
    "        -eps:批标准化的参数\n",
    "        -bn_momentum:批标准化的更新权重\n",
    "         -config:梯度下降的参数\n",
    "            -lr:学习率，默认1e-5\n",
    "            -momentum:momentum和adam的参数,默认0.9\n",
    "            -decay_rate:rmsprop和adam的参数,默认0.99\n",
    "            -eps:rmsprop和adam的参数，默认1e-8\n",
    "        -grad_function:梯度下降函数,默认sgd\n",
    "            -sgd\n",
    "            -rmsprop\n",
    "            -adam\n",
    "            -momentum\n",
    "        \n",
    "        参数:\n",
    "        -self.params:一个字典，包含了 每一层的 信息，用 W1,b1等表示\n",
    "        '''\n",
    "        self.depth = len(hidden_dims) + 1\n",
    "        self.loss_function = loss_function\n",
    "        self.reg = reg\n",
    "        self.use_norm = use_norm\n",
    "        self.use_dropout = dropout != 1\n",
    "        self.dropout = dropout\n",
    "        self.activation_forward = None\n",
    "        self.activation_backward = None\n",
    "        self.bn_params = [{'mode':'train','eps':eps,'momentum':bn_momentum} for i in range(self.depth)]\n",
    "        self.dp_params = {'mode':'train','dropout':self.dropout}\n",
    "        self.config = config\n",
    "        self.grad_function = grad_function\n",
    "        \n",
    "        \n",
    "        \n",
    "        ## 初始化参数\n",
    "        self.params = {}\n",
    "        D = input_dims\n",
    "        for i,H in enumerate(hidden_dims):\n",
    "            self.params['W%d'%(i+1)] = weight_scale * np.random.randn(D,H)\n",
    "            self.params['b%d'%(i+1)] = np.zeros((H,))\n",
    "            if self.use_norm:                    #如果使用批标准化\n",
    "                self.params['gamma%d'%(i+1)] = np.ones((H,))\n",
    "                self.params['beta%d'%(i+1)] = np.zeros((H,))\n",
    "                self.bn_params[i]['running_mean'] = np.zeros((H,))\n",
    "                self.bn_params[i]['running_var'] = np.zeros((H,))\n",
    "            D = H\n",
    "        self.params['W%d'%(i+2)] = weight_scale * np.random.randn(D,num_classes)\n",
    "        self.params['b%d'%(i+2)] = np.zeros((num_classes,))\n",
    "        if self.use_norm:\n",
    "                self.params['gamma%d'%(i+2)] = np.ones((num_classes,))\n",
    "                self.params['beta%d'%(i+2)] = np.zeros((num_classes,))\n",
    "                self.bn_params[i+1]['running_mean'] = np.zeros((num_classes,))\n",
    "                self.bn_params[i+1]['running_var'] = np.zeros((num_classes,))\n",
    "        \n",
    "        \n",
    "        ## 设置激活函数\n",
    "        if activation_function == 'relu':\n",
    "            self.activation_forward = relu_forward\n",
    "            self.activation_backward = relu_backward\n",
    "        if activation_function == 'sigmoid':\n",
    "            self.activation_forward = sigmoid_forward\n",
    "            self.activation_backward = sigmoid_backward\n",
    "        if activation_function == 'tanh':\n",
    "            self.activation_forward = tanh_forward\n",
    "            self.activation_backward = tanh_backward\n",
    "        \n",
    "    def loss(self,x,y):\n",
    "        '''\n",
    "        Inputs:\n",
    "        -x：数据\n",
    "        -y:标签\n",
    "        '''\n",
    "        grads = {}\n",
    "        AllCache = {}\n",
    "        AllOut = {}\n",
    "        for i in range(self.depth):                        #forward\n",
    "            #全连接层\n",
    "            out,cache = affine_forward(x,self.params['W%d'%(i+1)],self.params['b%d'%(i+1)])\n",
    "            AllOut[\"affineOut%d\"%(i+1)] = out\n",
    "            AllCache['affineCache%d'%(i+1)] = cache\n",
    "            #批标准化层\n",
    "            if self.use_norm:\n",
    "                out,cache = norm_forward(out,self.params['gamma%d'%(i+1)],self.params['beta%d'%(i+1)],self.bn_params[i])\n",
    "                AllOut['normOut%d'%(i+1)] = out\n",
    "                AllCache['normCache%d'%(i+1)] = cache\n",
    "            #dropout\n",
    "            if self.use_dropout:\n",
    "                out,cache = dropout_forward(out,self.dp_params)\n",
    "                AllOut['dropOut%d'%(i+1)] = out\n",
    "                AllCache['dropCache%d'%(i+1)] = cache\n",
    "            #激活函数\n",
    "            out,cache = self.activation_forward(out)\n",
    "            AllOut['activationOut%d'%(i+1)] = out\n",
    "            AllCache['activationCache%d'%(i+1)] = cache\n",
    "            x = out\n",
    "        loss,dout = self.loss_function(x,y)                #computer loss\n",
    "        for i in reversed(range(self.depth)):              #backward\n",
    "            #激活函数\n",
    "            dx = self.activation_backward(dout,AllCache['activationCache%d'%(i+1)])\n",
    "            #dropout\n",
    "            if self.use_dropout:\n",
    "                dx = dropout_backward(dx,AllCache['dropCache%d'%(i+1)])\n",
    "            #批标准化层\n",
    "            if self.use_norm:\n",
    "                dx,dgamma,dbeta = norm_backward(dx,AllCache['normCache%d'%(i+1)])\n",
    "                grads['gamma%d'%(i+1)] = dgamma\n",
    "                grads['beta%d'%(i+1)] = dbeta\n",
    "            #全连接层\n",
    "            dx,dw,db = affine_backward(dx,AllCache['affineCache%d'%(i+1)])\n",
    "            \n",
    "            grads['W%d'%(i+1)] = dw + self.reg * self.params['W%d'%(i+1)]\n",
    "            grads['b%d'%(i+1)] = db\n",
    "            dout = dx\n",
    "            loss += 0.5 * self.reg * np.sqrt(np.sum(self.params['W%d'%(i+1)] ** 2))  #正则化\n",
    "        return loss,grads\n",
    "    \n",
    "    def predict(self,x,y = None):\n",
    "        '''\n",
    "        得到score和acc，\n",
    "        若y为None，只返回score\n",
    "        '''\n",
    "        for i in range(self.depth):              #forward\n",
    "            out,cache = affine_forward(x,self.params['W%d'%(i+1)],self.params['b%d'%(i+1)])\n",
    "            if self.use_norm:\n",
    "                if y is None:\n",
    "                    self.bn_params[i]['mode'] = 'test'\n",
    "                out,cache = norm_forward(out,self.params['gamma%d'%(i+1)],self.params['beta%d'%(i+1)],self.bn_params[i])\n",
    "            if self.use_dropout:\n",
    "                if y is None:\n",
    "                    self.dp_params['mode'] = 'test'\n",
    "                out,cache = dropout_forward(out,self.dp_params)\n",
    "            out,cache = self.activation_forward(out)\n",
    "            x = out\n",
    "        score =np.argmax(x,axis=1)\n",
    "        if y is None:\n",
    "            return score,_\n",
    "        acc = np.sum(score == y) / y.shape[0]\n",
    "        return score,acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "code_folding": [
     127
    ]
   },
   "outputs": [],
   "source": [
    "class ConvNets(object):\n",
    "    '''\n",
    "    卷积神经网络\n",
    "    '''\n",
    "    def __init__(self,input_dims,conv_dims,pool_dims,fc_dims,num_classes,\n",
    "                loss_function = svm_loss,activation_function = 'relu',\n",
    "                 pool_function = 'max_pool',\n",
    "                weight_scale = 1e-2,reg = 0,\n",
    "                 use_norm = None,dropout = 1,\n",
    "                 eps = 1e-5,bn_momentum = 0.9,\n",
    "                config = {'lr':1e-5},grad_function = sgd):\n",
    "        '''\n",
    "        Inputs:\n",
    "        -input_dims:数据特征(H,W,C)\n",
    "        -conv_dims:一个list，元素为元组，\n",
    "            -(h,w,f,stride,pad) 一个隐藏层卷积核的宽度，高度，数目,步长，填充\n",
    "        -pool_dims:一个list,元素为元组,\n",
    "            -(h,w,stride) 一个隐藏层池化层的宽度，高度，步长\n",
    "        -fc_dims:一个list,元素为int，每一个fc连接层神经元的个数\n",
    "        -num_classes:预测标签的数目\n",
    "        -loss_function:loss函数,有svm_loss,softmax_loss\n",
    "        -activation_function:激活函数，有sigmoid,relu,tanh\n",
    "        -pool_function:池化层的类型\n",
    "        -reg:正则化参数,默认0\n",
    "        -norm:批标准化，None为不标准化,默认None\n",
    "        -dropout:dropout操作，为消除神经元的百分比，1表示不dropout,默认1\n",
    "        -weight_scale:初始化W矩阵的权重\n",
    "        -eps:批标准化的参数\n",
    "        -bn_momentum:批标准化的更新权重\n",
    "        -config:梯度下降的参数\n",
    "            -lr:学习率，默认1e-5\n",
    "            -momentum:momentum和adam的参数,默认0.9\n",
    "            -decay_rate:rmsprop和adam的参数,默认0.99\n",
    "            -eps:rmsprop和adam的参数，默认1e-8\n",
    "        -grad_function:梯度下降函数,默认sgd\n",
    "            -sgd\n",
    "            -rmsprop\n",
    "            -adam\n",
    "            -momentum\n",
    "        \n",
    "        参数:\n",
    "        -self.params:一个字典，包含了 每一层的 信息，用 W1,b1等表示\n",
    "        '''\n",
    "        self.conv_depth = len(conv_dims)\n",
    "        self.fc_depth = len(fc_dims)+ 1\n",
    "        self.loss_function = loss_function\n",
    "        self.reg = reg\n",
    "        self.use_norm = use_norm\n",
    "        self.use_dropout = dropout != 1\n",
    "        self.dropout = dropout\n",
    "        self.activation_forward = None\n",
    "        self.activation_backward = None\n",
    "        self.conv_bn_params = [{'mode':'train','eps':eps,'momentum':bn_momentum} for i in range(self.conv_depth)]\n",
    "        self.fc_bn_params = [{'mode':'train','eps':eps,'momentum':bn_momentum} for i in range(self.fc_depth)]\n",
    "        self.dp_params = {'mode':'train','dropout':self.dropout}\n",
    "        self.config = config\n",
    "        self.grad_function = grad_function\n",
    "        \n",
    "        ## 初始化参数\n",
    "        self.params = {}      #所有的参数，包括卷积层，池化层，全连接层\n",
    "        Ho,Wo,C = input_dims\n",
    "        # initalize conv params\n",
    "        for i,layer in enumerate(zip(conv_dims,pool_dims)):\n",
    "            conv,pool = layer\n",
    "            #conv\n",
    "            H,W,F,stride,pad = conv\n",
    "            self.params['conv_W%d'%(i+1)] = weight_scale * np.random.randn(F,H,W,C)\n",
    "            self.params['conv_b%d'%(i+1)] = np.zeros((F,))\n",
    "            self.params['conv_stride%d'%(i+1)] = stride\n",
    "            self.params['conv_pad%d'%(i+1)] = pad\n",
    "            #after conv\n",
    "            Ho = int(1+(Ho + 2*pad - H)/stride)\n",
    "            Wo = int(1+(Wo + 2*pad - W)/stride)\n",
    "            #pooling\n",
    "            H,W,stride = pool\n",
    "            self.params['pool_H%d'%(i+1)] = H\n",
    "            self.params['pool_W%d'%(i+1)] = W\n",
    "            self.params['pool_stride%d'%(i+1)] = stride\n",
    "            #after pooling\n",
    "            Ho = int(1 + (Ho - H)/stride)\n",
    "            Wo = int(1 + (Wo - W)/stride)\n",
    "            #norm initialization\n",
    "            if self.use_norm:                    \n",
    "                self.params['conv_gamma%d'%(i+1)] = np.ones((C,))\n",
    "                self.params['conv_beta%d'%(i+1)] = np.zeros((C,))\n",
    "                self.conv_bn_params[i]['running_mean'] = np.zeros((C,))\n",
    "                self.conv_bn_params[i]['running_var'] = np.zeros((C,))\n",
    "            C = F\n",
    "        #initalize fc params\n",
    "        D = Ho * Wo * F   #fc 层的input dim\n",
    "        for i,H in enumerate(fc_dims):\n",
    "            self.params['fc_W%d'%(i+1)] = weight_scale * np.random.randn(D,H)\n",
    "            self.params['fc_b%d'%(i+1)] = np.zeros((H,))\n",
    "            if self.use_norm:                    #如果使用批标准化\n",
    "                self.params['fc_gamma%d'%(i+1)] = np.ones((H,))\n",
    "                self.params['fc_beta%d'%(i+1)] = np.zeros((H,))\n",
    "                self.fc_bn_params[i]['running_mean'] = np.zeros((H,))\n",
    "                self.fc_bn_params[i]['running_var'] = np.zeros((H,))\n",
    "            D = H\n",
    "        self.params['fc_W%d'%(i+2)] = weight_scale * np.random.randn(D,num_classes)\n",
    "        self.params['fc_b%d'%(i+2)] = np.zeros((num_classes,))\n",
    "        if self.use_norm:\n",
    "                self.params['fc_gamma%d'%(i+2)] = np.ones((num_classes,))\n",
    "                self.params['fc_beta%d'%(i+2)] = np.zeros((num_classes,))\n",
    "                self.fc_bn_params[i+1]['running_mean'] = np.zeros((num_classes,))\n",
    "                self.fc_bn_params[i+1]['running_var'] = np.zeros((num_classes,))\n",
    "                     \n",
    "        \n",
    "        ## 设置激活函数\n",
    "        if activation_function == 'relu':\n",
    "            self.activation_forward = relu_forward\n",
    "            self.activation_backward = relu_backward\n",
    "        if activation_function == 'sigmoid':\n",
    "            self.activation_forward = sigmoid_forward\n",
    "            self.activation_backward = sigmoid_backward\n",
    "        if activation_function == 'tanh':\n",
    "            self.activation_forward = tanh_forward\n",
    "            self.activation_backward = tanh_backward\n",
    "        \n",
    "        #设置pool function\n",
    "        if pool_function == 'max_pool':\n",
    "            self.pool_forward = max_pool_forward_naive\n",
    "            self.pool_backward = max_pool_backward_naive\n",
    "        \n",
    "        self.conv_forward = conv_forward_naive\n",
    "        self.conv_backward = conv_backward_naive\n",
    "            \n",
    "    def loss(self,x,y):\n",
    "        '''\n",
    "        Inputs:\n",
    "        -x：数据\n",
    "        -y:标签\n",
    "        '''\n",
    "        grads = {}\n",
    "        AllCache = {}\n",
    "        AllOut = {}\n",
    "        #forward\n",
    "        #卷积层\n",
    "        for i in range(self.conv_depth):\n",
    "            #conv\n",
    "            conv_param = {}\n",
    "            conv_param['stride'] = self.params['conv_stride%d'%(i+1)]\n",
    "            conv_param['pad'] = self.params['conv_pad%d'%(i+1)]\n",
    "            out,cache = self.conv_forward(x,self.params['conv_W%d'%(i+1)],self.params['conv_b%d'%(i+1)],conv_param)\n",
    "            AllOut[\"convOut%d\"%(i+1)] = out\n",
    "            AllCache['convCache%d'%(i+1)] = cache\n",
    "            #标准化\n",
    "            if self.use_norm:\n",
    "                out,cache = spatial_batchnorm_forward(out,self.params['conv_gamma%d'%(i+1)],self.params['conv_beta%d'%(i+1)],self.conv_bn_params[i])\n",
    "                AllOut['conv_normOut%d'%(i+1)] = out\n",
    "                AllCache['conv_normCache%d'%(i+1)] = cache\n",
    "            #激活函数\n",
    "            out,cache = self.activation_forward(out)\n",
    "            AllOut['activationConvOut%d'%(i+1)] = out\n",
    "            AllCache['activationConvCache%d'%(i+1)] = cache\n",
    "            #pooling\n",
    "            pool_param = {}\n",
    "            pool_param['pool_h'] = self.params['pool_H%d'%(i+1)]\n",
    "            pool_param['pool_w'] = self.params['pool_W%d'%(i+1)]\n",
    "            pool_param['stride'] = self.params['pool_stride%d'%(i+1)]\n",
    "            out,cache = self.pool_forward(out,pool_param)\n",
    "            AllOut[\"poolOut%d\"%(i+1)] = out\n",
    "            AllCache['poolCache%d'%(i+1)] = cache\n",
    "            \n",
    "            x = out\n",
    "        #全连接层\n",
    "        for i in range(self.fc_depth):\n",
    "            #fc\n",
    "            out,cache = affine_forward(x,self.params['fc_W%d'%(i+1)],self.params['fc_b%d'%(i+1)])\n",
    "            AllOut[\"affineOut%d\"%(i+1)] = out\n",
    "            AllCache['affineCache%d'%(i+1)] = cache\n",
    "            #批标准化层\n",
    "            if self.use_norm:\n",
    "                out,cache = norm_forward(out,self.params['fc_gamma%d'%(i+1)],self.params['fc_beta%d'%(i+1)],self.fc_bn_params[i])\n",
    "                AllOut['fc_normOut%d'%(i+1)] = out\n",
    "                AllCache['fc_normCache%d'%(i+1)] = cache\n",
    "            #dropout\n",
    "            if self.use_dropout:\n",
    "                out,cache = dropout_forward(out,self.dp_params)\n",
    "                AllOut['dropOut%d'%(i+1)] = out\n",
    "                AllCache['dropCache%d'%(i+1)] = cache\n",
    "            #激活函数\n",
    "            out,cache = self.activation_forward(out)\n",
    "            AllOut['activationFcOut%d'%(i+1)] = out\n",
    "            AllCache['activationFcCache%d'%(i+1)] = cache\n",
    "            \n",
    "            x = out\n",
    "        #compute loss\n",
    "        loss,dout = self.loss_function(x,y)\n",
    "        #fc_backward\n",
    "        for i in reversed(range(self.fc_depth)):         \n",
    "            #激活函数\n",
    "            dx = self.activation_backward(dout,AllCache['activationFcCache%d'%(i+1)])\n",
    "            #dropout\n",
    "            if self.use_dropout:\n",
    "                dx = dropout_backward(dx,AllCache['dropCache%d'%(i+1)])\n",
    "            #批标准化层\n",
    "            if self.use_norm:\n",
    "                dx,dgamma,dbeta = norm_backward(dx,AllCache['fc_normCache%d'%(i+1)])\n",
    "                grads['fc_gamma%d'%(i+1)] = dgamma\n",
    "                grads['fc_beta%d'%(i+1)] = dbeta\n",
    "            #全连接层\n",
    "            dx,dw,db = affine_backward(dx,AllCache['affineCache%d'%(i+1)])\n",
    "            \n",
    "            grads['fc_W%d'%(i+1)] = dw + self.reg * self.params['fc_W%d'%(i+1)]\n",
    "            grads['fc_b%d'%(i+1)] = db\n",
    "            dout = dx\n",
    "            loss += 0.5 * self.reg * np.sqrt(np.sum(self.params['fc_W%d'%(i+1)] ** 2))  #正则化\n",
    "        #conv_backward\n",
    "        for i in reversed(range(self.conv_depth)):\n",
    "            #pooling\n",
    "            dx = self.pool_backward(dout,AllCache['poolCache%d'%(i+1)])\n",
    "            #激活函数\n",
    "            dx = self.activation_backward(dx,AllCache['activationConvCache%d'%(i+1)])\n",
    "            #标准化\n",
    "            if self.use_norm:\n",
    "                dx,dgamma,dbeta = spatial_batchnorm_backward(dx,AllCache['conv_normCache%d'%(i+1)])\n",
    "                grads['conv_gamma%d'%(i+1)] = dgamma\n",
    "                grads['conv_beta%d'%(i+1)] = dbeta\n",
    "            #conv\n",
    "            dx,dw,db = self.conv_backward(dx,AllCache['convCache%d'%(i+1)])\n",
    "            \n",
    "            grads['conv_W%d'%(i+1)] = dw + self.reg * self.params['conv_W%d'%(i+1)]\n",
    "            grads['conv_b%d'%(i+1)] = db\n",
    "            dout = dx\n",
    "            loss += 0.5 * self.reg * np.sqrt(np.sum(self.params['conv_W%d'%(i+1)] ** 2))  #正则化\n",
    "        return loss,grads\n",
    "    \n",
    "    def predict(self,x,y = None):\n",
    "        '''\n",
    "        得到score和acc，\n",
    "        若y为None，只返回score\n",
    "        '''\n",
    "        #conv_forward\n",
    "        for i in range(self.conv_depth):\n",
    "            #conv\n",
    "            conv_param = {}\n",
    "            conv_param['stride'] = self.params['conv_stride%d'%(i+1)]\n",
    "            conv_param['pad'] = self.params['conv_pad%d'%(i+1)]\n",
    "            out,cache = self.conv_forward(x,self.params['conv_W%d'%(i+1)],self.params['conv_b%d'%(i+1)],conv_param)\n",
    "            if self.use_norm:\n",
    "                if y is None:\n",
    "                    self.conv_bn_params[i]['mode'] = 'test'\n",
    "                out,cache = spatial_batchnorm_forward(out,self.params['conv_gamma%d'%(i+1)],self.params['conv_beta%d'%(i+1)],self.conv_bn_params[i])\n",
    "            #激活函数\n",
    "            out,cache = self.activation_forward(out)\n",
    "\n",
    "            #pooling\n",
    "            pool_param = {}\n",
    "            pool_param['pool_h'] = self.params['pool_pH%d'%(i+1)]\n",
    "            pool_param['pool_w'] = self.params['pool_pW%d'%(i+1)]\n",
    "            pool_param['stride'] = self.params['pool_pstride%d'%(i+1)]\n",
    "            out,cache = self.pool_forward(out,pool_param)\n",
    "            \n",
    "            x = out\n",
    "            \n",
    "        #fc_forward\n",
    "        for i in range(self.fc_depth):              #forward\n",
    "            out,cache = affine_forward(x,self.params['fc_W%d'%(i+1)],self.params['fc_b%d'%(i+1)])\n",
    "            if self.use_norm:\n",
    "                if y is None:\n",
    "                    self.fc_bn_params[i]['mode'] = 'test'\n",
    "                out,cache = norm_forward(out,self.params['fc_gamma%d'%(i+1)],self.params['fc_beta%d'%(i+1)],self.fc_bn_params[i])\n",
    "            if self.use_dropout:\n",
    "                if y is None:\n",
    "                    self.dp_params['mode'] = 'test'\n",
    "                out,cache = dropout_forward(out,self.dp_params)\n",
    "            out,cache = self.activation_forward(out)\n",
    "            x = out\n",
    "        score =np.argmax(x,axis=1)\n",
    "        if y is None:\n",
    "            return score,_\n",
    "        acc = np.sum(score == y) / y.shape[0]\n",
    "        return score,acc"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}