From cbfbd41d9422d0450e5f0d7e1b72e012f65dcb05 Mon Sep 17 00:00:00 2001 From: wassname Date: Tue, 3 Nov 2020 07:29:34 +0000 Subject: [PATCH] early results --- .../HyperparamOptimization.ipynb | 239 +++++++++++++----- .../HyperparamOptimization.py | 24 +- 2 files changed, 192 insertions(+), 71 deletions(-) diff --git a/notebooks/c06_Hyperparameter_Optimization/HyperparamOptimization.ipynb b/notebooks/c06_Hyperparameter_Optimization/HyperparamOptimization.ipynb index 869c40b..3ced84d 100644 --- a/notebooks/c06_Hyperparameter_Optimization/HyperparamOptimization.ipynb +++ b/notebooks/c06_Hyperparameter_Optimization/HyperparamOptimization.ipynb @@ -25,15 +25,15 @@ "execution_count": 1, "metadata": { "ExecuteTime": { - "end_time": "2020-10-12T05:32:45.890585Z", - "start_time": "2020-10-12T05:32:45.000957Z" + "end_time": "2020-11-03T07:25:11.265080Z", + "start_time": "2020-11-03T07:25:10.502463Z" } }, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 1, @@ -62,8 +62,8 @@ "execution_count": 2, "metadata": { "ExecuteTime": { - "end_time": "2020-10-12T05:32:45.909164Z", - "start_time": "2020-10-12T05:32:45.893935Z" + "end_time": "2020-11-03T07:25:11.278143Z", + "start_time": "2020-11-03T07:25:11.266610Z" } }, "outputs": [ @@ -107,8 +107,8 @@ "execution_count": 3, "metadata": { "ExecuteTime": { - "end_time": "2020-10-12T05:32:45.917050Z", - "start_time": "2020-10-12T05:32:45.912884Z" + "end_time": "2020-11-03T07:25:11.326665Z", + "start_time": "2020-11-03T07:25:11.279767Z" } }, "outputs": [], @@ -121,8 +121,8 @@ "execution_count": 4, "metadata": { "ExecuteTime": { - "end_time": "2020-10-12T05:32:45.926592Z", - "start_time": "2020-10-12T05:32:45.919151Z" + "end_time": "2020-11-03T07:25:11.373603Z", + "start_time": "2020-11-03T07:25:11.328006Z" } }, "outputs": [], @@ -147,8 +147,8 @@ "execution_count": 5, "metadata": { "ExecuteTime": { - "end_time": "2020-10-12T05:32:45.976068Z", - "start_time": "2020-10-12T05:32:45.928821Z" + "end_time": "2020-11-03T07:25:11.466074Z", + "start_time": "2020-11-03T07:25:11.374776Z" } }, "outputs": [], @@ -170,6 +170,43 @@ ")" ] }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-03T07:25:11.615737Z", + "start_time": "2020-11-03T07:25:11.467972Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAN80lEQVR4nO3df6hcdXrH8c+ncf3DrBpTMYasNhuRWBWbLRqLSl2RrD9QNOqWDVgsBrN/GHChhEr6xyolEuqP0qAsuYu6sWyzLqgYZVkVo6ZFCF5j1JjU1YrdjV6SSozG+KtJnv5xT+Su3vnOzcyZOZP7vF9wmZnzzJnzcLife87Md879OiIEYPL7k6YbANAfhB1IgrADSRB2IAnCDiRxRD83ZpuP/oEeiwiPt7yrI7vtS22/aftt27d281oAesudjrPbniLpd5IWSNou6SVJiyJia2EdjuxAj/XiyD5f0tsR8U5EfCnpV5Ku6uL1APRQN2GfJekPYx5vr5b9EdtLbA/bHu5iWwC61M0HdOOdKnzjND0ihiQNSZzGA03q5si+XdJJYx5/R9L73bUDoFe6CftLkk61/V3bR0r6kaR19bQFoG4dn8ZHxD7bSyU9JWmKpAci4o3aOgNQq46H3jraGO/ZgZ7ryZdqABw+CDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUii4ymbcXiYMmVKsX7sscf2dPtLly5tWTvqqKOK686dO7dYv/nmm4v1u+66q2Vt0aJFxXU///zzYn3lypXF+u23316sN6GrsNt+V9IeSfsl7YuIs+toCkD96jiyXxQRH9TwOgB6iPfsQBLdhj0kPW37ZdtLxnuC7SW2h20Pd7ktAF3o9jT+/Ih43/YJkp6x/V8RsWHsEyJiSNKQJNmOLrcHoENdHdkj4v3qdqekxyTNr6MpAPXrOOy2p9o++uB9ST+QtKWuxgDUq5vT+BmSHrN98HX+PSJ+W0tXk8zJJ59crB955JHF+nnnnVesX3DBBS1r06ZNK6577bXXFutN2r59e7G+atWqYn3hwoUta3v27Cmu++qrrxbrL7zwQrE+iDoOe0S8I+kvauwFQA8x9AYkQdiBJAg7kARhB5Ig7EASjujfl9om6zfo5s2bV6yvX7++WO/1ZaaD6sCBA8X6jTfeWKx/8sknHW97ZGSkWP/www+L9TfffLPjbfdaRHi85RzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtlrMH369GJ948aNxfqcOXPqbKdW7XrfvXt3sX7RRRe1rH355ZfFdbN+/6BbjLMDyRF2IAnCDiRB2IEkCDuQBGEHkiDsQBJM2VyDXbt2FevLli0r1q+44opi/ZVXXinW2/1L5ZLNmzcX6wsWLCjW9+7dW6yfccYZLWu33HJLcV3UiyM7kARhB5Ig7EAShB1IgrADSRB2IAnCDiTB9ewD4JhjjinW200vvHr16pa1xYsXF9e9/vrri/W1a9cW6xg8HV/PbvsB2zttbxmzbLrtZ2y/Vd0eV2ezAOo3kdP4X0i69GvLbpX0bEScKunZ6jGAAdY27BGxQdLXvw96laQ11f01kq6uuS8ANev0u/EzImJEkiJixPYJrZ5oe4mkJR1uB0BNen4hTEQMSRqS+IAOaFKnQ287bM+UpOp2Z30tAeiFTsO+TtIN1f0bJD1eTzsAeqXtabzttZK+L+l429sl/VTSSkm/tr1Y0u8l/bCXTU52H3/8cVfrf/TRRx2ve9NNNxXrDz/8cLHebo51DI62YY+IRS1KF9fcC4Ae4uuyQBKEHUiCsANJEHYgCcIOJMElrpPA1KlTW9aeeOKJ4roXXnhhsX7ZZZcV608//XSxjv5jymYgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSIJx9knulFNOKdY3bdpUrO/evbtYf+6554r14eHhlrX77ruvuG4/fzcnE8bZgeQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJxtmTW7hwYbH+4IMPFutHH310x9tevnx5sf7QQw8V6yMjIx1vezJjnB1IjrADSRB2IAnCDiRB2IEkCDuQBGEHkmCcHUVnnnlmsX7PPfcU6xdf3Plkv6tXry7WV6xYUay/9957HW/7cNbxOLvtB2zvtL1lzLLbbL9ne3P1c3mdzQKo30RO438h6dJxlv9LRMyrfn5Tb1sA6tY27BGxQdKuPvQCoIe6+YBuqe3XqtP841o9yfYS28O2W/8zMgA912nYfybpFEnzJI1IurvVEyNiKCLOjoizO9wWgBp0FPaI2BER+yPigKSfS5pfb1sA6tZR2G3PHPNwoaQtrZ4LYDC0HWe3vVbS9yUdL2mHpJ9Wj+dJCknvSvpxRLS9uJhx9sln2rRpxfqVV17ZstbuWnl73OHir6xfv75YX7BgQbE+WbUaZz9iAisuGmfx/V13BKCv+LoskARhB5Ig7EAShB1IgrADSXCJKxrzxRdfFOtHHFEeLNq3b1+xfskll7SsPf/888V1D2f8K2kgOcIOJEHYgSQIO5AEYQeSIOxAEoQdSKLtVW/I7ayzzirWr7vuumL9nHPOaVlrN47eztatW4v1DRs2dPX6kw1HdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2SW7u3LnF+tKlS4v1a665plg/8cQTD7mnidq/f3+xPjJS/u/lBw4cqLOdwx5HdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgnH2w0C7sexFi8abaHdUu3H02bNnd9JSLYaHh4v1FStWFOvr1q2rs51Jr+2R3fZJtp+zvc32G7ZvqZZPt/2M7beq2+N63y6ATk3kNH6fpL+PiD+X9FeSbrZ9uqRbJT0bEadKerZ6DGBAtQ17RIxExKbq/h5J2yTNknSVpDXV09ZIurpXTQLo3iG9Z7c9W9L3JG2UNCMiRqTRPwi2T2ixzhJJS7prE0C3Jhx229+W9Iikn0TEx/a4c8d9Q0QMSRqqXoOJHYGGTGjozfa3NBr0X0bEo9XiHbZnVvWZknb2pkUAdWh7ZPfoIfx+Sdsi4p4xpXWSbpC0srp9vCcdTgIzZswo1k8//fRi/d577y3WTzvttEPuqS4bN24s1u+8886WtccfL//KcIlqvSZyGn++pL+V9LrtzdWy5RoN+a9tL5b0e0k/7E2LAOrQNuwR8Z+SWr1Bv7jedgD0Cl+XBZIg7EAShB1IgrADSRB2IAkucZ2g6dOnt6ytXr26uO68efOK9Tlz5nTUUx1efPHFYv3uu+8u1p966qli/bPPPjvkntAbHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IIk04+znnntusb5s2bJiff78+S1rs2bN6qinunz66acta6tWrSque8cddxTre/fu7agnDB6O7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQRJpx9oULF3ZV78bWrVuL9SeffLJY37dvX7FeuuZ89+7dxXWRB0d2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUjCEVF+gn2SpIcknSjpgKShiPhX27dJuknS/1ZPXR4Rv2nzWuWNAehaRIw76/JEwj5T0syI2GT7aEkvS7pa0t9I+iQi7ppoE4Qd6L1WYZ/I/Owjkkaq+3tsb5PU7L9mAXDIDuk9u+3Zkr4naWO1aKnt12w/YPu4FusssT1se7irTgF0pe1p/FdPtL8t6QVJKyLiUdszJH0gKST9k0ZP9W9s8xqcxgM91vF7dkmy/S1JT0p6KiLuGac+W9KTEXFmm9ch7ECPtQp729N425Z0v6RtY4NefXB30EJJW7ptEkDvTOTT+Ask/Yek1zU69CZJyyUtkjRPo6fx70r6cfVhXum1OLIDPdbVaXxdCDvQex2fxgOYHAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ9HvK5g8k/c+Yx8dXywbRoPY2qH1J9NapOnv7s1aFvl7P/o2N28MRcXZjDRQMam+D2pdEb53qV2+cxgNJEHYgiabDPtTw9ksGtbdB7Uuit071pbdG37MD6J+mj+wA+oSwA0k0Enbbl9p+0/bbtm9toodWbL9r+3Xbm5uen66aQ2+n7S1jlk23/Yztt6rbcefYa6i322y/V+27zbYvb6i3k2w/Z3ub7Tds31Itb3TfFfrqy37r+3t221Mk/U7SAknbJb0kaVFEbO1rIy3YflfS2RHR+BcwbP+1pE8kPXRwai3b/yxpV0SsrP5QHhcR/zAgvd2mQ5zGu0e9tZpm/O/U4L6rc/rzTjRxZJ8v6e2IeCcivpT0K0lXNdDHwIuIDZJ2fW3xVZLWVPfXaPSXpe9a9DYQImIkIjZV9/dIOjjNeKP7rtBXXzQR9lmS/jDm8XYN1nzvIelp2y/bXtJ0M+OYcXCarer2hIb7+bq203j309emGR+YfdfJ9OfdaiLs401NM0jjf+dHxF9KukzSzdXpKibmZ5JO0egcgCOS7m6ymWqa8Uck/SQiPm6yl7HG6asv+62JsG+XdNKYx9+R9H4DfYwrIt6vbndKekyjbzsGyY6DM+hWtzsb7ucrEbEjIvZHxAFJP1eD+66aZvwRSb+MiEerxY3vu/H66td+ayLsL0k61fZ3bR8p6UeS1jXQxzfYnlp9cCLbUyX9QIM3FfU6STdU92+Q9HiDvfyRQZnGu9U042p43zU+/XlE9P1H0uUa/UT+vyX9YxM9tOhrjqRXq583mu5N0lqNntb9n0bPiBZL+lNJz0p6q7qdPkC9/ZtGp/Z+TaPBmtlQbxdo9K3ha5I2Vz+XN73vCn31Zb/xdVkgCb5BByRB2IEkCDuQBGEHkiDsQBKEHUiCsANJ/D+f1mbtgJ8kQQAAAABJRU5ErkJggg==\n", + "text/plain": [ + "
" + ] + }, + "metadata": { + "needs_background": "light" + }, + "output_type": "display_data" + } + ], + "source": [ + "plt.imshow(train_loader.dataset[0][0][0], cmap='gray')" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -189,11 +226,11 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": { "ExecuteTime": { - "end_time": "2020-10-12T05:32:45.985654Z", - "start_time": "2020-10-12T05:32:45.978381Z" + "end_time": "2020-11-03T07:25:11.621117Z", + "start_time": "2020-11-03T07:25:11.617025Z" } }, "outputs": [], @@ -227,11 +264,11 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": { "ExecuteTime": { - "end_time": "2020-10-12T05:32:46.006848Z", - "start_time": "2020-10-12T05:32:45.989875Z" + "end_time": "2020-11-03T07:25:11.674767Z", + "start_time": "2020-11-03T07:25:11.623013Z" } }, "outputs": [], @@ -270,11 +307,11 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": { "ExecuteTime": { - "end_time": "2020-10-12T05:32:46.031035Z", - "start_time": "2020-10-12T05:32:46.012668Z" + "end_time": "2020-11-03T07:25:11.733318Z", + "start_time": "2020-11-03T07:25:11.676088Z" } }, "outputs": [], @@ -282,7 +319,6 @@ "def train(model, dataloader, criterion, optimizer, n_epochs=1, bs=64, device=\"cpu\"):\n", " # Set model in train mode\n", " model.train().to(device)\n", - " running_loss = 0.0\n", " for epoch in tqdm(range(n_epochs), desc='train', unit='epoch'):\n", " for (x, y) in tqdm(dataloader, leave=False, desc='training'):\n", " y = y.to(device)\n", @@ -294,9 +330,6 @@ " loss = criterion(outputs, y) # Calculate loss\n", " loss.backward() # Do backpropagation\n", " optimizer.step() # Update weights\n", - " # print statistics\n", - " running_loss += loss.item()\n", - " running_loss = 0.0\n", "\n", " print(\"Finished Training\")\n", " return model.eval()" @@ -304,11 +337,11 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": { "ExecuteTime": { - "end_time": "2020-10-12T05:32:46.052711Z", - "start_time": "2020-10-12T05:32:46.036067Z" + "end_time": "2020-11-03T07:25:11.799318Z", + "start_time": "2020-11-03T07:25:11.734484Z" } }, "outputs": [], @@ -344,11 +377,11 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": { "ExecuteTime": { - "end_time": "2020-10-12T05:32:46.086001Z", - "start_time": "2020-10-12T05:32:46.056109Z" + "end_time": "2020-11-03T07:25:11.843196Z", + "start_time": "2020-11-03T07:25:11.800518Z" } }, "outputs": [], @@ -421,6 +454,7 @@ " best_parameters[\"learning_rate\"] = learning_rate\n", "\n", " print('Accuracy Testing: {}'.format(accuracy))\n", + " print('best_parameters', best_parameters)\n", " return best_parameters" ] }, @@ -491,6 +525,18 @@ "" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-03T07:22:41.094486Z", + "start_time": "2020-11-03T07:08:06.775732Z" + } + }, + "outputs": [], + "source": [] + }, { "cell_type": "markdown", "metadata": {}, @@ -502,11 +548,11 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": { "ExecuteTime": { - "end_time": "2020-10-12T05:32:46.109234Z", - "start_time": "2020-10-12T05:32:46.093352Z" + "end_time": "2020-11-03T07:25:11.894576Z", + "start_time": "2020-11-03T07:25:11.844423Z" } }, "outputs": [], @@ -558,6 +604,7 @@ " best_parameters[\"learning_rate\"] = l_rate\n", "\n", " print('Accuracy Testing: {}'.format(accuracy))\n", + " print('best_parameters', best_parameters)\n", "\n", " return best_parameters" ] @@ -574,7 +621,7 @@ "metadata": {}, "source": [ "
\n", - "Exercise 1:
\n", + "Exercise 2:
\n", " \n", "**Warning:** Executing `RandomizedGridSearch` might take around 5 minutes on a GPU.\n", " \n", @@ -635,10 +682,25 @@ { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "ExecuteTime": { + "end_time": "2020-11-03T07:25:07.567433Z", + "start_time": "2020-11-03T07:22:41.095947Z" + } + }, "outputs": [], "source": [] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# More meta\n", + "\n", + "- You can you bayesian priors sample well, and reduce the search time https://scikit-optimize.github.io/stable/\n", + "- You can machine learn the machine learning, of popular example is [MAML](https://arxiv.org/abs/1703.03400)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -650,18 +712,18 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": { "ExecuteTime": { - "end_time": "2020-10-12T05:33:15.365301Z", - "start_time": "2020-10-12T05:32:46.112362Z" + "end_time": "2020-11-03T07:25:50.235122Z", + "start_time": "2020-11-03T07:25:11.895801Z" } }, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "efecf6693a8a4ef29980ffabf6294ce3", + "model_id": "50f4a218e9c94aeca50c51aa66cdd14c", "version_major": 2, "version_minor": 0 }, @@ -682,7 +744,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "3f178fd0a586404cac9c01bee3e0bece", + "model_id": "232a343ab352494e89a83b161ba76aaf", "version_major": 2, "version_minor": 0 }, @@ -696,7 +758,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "96d72640117b4b14967a5d4c0eefa5d5", + "model_id": "", "version_major": 2, "version_minor": 0 }, @@ -708,17 +770,37 @@ "output_type": "display_data" }, { - "ename": "KeyboardInterrupt", - "evalue": "", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 11\u001b[0m \u001b[0;31m######################################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0;31m# YOU CODE GOES HERE\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0mbest_params\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGridSearch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhyperparameters\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 14\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'best_params'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbest_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 15\u001b[0m \u001b[0;31m######################################################\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36mGridSearch\u001b[0;34m(train_data, test_data, hyperparameters)\u001b[0m\n\u001b[1;32m 54\u001b[0m \u001b[0mn_epochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[0mbs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m64\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 57\u001b[0m )\n\u001b[1;32m 58\u001b[0m \u001b[0maccuracy\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, dataloader, criterion, optimizer, n_epochs, bs, device)\u001b[0m\n\u001b[1;32m 8\u001b[0m \u001b[0mx\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 9\u001b[0m \u001b[0;31m# zero the parameter gradients\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 11\u001b[0m \u001b[0;31m# forward + backward + optimize\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 12\u001b[0m \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Get the prediction here\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/.pyenv/versions/jup3.7.3/lib/python3.7/site-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36mzero_grad\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 163\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 164\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 165\u001b[0;31m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgrad\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzero_\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 167\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Finished Training\n", + "Evaluation...\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='testing', max=79.0, style=ProgressStyle(description_width…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy Testing: 0.9816\n", + "best_parameters {'epoch': 1, 'activation': 'selu', 'optimizer': 'adagrad', 'learning_rate': 0.01}\n", + "\n", + "best_params {'epoch': 1, 'activation': 'selu', 'optimizer': 'adagrad', 'learning_rate': 0.01}\n", + "--- 0.6379638552665711 minutes ---\n" ] } ], @@ -743,10 +825,11 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": { "ExecuteTime": { - "start_time": "2020-10-12T05:33:16.142Z" + "end_time": "2020-11-03T07:26:27.011120Z", + "start_time": "2020-11-03T07:25:50.236427Z" }, "lines_to_next_cell": 0 }, @@ -754,7 +837,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "a85be4dfc9134129905ed3b17b906a17", + "model_id": "4f001f69188249f49eaa5d35994e5d53", "version_major": 2, "version_minor": 0 }, @@ -769,13 +852,13 @@ "name": "stdout", "output_type": "stream", "text": [ - "Training NN...optimizer:adam, activation:selu, epochs:1, learning rate:0.001\n" + "Training NN...optimizer:adagrad, activation:sigmoid, epochs:1, learning rate:0.01\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "545a4ae347e64fa9b762cf3d5833935e", + "model_id": "db511c1850e04393b361ebb503f83588", "version_major": 2, "version_minor": 0 }, @@ -789,7 +872,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fb66b13e7b074f13a196f484cd4afed5", + "model_id": "", "version_major": 2, "version_minor": 0 }, @@ -799,6 +882,40 @@ }, "metadata": {}, "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Finished Training\n", + "Evaluation...\n" + ] + }, + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "HBox(children=(FloatProgress(value=0.0, description='testing', max=79.0, style=ProgressStyle(description_width…" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accuracy Testing: 0.8444\n", + "best_parameters {'epoch': 1, 'activation': 'sigmoid', 'optimizer': 'adagrad', 'learning_rate': 0.01}\n", + "\n", + "{'epoch': 1, 'activation': 'sigmoid', 'optimizer': 'adagrad', 'learning_rate': 0.01}\n", + "--- 0.6128351330757141 minutes ---\n" + ] } ], "source": [ @@ -841,9 +958,9 @@ "formats": "ipynb,py" }, "kernelspec": { - "display_name": "Python 3", + "display_name": "deep_ml_curriculum", "language": "python", - "name": "python3" + "name": "deep_ml_curriculum" }, "language_info": { "codemirror_mode": { @@ -855,7 +972,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.7.3" + "version": "3.7.0" }, "toc": { "base_numbering": 1, diff --git a/notebooks/c06_Hyperparameter_Optimization/HyperparamOptimization.py b/notebooks/c06_Hyperparameter_Optimization/HyperparamOptimization.py index 32b6ec2..e19cffe 100644 --- a/notebooks/c06_Hyperparameter_Optimization/HyperparamOptimization.py +++ b/notebooks/c06_Hyperparameter_Optimization/HyperparamOptimization.py @@ -8,9 +8,9 @@ # format_version: '1.5' # jupytext_version: 1.6.0 # kernelspec: -# display_name: Python 3 +# display_name: deep_ml_curriculum # language: python -# name: python3 +# name: deep_ml_curriculum # --- # # Hyperparameter Optimization @@ -82,9 +82,10 @@ batch_size=batch_size + 64, shuffle=True, ) +# - +plt.imshow(train_loader.dataset[0][0][0], cmap='gray') -# - # # GridSearchCV # @@ -146,7 +147,6 @@ def forward(self, x): def train(model, dataloader, criterion, optimizer, n_epochs=1, bs=64, device="cpu"): # Set model in train mode model.train().to(device) - running_loss = 0.0 for epoch in tqdm(range(n_epochs), desc='train', unit='epoch'): for (x, y) in tqdm(dataloader, leave=False, desc='training'): y = y.to(device) @@ -158,9 +158,6 @@ def train(model, dataloader, criterion, optimizer, n_epochs=1, bs=64, device="cp loss = criterion(outputs, y) # Calculate loss loss.backward() # Do backpropagation optimizer.step() # Update weights - # print statistics - running_loss += loss.item() - running_loss = 0.0 print("Finished Training") return model.eval() @@ -259,9 +256,8 @@ def GridSearch(train_data, test_data, hyperparameters): best_parameters["learning_rate"] = learning_rate print('Accuracy Testing: {}'.format(accuracy)) + print('best_parameters', best_parameters) return best_parameters - - # - #
@@ -321,6 +317,8 @@ def GridSearch(train_data, test_data, hyperparameters): # ``` # + + # # Randomized Grid Search # # As you can notice, GridSearch evaluates the model in combining every parameter. This of course, can be very inneficient. By contrast, Random Search sets up a grid of hyperparameter values and selects random combinations to train the model and evaluate. In random search you can control the number of hyperparameter combinations that attempted. It is possible that Random search does not find the best possible answer but in cases where the search space and computational time is limited it will work the best. @@ -372,13 +370,14 @@ def RandomizedGridSearch(train_data, test_data, hyperparameters, num_combination best_parameters["learning_rate"] = l_rate print('Accuracy Testing: {}'.format(accuracy)) + print('best_parameters', best_parameters) return best_parameters # In the next exercise we will increase the parameter space search but will limit to a number of 10 random searches. #
-# Exercise 1:
+# Exercise 2:
# # **Warning:** Executing `RandomizedGridSearch` might take around 5 minutes on a GPU. # @@ -432,6 +431,11 @@ def RandomizedGridSearch(train_data, test_data, hyperparameters, num_combination +# # More meta +# +# - You can you bayesian priors sample well, and reduce the search time https://scikit-optimize.github.io/stable/ +# - You can machine learn the machine learning, of popular example is [MAML](https://arxiv.org/abs/1703.03400) + # # Auto:Test # # Ignore the below, it's for automatic testing