lstm_for_delta_loop.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "import math\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import keras\n",
    "from keras.preprocessing import image\n",
    "from keras import applications\n",
    "from keras.models import Sequential\n",
    "from keras.applications import vgg16\n",
    "from keras.applications.vgg16 import VGG16\n",
    "from keras.applications.vgg16 import preprocess_input\n",
    "from keras.models import Model\n",
    "from keras.layers import Conv2D, Conv3D,Input, ZeroPadding3D, Reshape,LSTM\n",
    "from keras.layers.convolutional import Convolution2D, Convolution3D, MaxPooling2D, ZeroPadding2D,ZeroPadding3D \n",
    "from keras.layers.core import Reshape\n",
    "import os\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import Sequential\n",
    "from keras.callbacks import CSVLogger\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Flatten, Dense, Dropout\n",
    "from keras.optimizers import SGD\n",
    "from keras.layers import Merge \n",
    "import numpy as np\n",
    "import keras\n",
    "import keras.backend as K\n",
    "from keras.callbacks import LambdaCallback\n",
    "from keras import regularizers\n",
    "from keras.models import load_model\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import SGDClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X = np.load(\"train_X_normalized_lstm_delta.pkl.npy\")\n",
    "# Y = np.load(\"train_Y_normalized_lstm_delta.pkl.npy\")\n",
    "# X_test = np.load(\"test_X_normalized_lstm_delta.pkl.npy\")\n",
    "# Y_test = np.load(\"test_Y_normalized_lstm_delta.pkl.npy\")\n",
    "\n",
    "# de_norm_test = np.load(\"de_norm_lstm_delta_test.pkl.npy\")\n",
    "# unnormalized_Y = np.load(\"unnormalized_Y_test_v2.pkl.npy\")\n",
    "# Y_reg = np.load(\"test_Y_normalized_lstm_regression_v2.pkl.npy\")\n",
    "# X = np.load(\"train_X_normalized_lstm_classification.pkl.npy\")\n",
    "# Y = np.load(\"train_Y_normalized_lstm_classification.pkl.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "# of all y: (810567,)\n",
      "# of positive y: (349583,)\n",
      "# of negative y: (316705,)\n"
     ]
    }
   ],
   "source": [
    "## Preprocessing\n",
    "Inf = math.inf\n",
    "with open('BTC-USD-60.pkl', 'rb') as f:\n",
    "    data = pickle.load(f)\n",
    "\n",
    "# sort the data based on ascending-order of time\n",
    "data = data.sort_values(by=['time'])\n",
    "\n",
    "# save the original valuse for the fulture delta calculation\n",
    "data_original = data\n",
    "data_original = data_original[data_original.close > 200]\n",
    "close_price = np.array(data_original.close)\n",
    "\n",
    "# drop time\n",
    "data = data[data.close > 200]\n",
    "time = np.array(data.time)\n",
    "data = data.drop('time', 1)\n",
    "\n",
    "# get all delta y's\n",
    "y_distribution = []\n",
    "for i in range(len(close_price) - 60):       \n",
    "    if time[i+60] - time[i] == 60*60:\n",
    "          y_distribution.append( (close_price[i + 60] - close_price[i + 59]) / close_price[i + 59] * 100)  \n",
    "\n",
    "# since <0 and >0 buckets are not symetrical, we can do it separately\n",
    "y_distribution = sorted(y_distribution)\n",
    "print(\"# of all y:\",np.shape(y_distribution))\n",
    "y_distribution = list(filter(lambda a: a != 0, y_distribution))\n",
    "y_pos = list(filter(lambda a: a > 0, y_distribution))\n",
    "print(\"# of positive y:\",np.shape(y_pos))\n",
    "y_neg = list(filter(lambda a: a <0, y_distribution))\n",
    "print(\"# of negative y:\",np.shape(y_neg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-11.428571428571429, -7.889116022099449, -7.546567972598969, -5.712489761709748, -5.356520949107373, -5.039983164983171, -4.886534012031465, -4.8548350398179725, -4.737981401433671, -4.444333333333331, -4.128440366972478, -3.7993371970660843, -3.646082949308751, -3.5780875432833135, -3.479230769230768, -3.4373268106490786, -3.4135850062166795, -3.4066245386046585, -3.225806451612903, -3.1926865798069235, -3.1925665080687815, -3.1849984079310842, -3.0914911552145217, -2.9788056847600752, -2.941176470588235, -2.912621359223301, -2.868985771243238, -2.834653187306931, -2.7906280400422623, -2.754458181660753, -2.746268656716418, -2.7418022933717903, -2.702791461412157, -2.6490083768267394, -2.635066578974331, -2.609998161973131, -2.5939458560825734, -2.58129021782787, -2.576257960597552, -2.556394213729383, -2.555860600080112, -2.5069637883008355, -2.5031088668651256, -2.4721689842188965, -2.4604105571847468, -2.423187186115925, -2.407273275216995, -2.4036968929859333, -2.3801262431525037, -2.3739364088531074, -2.3563631103373024, -2.3391228070175534, -2.3368191337232562, -2.317550175419328, -2.306280193236715, -2.2969766663270166, -2.287107200832987, -2.2685777907140037, -2.267841409691631, -2.2555656056884237, -2.24401544401544, -2.237926972909305, -2.2297961412449965, -2.206575274266106, -2.2058823529411766, -2.1880886827995942, -2.186573937766864, -2.184329199549031, -2.184313725490189, -2.13959266675227, -2.0858784561565593, -2.083333333333333, -2.068964090369593, -2.0604132608727683, -2.0287490348763586, -2.0270283966408087, -2.0141726618704987, -2.005073842529683, -2.0041833430542377, -2.0021044438358513, -1.9941777132835463, -1.9931408486199738, -1.9752761722380523, -1.9692267947704678, -1.9687112240668758, -1.9515400880050402, -1.9469720729401268, -1.939911534358958, -1.9314891112419061, -1.9239329343686675, -1.923919419631261, -1.9193827282197742, -1.9090909090909092, -1.9027171096468447, -1.901886534984474, -1.8990037359900347, -1.89893069834461, -1.8956173129748246, -1.893018796405719, -1.892331161072767]\n"
     ]
    }
   ],
   "source": [
    "print(y_distribution[:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xm8HFWZ//HPlwRCACEJCRiSSFgiAjoqZDBuIxKEgA5hZmAIoxIQjSDOjDM6GkB/KMIIjoLigqKERZFFHCQMMCEGoqOyJOybmAtEEhPIhYRdAoHn98c5DZVO9719l+6um3zfr1e/uurUqaqnTlf301V1uloRgZmZWdls1O4AzMzManGCMjOzUnKCMjOzUnKCMjOzUnKCMjOzUnKCMjOzUnKCKgFJ50s6JQ+/V9ID/bjsayVNz8NHSvptPy77w5Ku66/l9WC975a0SNKzkg7uh+X1a5tb30kKSTu3O45qkoZIuk/S69sdS18U21fSGZKOaXdMtThBlUxE/F9E7NJdPUlflvTTBpZ3QERc0Ne4JI3PO/XgwrIvioj9+rrsXjgZ+G5EbBERv+zrwhpt8/VR8cuRNWQG8JuIeLTdgfSj/wJOlLRJuwOp5gS1nlKyvr6+2wP39seCigm3rAZCjBuQTwI/6elMZX4NI2I58AfgoHbHso6I8KPFD+DtwG3AM8ClwCXAKXna3sDSQt0vAH/OdR8AJgNTgBeBl4BngTtz3fnAqcDvgL8AO+eyj+fpR+Zp3wGeIu2UkwvrWgzsWxj/MvDTPPwIEHl9zwLvzMv7baH+u4AFedkLgHcVps0HvprX/wxwHTCyizb6BNABrARmA9vl8geBV/L2PQsMqTHvYuB44D5gFXAesGmxfXO7Pkr6sKlu83HAfwOdwBOko7XKtI8B9+flzgG2rxP/+NxeM4BlwHLgs4XpGwEz8/Y8AVwGjKia9+jc7r/J5e8Bfg88CSwBjszlQ4Bv5LqPAT8AhlZt72eBFTmOo/K0GaR96MXcllfl8kpcz+Q2/LtC3IOAbwKPAw8Dn86xDs7TtwLOzev5M3AKMKhOGw0CTiis61ZgXJ4WwDHAotzW3wOUp+0EXJ/b7XHgImBY1ev/OeAu0r54aeX1z9M/n+NbBnw8r2vnBtryDaT9bnBhWUNze/wpr+u3uazea3gQ6cvVk6T3xK493G9PyNu8GPhwYd66cefp/1HY5o8VtzlPPxE4r92fjevsI+0OYEN7AJvknfnfgI2BQ0gfEuskKGAX0gdR5cN5PLBTHv4yOXkUlj0/76C7A4Pz8uezdoJaU1j3YflNVflgXEz9BFV5wxXfnEeSExQwIr+pPprXfXge37oQ24PAG/MbeD5wWp022ie/CffIb7zvVN7gteKsMf9i4B5SohlBSorF9l0DnJ6XPbSqzQcBdwJnApsDmwLvydMOJiXNXfM2fhH4fZ0YKu11cV7OW0gJb988/TPATcDYHMcPgYur5r0wzzuU9OH4TG7XjYGtgbfl+t8iJfERwOuAq4CvVW3vyXm+A4HngeF5+vmVtinEfiiwHSmJHgY8B4zO044hfYCOBYYDv2LtBPXLvC2bA9sAtwCfrNNG/wHcTdrPBbyV1/aXAP4HGJa3vROYkqftDHwgt9so4DfAt6pe/1vyNowgfaE4Jk+bQvpisjuwGekLSjFBddWWHwTurdqG75H25TGkfeddOa5ar+Ebc1t+IL8WnyftT5v0YL89Iy//fXlZuzQQ9xRS0npzjuVnrJug/h64rd2fj+vsI+0OYEN7AH9D+hajQtnvqZ2gdiZ9690X2LhqOV+mdoI6uUZZMUFVr/sW4KN5eDG9T1AfBW6pWveNvPYtfz7wxcK0TwH/W6eNzgW+XhjfgpTEx9eKs8b8i8kfSHn8QODBQvu+yNrfqItt/k7Sh+HgGsu9Fji6ML4R6cN++xp1K+31pkLZ14Fz8/D9rH30Ojpv4+DCvDsWph8PXFFjPSJ9UO1UKHsn8HBh26q/9a8AJuXh86lKUDXWcQcwNQ9fTyHhkPbNyHFvC6xm7W/uhwM31FnuA5Xl1pgW5C8GefwyYGadugcDt1e9/h+pavcf5OFZ5A/uwnss8nN3bflh4Kaq1/8vwFu7eP2Lr+GXgMuq5v8zsHeD++0aYPOqNvlSA3HPovBlkJQoqxPUB4CHutoP2vEo7XnR9dh2wJ8j7xXZn2pVjIgOSZ8hJYrdJc0B/j0ilnWx/CXdrL/WurfrPuxubce62/En0jfLiuKF5edJiafesm6rjETEs5KeyMta3GA8xXao3sbOiHihznzjgD9FxJoa07YHvi3pm4Uy5bhqvoY14nhLYVlXSHqlMP1l0od8rXnHkY5Aq40iHQncKqkY06BCnSeqtqertkfSEcC/kz5kyXVH5uHtquIqDm9POjJYXohlI+rvk/W2qaLm/iJpG+As4L2ko4WNSEfrXc1bef23AxbWib+7tlyV11cxknSE3dU2FJe/1nskIl6RtIS13yNd7berIuK5GtO7i3s70unT4nzVXkc67Vgq6+tF9DJbDoxRYU8incKoKSJ+FhHvIb35g3Rqijxcc5Zu1l9r3ZWE9xxpR68odqXtbrnLcoxFbyB9Q+yptZYlaXPSKa2eLGtcVRzFpN7VtiwB3lDnovYS0tHDsMJjaET8vhdxLAEOqFrWphFR3MZinEtI116qPU76Fr97YTlbRUTdBFRlrbaQtD3wI9K1pa0jYhjptFNln1lOOr1Xa/uWkI6gRhZi2TIidq+z7nrb1J2v5bj/KiK2BD5SiK87XcXfXVveBexY2DceB17oZhuK7Vu9Xyuvv/iad7XfDs/vherp3cW9vMZyq+1KOrVdKk5QrXcj6VD9XyQNlvT3wF61KkraRdI+koaQ3gh/IX3LhnROeXwveuptk9e9saRDSTvmNXnaHcC0PG0i6fpYRSepc8KOdZZ7DfBGSf+Ut+swYDfSdYSe+hlwlKS35W3/T+DmiFjcg2UcJ2mspBGkC8uXNjjfLaQ39GmSNpe0qaR352k/AI6XtDuApK1yG3blS5I2y/McVYjjB8CpOSEgaZSkqV0s5yJgX0n/mNt3a0lvi4hXSAnlzHxkgaQxkvZvcHsfY+3XdHPSh2pnXtZRpGsXFZcB/5rXMYzU2QR4tTfYdcA3JW0paSNJO0l6X511/xj4qqQJudfpX0nauoGYX0fq1PGkpDGka1mNuoy0b+0qaTPg/xXi77ItI2IpqdPGXoX6s4AzJG0naZCkd+Z9tt66PyhpsqSNSR1XVpNO8Vd0t99+RdImkt4LfAj4eQP7wGXAkZJ2y9t8Uo3Y3kc6hV0qTlAtFhEvki5IHkk6ZXAYqcdYLUOA00jfkB4lJZcT8rSf5+cnJN1WY956bgYm5GWeChwSEU/kaV8ifRtcBXyFlCgqcT+f6/9O0pOSJlVt1xOkN8xnSb2rPg98KCIe70FslWXNy7H8gpQsdgKm9XAxPyN9WD6UHw391iciXgb+lnRN4hFSz6nD8rQrSEewl0h6mnRkcUA3i/w16UL4POAbEVH5YfO3SRe1r5P0DKnDxDu6iOsR0jWJz5J6Nt5B6lQAKUl0ADfluH5F6njQiHOB3fJr+suIuI/UK+1GUvJ6C+lifcWPSO16F3A76YvJGl774nQEqSNQpSfa5aTra7WcQfrwvA54OscytIGYv0LqQPMUcDX13z/riIhrSacHbyC12Y150ur83F1b/pB0vbXic6SOHgtIr8vp1PlcjYgHSEd73yG9//4W+Nv8mVDR1X77KKlNl5G+sBwTEX/oLu68zd8iXT/syM+vkjSa9GWyz78p7G+Vbptm6w1Ji0kdQ37VxhjGk7phb1znetZ6QdIBpA4I1ad3BwRJu5K+aAxp5HXKR0e3kzq4LO/nWBZTZ7+VtDepw9LY6mn9sN5vkjpjfL+/l91X7iRhZg2TNBR4P+lb/rak00VXtDWoHpL0d6Qjr81JRzxXNfolIiJWk4421hsR8dl2x1CPT/GZWU+IdIptFelI4n4K13EGiE+SrrE9SDo1eWx7w7F6fIrPzMxKyUdQZmZWSr4GlY0cOTLGjx/f7jDMzNZ7t9566+MRMaq7ek5Q2fjx41m4cGH3Fc3MrE8k1bvzylp8is/MzErJCcrMzErJCcrMzErJCcrMzErJCcrMzErJCcrMzErJCcrMzErJCcrMzErJCcrMzErJd5IwG8DGz7z61eHFp32wjZGY9T8fQZmZWSk5QZmZWSk5QZmZWSk5QZmZWSk5QZmZWSk5QZmZWSk5QZmZWSk5QZmZWSk5QZmZWSk5QZmZWSk5QZmZWSk5QZmZWSk1LUFJmiVphaR7akz7nKSQNDKPS9JZkjok3SVpj0Ld6ZIW5cf0Qvmeku7O85wlSbl8hKS5uf5cScObtY1mZtY8zTyCOh+YUl0oaRzwAeCRQvEBwIT8mAGcneuOAE4C3gHsBZxUSDhn57qV+SrrmgnMi4gJwLw8bmZmA0zTElRE/AZYWWPSmcDngSiUTQUujOQmYJik0cD+wNyIWBkRq4C5wJQ8bcuIuDEiArgQOLiwrAvy8AWFcjMzG0Baeg1K0kHAnyPizqpJY4AlhfGluayr8qU1ygG2jYjlAPl5my7imSFpoaSFnZ2dvdgiMzNrlpYlKEmbAScC/6/W5Bpl0YvyHomIcyJiYkRMHDVqVE9nNzOzJmrlEdROwA7AnZIWA2OB2yS9nnQENK5QdyywrJvysTXKAR7LpwDJzyv6fUvMzKzpWpagIuLuiNgmIsZHxHhSktkjIh4FZgNH5N58k4Cn8um5OcB+kobnzhH7AXPytGckTcq9944Arsyrmg1UevtNL5SbmdkA0sxu5hcDNwK7SFoq6eguql8DPAR0AD8CPgUQESuBrwIL8uPkXAZwLPDjPM+DwLW5/DTgA5IWkXoLntaf22VmZq0xuFkLjojDu5k+vjAcwHF16s0CZtUoXwi8uUb5E8DkHoZrZmYl4ztJmJlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTUtQUmaJWmFpHsKZf8l6Q+S7pJ0haRhhWnHS+qQ9ICk/QvlU3JZh6SZhfIdJN0saZGkSyVtksuH5PGOPH18s7bRzMyap5lHUOcDU6rK5gJvjoi/Av4IHA8gaTdgGrB7nuf7kgZJGgR8DzgA2A04PNcFOB04MyImAKuAo3P50cCqiNgZODPXMzOzAaZpCSoifgOsrCq7LiLW5NGbgLF5eCpwSUSsjoiHgQ5gr/zoiIiHIuJF4BJgqiQB+wCX5/kvAA4uLOuCPHw5MDnXNzOzAaSd16A+Blybh8cASwrTluayeuVbA08Wkl2lfK1l5elP5fpmZjaAtCVBSToRWANcVCmqUS16Ud7VsmrFMUPSQkkLOzs7uw7azMxaquUJStJ04EPAhyOikjiWAuMK1cYCy7oofxwYJmlwVflay8rTt6LqVGNFRJwTERMjYuKoUaP6umlmZtaPWpqgJE0BvgAcFBHPFybNBqblHng7ABOAW4AFwITcY28TUkeK2Tmx3QAckuefDlxZWNb0PHwIcH0hEZqZ2QAxuPsqvSPpYmBvYKSkpcBJpF57Q4C5ud/CTRFxTETcK+ky4D7Sqb/jIuLlvJxPA3OAQcCsiLg3r+ILwCWSTgFuB87N5ecCP5HUQTpymtasbTQzs+ZpWoKKiMNrFJ9bo6xS/1Tg1Brl1wDX1Ch/iNTLr7r8BeDQHgVrZmal07QEZWbNMX7m1e0OwawlfKsjMzMrJScoMzMrJScoMzMrJScoMzMrJScoMzMrJScoMzMrJScoMzMrJScoMzMrJScoMzMrJScoMzMrJScoMzMrJScoMzMrJScoMzMrpYYSlKTtJe2bh4dKel1zwzIzsw1dtwlK0ieAy4Ef5qKxwC+bGZSZmVkjR1DHAe8GngaIiEXANs0MyszMrJEEtToiXqyMSBoMRPNCMjMzayxB/VrSCcBQSR8Afg5c1dywzMxsQ9dIgpoJdAJ3A58ErgG+2N1MkmZJWiHpnkLZCElzJS3Kz8NzuSSdJalD0l2S9ijMMz3XXyRpeqF8T0l353nOkqSu1mFmZgNLIwlqKDArIg6NiEOAWbmsO+cDU6rKZgLzImICMC+PAxwATMiPGcDZkJINcBLwDmAv4KRCwjk7163MN6WbdZiZ2QDSSIKax9oJaSjwq+5miojfACuriqcCF+ThC4CDC+UXRnITMEzSaGB/YG5ErIyIVcBcYEqetmVE3BgRAVxYtaxa6zAzswGkkQS1aUQ8WxnJw5v1cn3bRsTyvJzlvNYbcAywpFBvaS7rqnxpjfKu1rEOSTMkLZS0sLOzs5ebZGZmzdBIgnqu6prQnsBf+jkO1SiLXpT3SEScExETI2LiqFGjejq7mZk10eAG6nwG+LmkZXl8NHBYL9f3mKTREbE8n6ZbkcuXAuMK9cYCy3L53lXl83P52Br1u1qHmZkNIN0eQUXEAuBNwLHAp4BdI+LWXq5vNlDpiTcduLJQfkTuzTcJeCqfnpsD7CdpeO4csR8wJ097RtKk3HvviKpl1VqHmZkNII0cQQH8NTA+13+7JCLiwq5mkHQx6ehnpKSlpN54pwGXSToaeAQ4NFe/BjgQ6ACeB44CiIiVkr4KLMj1To6ISseLY0k9BYcC1+YHXazDzMwGkG4TlKSfADsBdwAv5+JKz7m6IuLwOpMm16gbpFsq1VrOLFLX9uryhcCba5Q/UWsdZmY2sDRyBDUR2C0nETMzs5ZopBffPcDrmx2ImZlZUSNHUCOB+yTdAqyuFEbEQU2LyszMNniNJKgvNzsIMzOzat0mqIj4dSsCMTMzK2rkH3UnSVog6VlJL0p6WdLTrQjOzMw2XI10kvgucDiwiPSbo4/nMjMzs6Zp6Ie6EdEhaVBEvAycJ+n3TY7LzMw2cI0kqOclbQLcIenrwHJg8+aGZWZmG7pGTvF9NNf7NPAc6aau/9DMoMzMzLo8gpI0CDg1Ij4CvAB8pSVRmZnZBq/LI6h8zWlUPsVnZmbWMo1cg1oM/E7SbNIpPgAi4oxmBWVmZtZIglqWHxsBr2tuOGZmZkkjd5LwdSczM2u5Rv4P6gbS/z+tJSL2aUpEZmZmNHaK73OF4U1JXczXNCccMzOzpJFTfLdWFf1Okm8ga2ZmTdXIzWJHFB4jJe1PH//AUNK/SbpX0j2SLpa0qaQdJN0saZGkSytd2yUNyeMdefr4wnKOz+UP5Lgq5VNyWYekmX2J1czM2qORO0ncCizMzzcCnwWO7u0KJY0B/gWYGBFvBgYB04DTgTMjYgKwqrCOo4FVEbEzcGauh6Td8ny7A1OA70salH9c/D3gAGA34PBc18zMBpBGTvHt0KT1DpX0ErAZ6f5++wD/lKdfQPqjxLOBqbz2p4mXA9+VpFx+SUSsBh6W1AHslet1RMRDAJIuyXXva8J2mJlZkzRyiu84ScMK48Mlfaq3K4yIPwPfAB4hJaanSEdnT0ZEpfPFUmBMHh4DLMnzrsn1ty6WV81Tr9zMzAaQRk7xfSIinqyMRMQq4BO9XaGk4aQjmh2A7Uh3Rj+gRtVK13bVmdbT8lqxzJC0UNLCzs7O7kI3M7MWaiRBbZRPqQGv3kC2L/fm2xd4OCI6I+Il4L+BdwHDJFVOOY4l3b0C0hHQuLzuwcBWwMpiedU89crXERHnRMTEiJg4atSoPmySmZn1t0YS1BzgMkmTJe0DXAz8bx/W+QgwSdJmOfFNJl0fugE4JNeZDlyZh2fncfL06yMicvm03MtvB2ACcAuwAJiQewVuQupIMbsP8ZqZWRs08kPdLwAzgGNJp8+uA37c2xVGxM2SLgduI/3g93bgHOBq4BJJp+Syc/Ms5wI/yZ0gVpISDhFxr6TLSMltDXBcvvs6kj5NSqyDgFkRcW9v4zUzs/ZQOhjpooK0OfBC4cN/EDAkIp5vQXwtM3HixFi4cGG7wzDr1viZV9csX3zaB1sciVnvSLo1IiZ2V6+RU3zzgKGF8aHAr3obmJmZWSMaSVCbRsSzlZE8vFnzQjIzM2ssQT0naY/KiKQ9gb80LyQzM7PGOkl8Bvi5pEpX7dHAYc0LyczMrLFbHS2Q9CZgF1Ivvj/k3y+ZmZk1TSN/WLgxqYv53+Si+ZJ+6CRlZmbN1MgpvrOBjYHv5/GP5rKPNysoMzOzRhLUX0fEWwvj10u6s1kBmZmZQWO9+F6WtFNlRNKOwMvNC8nMzKyxI6j/AG6Q9BCpk8T2wFFNjcrMzDZ4jfTimydpAmv34lvd9MjMzGyD1sgRFDkh3dXkWMzMzF7VyDUoMzOzlquboCS9Oz8PaV04ZmZmSVdHUGfl5xtbEYiZmVlRV9egXpJ0HjBG0lnVEyPiX5oXlpmZbei6SlAfAvYF9gFubU04ZmZmSd0EFRGPk/6C/f6I8J0jzMyspRrpxfeEpCskrZD0mKRfSBrb9MjMzGyD1kiCOg+YDWwHjAGuymW9JmmYpMsl/UHS/ZLeKWmEpLmSFuXn4bmuJJ0lqUPSXVV/njg9118kaXqhfE9Jd+d5zpKkvsRrZmat10iC2iYizouINflxPjCqj+v9NvC/EfEm4K3A/cBMYF5ETADm5XGAA4AJ+TGDdCd1JI0ATgLeAewFnFRJarnOjMJ8U/oYr5mZtVgjCapT0kckDcqPjwBP9HaFkrYk/bfUuQAR8WJEPAlMBS7I1S4ADs7DU4ELI7kJGCZpNLA/MDciVkbEKmAuMCVP2zIiboyIAC4sLMvMzAaIRhLUx4B/BB4FlgOH5LLe2hHoBM6TdLukH0vaHNg2IpYD5Odtcv0xwJLC/EtzWVflS2uUr0PSDEkLJS3s7OzswyaZmVl/6zZBRcQjEXFQRIyKiG0i4uCI+FMf1jkY2AM4OyLeDjzHa6fzaql1/Sh6Ub5uYcQ5ETExIiaOGtXXs5ZmZtaf2nEvvqXA0oi4OY9fTkpYj+XTc+TnFYX64wrzjwWWdVM+tka5mZkNIC1PUBHxKLBE0i65aDJwH6mnYKUn3nTgyjw8Gzgi9+abBDyVTwHOAfaTNDx3jtgPmJOnPSNpUu69d0RhWWZmNkA09HcbTfDPwEWSNgEeIv0B4kbAZZKOBh4BDs11rwEOBDqA53NdImKlpK8CC3K9kyNiZR4+FjgfGApcmx9mZjaAdJugJH0xIk7Jw0P6488KI+IOYGKNSZNr1A3guDrLmQXMqlG+EHhzH8M0M7M26urvNj4v6Z2kXnsVvrO5mZm1RFdHUA+QTrPtKOn/SD+m3VrSLhHxQEuiMzOzDVZXnSRWASeQrv3szWv/DzVT0u+bHJeZmW3gujqCmkK6ldBOwBnAncBzEXFUKwIzM7MNW90jqIg4ISImA4uBn5KS2ShJv5V0VYviMzOzDVQj3cznRMQCYIGkYyPiPZJGNjswMzPbsDVyq6PPF0aPzGWPNysgMzMz6OEPdf3PumblNX7m1a8OLz7tg22MxKx/tONefGZmZt1ygjIzs1JygjIzs1JygjIzs1JygjIzs1JygjIzs1JygjIzs1JygjIzs1JygjIzs1JygjIzs1Lq0a2OzKw9ircxMttQtO0IStIgSbdL+p88voOkmyUtknSppE1y+ZA83pGnjy8s4/hc/oCk/QvlU3JZh6SZrd42MzPru3ae4vtX0t/IV5wOnBkRE0j/5nt0Lj8aWBUROwNn5npI2g2YBuxO+nPF7+ekNwj4HnAAsBtweK5rZmYDSFsSlKSxwAeBH+dxAfsAl+cqFwAH5+GpeZw8fXKuPxW4JCJWR8TDpL+m3ys/OiLioYh4Ebgk1zUzswGkXUdQ3wI+D7ySx7cGnoyINXl8KTAmD48BlgDk6U/l+q+WV81Tr3wdkmZIWihpYWdnZ1+3yczM+lHLE5SkDwErIuLWYnGNqtHNtJ6Wr1sYcU5ETIyIiaNGjeoiajMza7V29OJ7N3CQpAOBTYEtSUdUwyQNzkdJY4Fluf5SYBywVNJgYCtgZaG8ojhPvXIzMxsgWn4EFRHHR8TYiBhP6uRwfUR8GLgBOCRXmw5cmYdn53Hy9OsjInL5tNzLbwdgAnALsACYkHsFbpLXMbsFm2ZmZv2oTL+D+gJwiaRTgNuBc3P5ucBPJHWQjpymAUTEvZIuA+4D1gDHRcTLAJI+DcwBBgGzIuLelm6JmZn1WVsTVETMB+bn4YdIPfCq67wAHFpn/lOBU2uUXwNc04+hmplZi/lWR2ZmVkplOsVnZv2k+tZIi0/7YJsiMes9H0GZmVkpOUGZmVkpOUGZmVkpOUGZmVkpOUGZmVkpOUGZmVkpuZu5WUn5X3RtQ+cjKDMzKyUnKDMzKyUnKDMzKyUnKDMzKyUnKDMzKyUnKDMzKyV3MzfbABS7rPvO5jZQ+AjKzMxKyQnKzMxKqeUJStI4STdIul/SvZL+NZePkDRX0qL8PDyXS9JZkjok3SVpj8Kypuf6iyRNL5TvKenuPM9ZktTq7TQzs75pxxHUGuCzEbErMAk4TtJuwExgXkRMAOblcYADgAn5MQM4G1JCA04C3gHsBZxUSWq5zozCfFNasF1mZtaPWt5JIiKWA8vz8DOS7gfGAFOBvXO1C4D5wBdy+YUREcBNkoZJGp3rzo2IlQCS5gJTJM0HtoyIG3P5hcDBwLWt2D6z3vK998zW1tZrUJLGA28Hbga2zcmrksS2ydXGAEsKsy3NZV2VL61RXmv9MyQtlLSws7Ozr5tjZmb9qG0JStIWwC+Az0TE011VrVEWvShftzDinIiYGBETR40a1V3IZmbWQm1JUJI2JiWniyLiv3PxY/nUHfl5RS5fCowrzD4WWNZN+dga5WZmNoC0oxefgHOB+yPijMKk2UClJ9504MpC+RG5N98k4Kl8CnAOsJ+k4blzxH7AnDztGUmT8rqOKCzLbIM3fubVrz7Myqwdd5J4N/BR4G5Jd+SyE4DTgMskHQ08Ahyap10DHAh0AM8DRwFExEpJXwUW5HonVzpMAMcC5wNDSZ0j3EHCzGyAaUcvvt9S+zoRwOQa9QM4rs6yZgGzapQGjsq3AAAJbklEQVQvBN7chzDNzKzNfC8+szbyaTaz+nyrIzMzKyUnKDMzKyWf4jPbgPlvOKzMfARlZmal5CMosxZzxwizxvgIyszMSslHUGYG+HqUlY8TlFkL+LSeWc/5FJ+ZmZWSj6DMbB0+3Wdl4ARl1iQ+rWfWN05QZv1ofUxK1dvkIyprFV+DMjOzUvIRlFkfrI9HTN3x9SlrFScosx7aEJNSPU5W1kxOUGYNcFLqnpOV9TcnKLM6nJR6z8nK+oMTlG3wnIiaq177OnFZd9bbBCVpCvBtYBDw44g4rc0hWYs58ZRbV6+Pk5fBepqgJA0Cvgd8AFgKLJA0OyLua29k1ltONhuWnr7eTmjrp/UyQQF7AR0R8RCApEuAqYATVFbvGoETgQ1EA2W/dSLtmfU1QY0BlhTGlwLvqK4kaQYwI48+K+mBPqxzJPB4H+Zvpi5j0+ktjKS2Adt2JeD4eq/lsfXwvbY+t932jVRaXxOUapTFOgUR5wDn9MsKpYURMbE/ltXfyhwblDu+MscGjq8vyhwblDu+VsW2vt7qaCkwrjA+FljWpljMzKwX1tcEtQCYIGkHSZsA04DZbY7JzMx6YL08xRcRayR9GphD6mY+KyLubfJq++VUYZOUOTYod3xljg0cX1+UOTYod3wtiU0R61yaMTMza7v19RSfmZkNcE5QZmZWSk5QPSDpUEn3SnpFUt0ulpKmSHpAUoekmYXyHSTdLGmRpEtzB47+im2EpLl52XMlDa9R5/2S7ig8XpB0cJ52vqSHC9Pe1l+xNRpfrvdyIYbZhfJ2t93bJN2YX/+7JB1WmNaUtqu3HxWmD8lt0ZHbZnxh2vG5/AFJ+/dHPD2M7d8l3Zfbap6k7QvTar7GLY7vSEmdhTg+Xpg2Pe8LiyRNb0NsZxbi+qOkJwvTmtp2kmZJWiHpnjrTJemsHPtdkvYoTOv/dosIPxp8ALsCuwDzgYl16gwCHgR2BDYB7gR2y9MuA6bl4R8Ax/ZjbF8HZubhmcDp3dQfAawENsvj5wOHNLHtGooPeLZOeVvbDngjMCEPbwcsB4Y1q+262o8KdT4F/CAPTwMuzcO75fpDgB3ycga1OLb3F/atYyuxdfUatzi+I4Hv1ph3BPBQfh6eh4e3Mraq+v9M6uTVqrb7G2AP4J460w8EriX91nQScHMz281HUD0QEfdHRHd3m3j1NksR8SJwCTBVkoB9gMtzvQuAg/sxvKl5mY0u+xDg2oh4vh9j6EpP43tVGdouIv4YEYvy8DJgBTCqH2OoVnM/qqpTjPtyYHJuq6nAJRGxOiIeBjry8loWW0TcUNi3biL9FrFVGmm7evYH5kbEyohYBcwFprQxtsOBi/tx/V2KiN+QvrjWMxW4MJKbgGGSRtOkdnOC6n+1brM0BtgaeDIi1lSV95dtI2I5QH7eppv601h3xz81H7afKWlIP8bWk/g2lbRQ0k2V04+UrO0k7UX69vtgobi/267eflSzTm6bp0ht1ci8zY6t6GjSt+6KWq9xf2o0vn/Ir9nlkio/7C9N2+XTojsA1xeKm9123akXf1Pabb38HVRfSPoV8Poak06MiCsbWUSNsuiivF9i6+FyRgNvIf1OrOJ44FHSB+85wBeAk9sQ3xsiYpmkHYHrJd0NPF2jXjvb7ifA9Ih4JRf3ue1qrapGWfU2N21f60bDy5f0EWAi8L5C8TqvcUQ8WGv+JsZ3FXBxRKyWdAzpSHSfBudtdmwV04DLI+LlQlmz2647Ld3nnKCqRMS+fVxEvdssPU46HB6cv+32+PZLXcUm6TFJoyNief4QXdHFov4RuCIiXiose3keXC3pPOBzPYmtv+LLp8+IiIckzQfeDvyCErSdpC2Bq4Ev5tMblWX3ue1qaOR2XZU6SyUNBrYinZ5p9q2+Glq+pH1JXwDeFxGrK+V1XuP+/JDtNr6IeKIw+iOgchvXpcDeVfPOb2VsBdOA44oFLWi77tSLvynt5lN8/a/mbZYiXUm8gXTtB2A60MgRWaNm52U2sux1zmvnD+bK9Z6DgZq9eJoZn6ThldNjkkYC7wbuK0Pb5dfyCtL5959XTWtG2zVyu65i3IcA1+e2mg1MU+rltwMwAbilH2JqODZJbwd+CBwUESsK5TVf436MrdH4RhdGDwLuz8NzgP1ynMOB/Vj7TEPTY8vx7ULqbHBjoawVbded2cARuTffJOCp/AWtOe3WzB4h69sD+DvSN4XVwGPAnFy+HXBNod6BwB9J32xOLJTvSPqg6AB+Dgzpx9i2BuYBi/LziFw+kfSPwpV644E/AxtVzX89cDfpw/WnwBb93Hbdxge8K8dwZ34+uixtB3wEeAm4o/B4WzPbrtZ+RDp1eFAe3jS3RUdumx0L856Y53sAOKAJ74XuYvtVfo9U2mp2d69xi+P7GnBvjuMG4E2FeT+W27QDOKrVseXxLwOnVc3X9LYjfXFdnvf1paTrh8cAx+TpIv0Z7IM5homFefu93XyrIzMzKyWf4jMzs1JygjIzs1JygjIzs1JygjIzs1JygjIzs1JygjJrI0kn6rU7pN8h6R1NWMcJ/b1Ms1ZwN3OzNpH0TuAMYO9It9wZCWwS+W4B/bB8kX638nREbNEfyzRrJR9BmbXPaODxyLcBiojHI91nbbGk/1T6/6mFkvaQNEfSg/m+cUjaQul/lm6TdLekqbl8vKT7JX0fuA04Fxiaj84ukrS5pKsl3SnpHhX+18qsbHwEZdYmkrYAfgtsRrrzwqUR8WtJi0n/SXW2pDOByaTb2mwK3BsR2+R7720WEU/nI6+bSLc02p70Xzzviny/QEnPVo6gJP0DMCUiPpHHt4qIp1q42WYN8xGUWZtExLPAnsAMoBO4VNKReXLl/mx3k/4U7pmI6ARekDSMdOruPyXdRUpuY4Bt8zx/isLNbKvcDewr6XRJ73VysjLz3czN2ijSXynMB+bnvxap3Py1cvfvVwrDlfHBwIdJf5i4Z0S8lI+6Ns11nutifX+UtCfpfnBfk3RdRPT1r0HMmsJHUGZtImkXSRMKRW8D/tTg7FsBK3Jyej/p1F49L0naOK9zO+D5iPgp8A3S33ublZKPoMzaZwvgO/mU3RrSXaBnAB9qYN6LgKskLSTdLfwPXdQ9B7hL0m3AhcB/SXqFdMfqY/sQv1lTuZOEmZmVkk/xmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKTlBmZlZKf1/+ZnFQfBfg4AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f08fcd59780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "\n",
    "\n",
    "\n",
    "# add a 'best fit' line\n",
    "y = np.array(y_distribution)\n",
    "y = y[y < 1]\n",
    "y =  y[y > -1]\n",
    "\n",
    "# the histogram of the data\n",
    "n, bins, patches = ax.hist(y, 100)\n",
    "\n",
    "ax.set_xlabel('Smarts')\n",
    "ax.set_ylabel('# of occurance')\n",
    "ax.set_title(r'distribution of price percentage change(cropped)')\n",
    "\n",
    "# Tweak spacing to prevent clipping of ylabel\n",
    "fig.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def normalize_windows(window_data):\n",
    "    window_data = (window_data / window_data[0]) - 1 \n",
    "    return window_data\n",
    "\n",
    "def switch(delta,train_Y,buckets):# separate y values according to bucket separating values and put Y in corresponding bucket\n",
    "    temp = [0]*len(buckets)\n",
    "    if delta == 0:\n",
    "        temp[len(buckets)//2] = 1\n",
    "\n",
    "    else:    \n",
    "        for i in range(len(buckets)-1):\n",
    "            if delta >= buckets[i] and delta < buckets[i+1]:\n",
    "                # because ==0 itself is a bucket, all >0 buckets will need index+1\n",
    "                if delta > 0:\n",
    "                    temp[i+1] = 1\n",
    "                if delta < 0:\n",
    "                    temp[i] = 1\n",
    "                \n",
    "    train_Y.append(temp)\n",
    "    return train_Y\n",
    "\n",
    "\n",
    "# Recursively get bucket separating values\n",
    "def bucket_separate(nBuckets,bucket_list,y_distribution):\n",
    "    mid = len(y_distribution)//2 # median index of input y_distribution list\n",
    "    if nBuckets % 2 == 1:\n",
    "        print(\"number of buckets must be (2^n+1) greater than 3\")\n",
    "\n",
    "    elif nBuckets == 2:\n",
    "        bucket_list.append(y_distribution[mid])\n",
    "\n",
    "    else:\n",
    "        bucket_list.append(y_distribution[mid])\n",
    "        bucket_list = bucket_separate(nBuckets/2,bucket_list,y_distribution[:mid])#left\n",
    "        bucket_list = bucket_separate(nBuckets/2,bucket_list,y_distribution[mid:])#right\n",
    "    bucket_list = sorted(bucket_list)\n",
    "\n",
    "    return bucket_list\n",
    "\n",
    "def lstm_preprocess(data, close_price, time, bucket_list,test_pct):\n",
    "    window_length = 60\n",
    "    A = np.array(data.low)\n",
    "    B = np.array(data.high)\n",
    "    C = np.array(data.open)\n",
    "    D = np.array(data.close)\n",
    "    E = np.array(data.volume)\n",
    "    i = 0\n",
    "    train_X = []\n",
    "    train_Y = []\n",
    "    test_X = []\n",
    "    test_Y = []\n",
    "    true_Y_test = []\n",
    "    de_norm_train = []\n",
    "    de_norm_test = []\n",
    "    error = 0\n",
    "    \n",
    "\n",
    "    \n",
    "    while i < len(C) - window_length:\n",
    "         \n",
    "        if time[i+window_length] - time[i] == window_length*60:\n",
    "            temp = []\n",
    "            copy_A = np.copy(A[i:i+window_length+1])\n",
    "            copy_B = np.copy(B[i:i+window_length+1])\n",
    "            copy_C = np.copy(C[i:i+window_length+1])\n",
    "            copy_D = np.copy(D[i:i+window_length+1])\n",
    "            copy_E = np.copy(E[i:i+window_length+1])\n",
    "             \n",
    "            normalized_temp_A = normalize_windows(copy_A)\n",
    "            normalized_temp_B = normalize_windows(copy_B)\n",
    "            normalized_temp_C = normalize_windows(copy_C)\n",
    "            normalized_temp_D = normalize_windows(copy_D)\n",
    "            normalized_temp_E = normalize_windows(copy_E)\n",
    "            #print(len(normalized_temp_D[:-1]))\n",
    "            temp.append(normalized_temp_A[:-1])\n",
    "            temp.append(normalized_temp_B[:-1])\n",
    "            temp.append(normalized_temp_C[:-1])\n",
    "            temp.append(normalized_temp_D[:-1])\n",
    "            temp.append(normalized_temp_E[:-1])\n",
    "#         temp.append(year[i:i+60])\n",
    "#         temp.append(hour[i:i+60])\n",
    "            \n",
    "            delta = (close_price[i + 60] - close_price[i + 59]) / close_price[i + 59] * 100\n",
    "            \n",
    "            # percentage of test\n",
    "            if test_pct == 0:\n",
    "                test = Inf\n",
    "            else:\n",
    "                test = (1/test_pct)\n",
    "            if i%test == 0 :\n",
    "                test_X.append(temp)\n",
    "                true_Y_test.append(close_price[i + 60])\n",
    "                # like switch command in C, decide which bucket we should label.\n",
    "                test_Y = switch(delta,test_Y,bucket_list)\n",
    "                de_norm_test.append([copy_A[0], copy_B[0], copy_C[0], copy_D[0], copy_E[0]])\n",
    "                \n",
    "            else:\n",
    "                train_X.append(temp)                       \n",
    "                # like switch command in C, decide which bucket we should label.\n",
    "                train_Y = switch(delta,train_Y,bucket_list)\n",
    "                de_norm_train.append([copy_A[0], copy_B[0], copy_C[0], copy_D[0], copy_E[0]])\n",
    "            i += 1\n",
    "        else:\n",
    "            i = i + 60\n",
    "    \n",
    "        \n",
    "    train_X = np.transpose(train_X, (0,2,1))   \n",
    "    train_X = train_X.reshape(train_X.shape[0], train_X.shape[1], -1)\n",
    "    #print(train_X.shape)\n",
    "    train_Y = np.array(train_Y)\n",
    "    # transform testsets if they are not empty!\n",
    "    if test_pct != 0:\n",
    "        test_X = np.transpose(test_X, (0,2,1))\n",
    "        test_X = test_X.reshape(test_X.shape[0], test_X.shape[1], -1)\n",
    "        test_Y = np.array(test_Y)\n",
    "        de_norm_test = np.array(de_norm_test)\n",
    "        de_norm_test=de_norm_test[:,3]\n",
    "        de_norm_test.reshape(de_norm_test.shape[0], -1)\n",
    "        ##################\n",
    "        #fit_transofrm\n",
    "        for i in range(test_X.shape[2]):\n",
    "            ss = StandardScaler()\n",
    "            test_X[:,:,i] = ss.fit_transform(test_X[:,:,i])\n",
    "    ###############\n",
    "    #fit_transform\n",
    "    for i in range(train_X.shape[2]):\n",
    "        ss = StandardScaler()\n",
    "        train_X[:,:,i] = ss.fit_transform(train_X[:,:,i])\n",
    "        \n",
    "    \n",
    "    \n",
    "    return train_X, train_Y, test_X, test_Y, de_norm_train, de_norm_test, true_Y_test\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # test if bucket separation is working\n",
    "# nBuckets = 33\n",
    "\n",
    "# buckets = bucket_separate((nBuckets-1)/2,[-Inf,0,Inf],y_neg)\n",
    "# buckets = bucket_separate((nBuckets-1)/2,buckets,y_pos)\n",
    "\n",
    "# print(buckets)\n",
    "\n",
    "# train_X, train_Y, test_X, test_Y, de_norm_train, de_norm_test, true_Y_test = lstm_preprocess(data, close_price, time, buckets)\n",
    "# count = np.sum(train_Y,axis = 0)\n",
    "# print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mse_pred(y_true,y_pred, de_norm_test, de_norm_flag, baskets):\n",
    "    se = 0\n",
    "    # change -inf and inf into min and max, as boundaries.\n",
    "    baskets[0] = np.min(y_neg) \n",
    "    baskets[-1]= np.max(y_pos)\n",
    "    nBuckets = len(baskets)\n",
    "    \n",
    "    #categories is a vector containing the mid points of each bucket\n",
    "    categories = np.array(baskets) #=0\n",
    "    for i in range(nBuckets//2): #<0\n",
    "        categories[i] = (baskets[i]+baskets[i+1])/2\n",
    "    for i in range((nBuckets+1)//2,nBuckets): #>0\n",
    "        categories[i] = (baskets[i-1]+baskets[i])/2\n",
    "    \n",
    "    print(categories)\n",
    "    #de_norm_flag is to set whether or not to convert to original value     \n",
    "    if de_norm_flag:\n",
    "        pred = de_norm_test*(categories[np.argmax(y_pred,axis = 1)]+1)\n",
    "    else:\n",
    "        pred = categories[np.argmax(y_pred,axis = 1)]\n",
    "    \n",
    "    mse = np.sum(abs(pred-y_true)**2)\n",
    "    print(\"mean squared error equivalent in test set is\", mse)\n",
    "    return mse,pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_model(X,nBuckets):\n",
    "    model = Sequential()\n",
    "\n",
    "    model.add(LSTM(512,\n",
    "        input_shape=(X.shape[1],X.shape[2]),\n",
    "        return_sequences=True))\n",
    "    model.add(Dropout(0.5))\n",
    "    \n",
    "#     model.add(LSTM(512,\n",
    "#         return_sequences=True))\n",
    "#     model.add(Dropout(0.5))\n",
    "\n",
    "    model.add(LSTM(\n",
    "        512,\n",
    "        return_sequences=False))\n",
    "    model.add(Dropout(0.5))\n",
    "\n",
    "    model.add(Dense(nBuckets, activation = 'softmax')) ############ change dense layer node number to number of baskets.\n",
    "    #Adm = keras.optimizers.Adam(lr = 0.001)\n",
    "    #model.compile(loss=\"categorical_crossentropy\", optimizer='Adam', metrics=['categorical_accuracy'])\n",
    "    model.compile(loss=\"categorical_crossentropy\", optimizer='sgd', metrics=['accuracy'])\n",
    "    # K.tensorflow_backend._get_available_gpus()\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Test switch function\n",
    "# nBuckets = 17\n",
    "# bucket_list = bucket_separate((nBuckets-1)/2,[-Inf,0,Inf],y_neg)\n",
    "# bucket_list = bucket_separate((nBuckets-1)/2,bucket_list,y_pos)\n",
    "# print(bucket_list)\n",
    "# #train_X, train_Y, test_X, test_Y, de_norm_train, de_norm_test, true_Y_test = lstm_preprocess(data, close_price, time, nBucket)\n",
    "# y = []\n",
    "# y = switch(-0.015,[],bucket_list)\n",
    "# print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "n is  1\n",
      "nBuckets: 3\n",
      "Train on 560376 samples, validate on 140095 samples\n",
      "Epoch 1/4\n",
      "560376/560376 [==============================] - 779s 1ms/step - loss: 1.0465 - acc: 0.4257 - val_loss: 1.0062 - val_acc: 0.4401\n",
      "Epoch 2/4\n",
      "560376/560376 [==============================] - 756s 1ms/step - loss: 1.0419 - acc: 0.4268 - val_loss: 1.0024 - val_acc: 0.4409\n",
      "Epoch 3/4\n",
      "560376/560376 [==============================] - 756s 1ms/step - loss: 1.0406 - acc: 0.4275 - val_loss: 0.9997 - val_acc: 0.4416\n",
      "Epoch 4/4\n",
      "560376/560376 [==============================] - 756s 1ms/step - loss: 1.0399 - acc: 0.4272 - val_loss: 0.9983 - val_acc: 0.4408\n",
      "77831/77831 [==============================] - 171s 2ms/step\n",
      "[1.0290596684634412, 0.4324626434201]\n",
      "[-5.71428571  0.          4.25063685]\n",
      "mean squared error equivalent in test set is 58008323492572.95\n",
      "n is  2\n",
      "nBuckets: 5\n",
      "Train on 560376 samples, validate on 140095 samples\n",
      "Epoch 1/4\n",
      "560376/560376 [==============================] - 758s 1ms/step - loss: 1.6006 - acc: 0.2408 - val_loss: 1.5926 - val_acc: 0.2155\n",
      "Epoch 2/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 1.5881 - acc: 0.2502 - val_loss: 1.5628 - val_acc: 0.2422\n",
      "Epoch 3/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 1.5825 - acc: 0.2541 - val_loss: 1.5565 - val_acc: 0.2496\n",
      "Epoch 4/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 1.5812 - acc: 0.2556 - val_loss: 1.5535 - val_acc: 0.2544\n",
      "77831/77831 [==============================] - 172s 2ms/step\n",
      "[1.573493085045257, 0.2591126928870168]\n",
      "[-5.73129616 -0.01701045  0.          0.01631586  4.26695272]\n",
      "mean squared error equivalent in test set is 28707045149063.89\n",
      "n is  3\n",
      "nBuckets: 9\n",
      "Train on 560376 samples, validate on 140095 samples\n",
      "Epoch 1/4\n",
      "560376/560376 [==============================] - 758s 1ms/step - loss: 2.1624 - acc: 0.1900 - val_loss: 2.1948 - val_acc: 0.1704\n",
      "Epoch 2/4\n",
      "560376/560376 [==============================] - 756s 1ms/step - loss: 2.1394 - acc: 0.1974 - val_loss: 2.1454 - val_acc: 0.1999\n",
      "Epoch 3/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 2.1298 - acc: 0.1998 - val_loss: 2.1364 - val_acc: 0.2036\n",
      "Epoch 4/4\n",
      "560376/560376 [==============================] - 756s 1ms/step - loss: 2.1273 - acc: 0.1999 - val_loss: 2.1345 - val_acc: 0.2055\n",
      "77831/77831 [==============================] - 171s 2ms/step\n",
      "[2.1264238549550685, 0.19939355783723456]\n",
      "[-5.76518454e+00 -6.79092756e-02 -1.99185007e-02 -2.90805205e-03\n",
      "  0.00000000e+00  3.26594598e-03  1.95818097e-02  6.24028202e-02\n",
      "  4.29672381e+00]\n",
      "mean squared error equivalent in test set is 19969931414143.17\n",
      "n is  4\n",
      "nBuckets: 17\n",
      "Train on 560376 samples, validate on 140095 samples\n",
      "Epoch 1/4\n",
      "560376/560376 [==============================] - 758s 1ms/step - loss: 2.7338 - acc: 0.1871 - val_loss: 2.8044 - val_acc: 0.1572\n",
      "Epoch 2/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 2.7006 - acc: 0.1917 - val_loss: 2.7492 - val_acc: 0.1734\n",
      "Epoch 3/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 2.6855 - acc: 0.1922 - val_loss: 2.7217 - val_acc: 0.1753\n",
      "Epoch 4/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 2.6793 - acc: 0.1919 - val_loss: 2.7103 - val_acc: 0.1757\n",
      "77831/77831 [==============================] - 172s 2ms/step\n",
      "[2.682236594289717, 0.18717477611863406]\n",
      "[-5.81045945e+00 -1.47072560e-01 -8.03291251e-02 -4.64407467e-02\n",
      " -2.54412296e-02 -1.13388331e-02 -3.58887066e-03 -6.80818616e-04\n",
      "  0.00000000e+00  7.20780461e-04  3.98672644e-03  1.17645218e-02\n",
      "  2.48144395e-02  4.37145403e-02  7.34856331e-02  1.31548894e-01\n",
      "  4.33609879e+00]\n",
      "mean squared error equivalent in test set is 12765332727676.28\n",
      "n is  5\n",
      "nBuckets: 33\n",
      "Train on 560376 samples, validate on 140095 samples\n",
      "Epoch 1/4\n",
      "560376/560376 [==============================] - 759s 1ms/step - loss: 3.3131 - acc: 0.1857 - val_loss: 3.4399 - val_acc: 0.1503\n",
      "Epoch 2/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 3.2713 - acc: 0.1889 - val_loss: 3.4148 - val_acc: 0.1578\n",
      "Epoch 3/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 3.2565 - acc: 0.1894 - val_loss: 3.3762 - val_acc: 0.1597\n",
      "Epoch 4/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 3.2452 - acc: 0.1891 - val_loss: 3.3524 - val_acc: 0.1602\n",
      "77831/77831 [==============================] - 172s 2ms/step\n",
      "[3.2624871027994837, 0.18157289512006022]\n",
      "[-5.86398631e+00 -2.45874332e-01 -1.64620144e-01 -1.19345237e-01\n",
      " -8.93751641e-02 -6.79066351e-02 -5.19241122e-02 -3.95042628e-02\n",
      " -2.93344212e-02 -2.07547536e-02 -1.37205371e-02 -8.19780811e-03\n",
      " -4.23118841e-03 -2.00395498e-03 -8.26865848e-04 -1.46047232e-04\n",
      "  0.00000000e+00  1.50592884e-04  8.71373345e-04  2.24065279e-03\n",
      "  4.78581831e-03  8.79204156e-03  1.40246714e-02  2.05297606e-02\n",
      "  2.83470486e-02  3.75759428e-02  4.86587556e-02  6.27406098e-02\n",
      "  8.14288897e-02  1.07511765e-01  1.46886745e-01  2.19220123e-01\n",
      "  4.38439504e+00]\n",
      "mean squared error equivalent in test set is 9097672024526.336\n",
      "n is  6\n",
      "nBuckets: 65\n",
      "Train on 560376 samples, validate on 140095 samples\n",
      "Epoch 1/4\n",
      "560376/560376 [==============================] - 759s 1ms/step - loss: 3.9038 - acc: 0.1853 - val_loss: 4.0461 - val_acc: 0.1469\n",
      "Epoch 2/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 3.8380 - acc: 0.1879 - val_loss: 4.0700 - val_acc: 0.1506\n",
      "Epoch 3/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 3.8277 - acc: 0.1883 - val_loss: 4.0609 - val_acc: 0.1517\n",
      "Epoch 4/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 3.8165 - acc: 0.1883 - val_loss: 4.0305 - val_acc: 0.1523\n",
      "77831/77831 [==============================] - 172s 2ms/step\n",
      "[3.8520586032394215, 0.1792216469027689]\n",
      "[-5.92504129e+00 -3.60456171e-01 -2.66895038e-01 -2.13368172e-01\n",
      " -1.76794702e-01 -1.49067379e-01 -1.27232449e-01 -1.09684866e-01\n",
      " -9.51359039e-02 -8.27134139e-02 -7.21392689e-02 -6.30932299e-02\n",
      " -5.51167690e-02 -4.81802851e-02 -4.20900913e-02 -3.66067257e-02\n",
      " -3.15583089e-02 -2.68718328e-02 -2.26835656e-02 -1.87903740e-02\n",
      " -1.51659422e-02 -1.20249173e-02 -9.32244775e-03 -6.94074373e-03\n",
      " -4.97024787e-03 -3.38533219e-03 -2.19025856e-03 -1.54794081e-03\n",
      " -1.08090503e-03 -5.46133650e-04 -2.06845045e-04 -6.07978132e-05\n",
      "  0.00000000e+00  6.16431601e-05  2.12236044e-04  5.63321433e-04\n",
      "  1.13350901e-03  1.68327216e-03  2.48236403e-03  3.83500869e-03\n",
      "  5.58108234e-03  7.59191992e-03  9.85206951e-03  1.24434422e-02\n",
      "  1.54159224e-02  1.87141526e-02  2.22467617e-02  2.60737380e-02\n",
      "  3.03584169e-02  3.49795442e-02  3.99237595e-02  4.53735086e-02\n",
      "  5.15121062e-02  5.84470637e-02  6.63903203e-02  7.56530670e-02\n",
      "  8.63980903e-02  9.91993423e-02  1.14537194e-01  1.33534531e-01\n",
      "  1.57571659e-01  1.89838163e-01  2.38134412e-01  3.24752795e-01\n",
      "  4.44163146e+00]\n",
      "mean squared error equivalent in test set is 6467488758887.539\n",
      "n is  7\n",
      "nBuckets: 129\n",
      "Train on 560376 samples, validate on 140095 samples\n",
      "Epoch 1/4\n",
      "560376/560376 [==============================] - 759s 1ms/step - loss: 4.5002 - acc: 0.1851 - val_loss: 4.6115 - val_acc: 0.1458\n",
      "Epoch 2/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 4.4106 - acc: 0.1874 - val_loss: 4.6415 - val_acc: 0.1480\n",
      "Epoch 3/4\n",
      "560376/560376 [==============================] - 758s 1ms/step - loss: 4.4028 - acc: 0.1877 - val_loss: 4.6546 - val_acc: 0.1486\n",
      "Epoch 4/4\n",
      "560376/560376 [==============================] - 757s 1ms/step - loss: 4.3969 - acc: 0.1878 - val_loss: 4.6590 - val_acc: 0.1486\n",
      "77831/77831 [==============================] - 172s 2ms/step\n",
      "[4.4455854991828, 0.1779625085131812]\n",
      "[-5.99465955e+00 -4.91129404e-01 -3.84551868e-01 -3.23496895e-01\n",
      " -2.81323561e-01 -2.48817401e-01 -2.22897357e-01 -2.01876652e-01\n",
      " -1.84036804e-01 -1.68484039e-01 -1.54770669e-01 -1.42596111e-01\n",
      " -1.31702987e-01 -1.22042616e-01 -1.13472503e-01 -1.05585291e-01\n",
      " -9.83242754e-02 -9.16625252e-02 -8.54654211e-02 -7.97046813e-02\n",
      " -7.44306158e-02 -6.96172106e-02 -6.50997871e-02 -6.08671532e-02\n",
      " -5.69355345e-02 -5.31917074e-02 -4.97161975e-02 -4.65235407e-02\n",
      " -4.34664798e-02 -4.05689427e-02 -3.78477254e-02 -3.52618969e-02\n",
      " -3.27526320e-02 -3.02900436e-02 -2.79725148e-02 -2.57486271e-02\n",
      " -2.36653205e-02 -2.17009409e-02 -1.97337408e-02 -1.78049288e-02\n",
      " -1.60014405e-02 -1.43058207e-02 -1.27318031e-02 -1.12863979e-02\n",
      " -9.92657706e-03 -8.66951268e-03 -7.46740635e-03 -6.34276671e-03\n",
      " -5.35693168e-03 -4.51107546e-03 -3.70493328e-03 -2.96587382e-03\n",
      " -2.44003050e-03 -1.98401634e-03 -1.66243937e-03 -1.47613579e-03\n",
      " -1.20465857e-03 -9.23926375e-04 -6.19964255e-04 -3.65925068e-04\n",
      " -2.45721695e-04 -1.60472276e-04 -1.06005770e-04 -4.52079566e-05\n",
      "  0.00000000e+00  4.56671051e-05  1.07310265e-04  1.66150783e-04\n",
      "  2.55100507e-04  3.83260323e-04  6.45395988e-04  9.76912804e-04\n",
      "  1.28496472e-03  1.54323004e-03  1.78494128e-03  2.16716583e-03\n",
      "  2.72454647e-03  3.37598637e-03  4.17125040e-03  4.97792297e-03\n",
      "  5.92873259e-03  6.98098744e-03  8.04101539e-03  9.20687983e-03\n",
      "  1.04070015e-02  1.16871454e-02  1.30783964e-02  1.46223548e-02\n",
      "  1.62035841e-02  1.78461174e-02  1.95631184e-02  2.13257621e-02\n",
      "  2.31413701e-02  2.50469339e-02  2.70583022e-02  2.91708193e-02\n",
      "  3.14441298e-02  3.38032047e-02  3.61510215e-02  3.85760660e-02\n",
      "  4.11724645e-02  4.39164516e-02  4.67698021e-02  4.98106051e-02\n",
      "  5.30958521e-02  5.65304993e-02  6.01802098e-02  6.42265786e-02\n",
      "  6.85201247e-02  7.30720871e-02  7.80412878e-02  8.33992996e-02\n",
      "  8.91751223e-02  9.55312601e-02  1.02556689e-01  1.10121732e-01\n",
      "  1.18434154e-01  1.27865285e-01  1.38550199e-01  1.50168794e-01\n",
      "  1.63521009e-01  1.79422603e-01  1.98336892e-01  2.21722243e-01\n",
      "  2.51104204e-01  2.90005540e-01  3.47241963e-01  4.49613293e-01\n",
      "  4.50925554e+00]\n",
      "mean squared error equivalent in test set is 2941041824263.0464\n",
      "n is  8\n",
      "nBuckets: 257\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 560376 samples, validate on 140095 samples\n",
      "Epoch 1/4\n",
      "560376/560376 [==============================] - 760s 1ms/step - loss: 5.0957 - acc: 0.1847 - val_loss: 5.1850 - val_acc: 0.1455\n",
      "Epoch 2/4\n",
      "560376/560376 [==============================] - 758s 1ms/step - loss: 4.9825 - acc: 0.1872 - val_loss: 5.2101 - val_acc: 0.1464\n",
      "Epoch 3/4\n",
      "560376/560376 [==============================] - 758s 1ms/step - loss: 4.9754 - acc: 0.1874 - val_loss: 5.2293 - val_acc: 0.1467\n",
      "Epoch 4/4\n",
      "560376/560376 [==============================] - 758s 1ms/step - loss: 4.9710 - acc: 0.1874 - val_loss: 5.2409 - val_acc: 0.1470\n",
      "77831/77831 [==============================] - 172s 2ms/step\n",
      "[5.022922889767651, 0.17753851293301393]\n",
      "[-6.07432943e+00 -6.40417551e-01 -5.19198505e-01 -4.49580246e-01\n",
      " -4.00753570e-01 -3.63794295e-01 -3.34540921e-01 -3.10445224e-01\n",
      " -2.89886030e-01 -2.71808393e-01 -2.55459200e-01 -2.41030677e-01\n",
      " -2.28283951e-01 -2.16792431e-01 -2.06415860e-01 -1.96886675e-01\n",
      " -1.87984735e-01 -1.79674073e-01 -1.71946019e-01 -1.64703918e-01\n",
      " -1.57836914e-01 -1.51365645e-01 -1.45349612e-01 -1.39646322e-01\n",
      " -1.34222613e-01 -1.29032780e-01 -1.24216370e-01 -1.19745831e-01\n",
      " -1.15492423e-01 -1.11392848e-01 -1.07449319e-01 -1.03661682e-01\n",
      " -1.00088845e-01 -9.66154662e-02 -9.32412652e-02 -9.00528937e-02\n",
      " -8.69701822e-02 -8.39614496e-02 -8.10740251e-02 -7.83220179e-02\n",
      " -7.56227042e-02 -7.31006459e-02 -7.07576397e-02 -6.84662929e-02\n",
      " -6.61833830e-02 -6.39573063e-02 -6.18548797e-02 -5.98483225e-02\n",
      " -5.78709897e-02 -5.59459281e-02 -5.40902683e-02 -5.22715028e-02\n",
      " -5.05498146e-02 -4.88930702e-02 -4.72527605e-02 -4.57168482e-02\n",
      " -4.42159936e-02 -4.26948450e-02 -4.12271740e-02 -3.98507855e-02\n",
      " -3.85362025e-02 -3.71913737e-02 -3.58842165e-02 -3.46432168e-02\n",
      " -3.34242846e-02 -3.21560194e-02 -3.08848489e-02 -2.96905258e-02\n",
      " -2.85417030e-02 -2.74184972e-02 -2.63536828e-02 -2.52530008e-02\n",
      " -2.41854308e-02 -2.32028061e-02 -2.21717188e-02 -2.11899639e-02\n",
      " -2.02089557e-02 -1.92235105e-02 -1.82647437e-02 -1.73213769e-02\n",
      " -1.64362494e-02 -1.55761278e-02 -1.47034967e-02 -1.38679984e-02\n",
      " -1.31135521e-02 -1.23750327e-02 -1.16575991e-02 -1.09507133e-02\n",
      " -1.02407413e-02 -9.58780624e-03 -8.97642071e-03 -8.37229140e-03\n",
      " -7.72494939e-03 -7.12697237e-03 -6.67075116e-03 -6.14408855e-03\n",
      " -5.52443760e-03 -5.06526519e-03 -4.71067669e-03 -4.32399288e-03\n",
      " -3.86029613e-03 -3.44083776e-03 -3.13317847e-03 -2.81357738e-03\n",
      " -2.52799919e-03 -2.32175697e-03 -2.09422018e-03 -1.84444823e-03\n",
      " -1.69337728e-03 -1.62157225e-03 -1.53717895e-03 -1.42268039e-03\n",
      " -1.31374603e-03 -1.15676737e-03 -9.64943775e-04 -8.41190237e-04\n",
      " -7.14551827e-04 -5.34343245e-04 -4.05828172e-04 -3.31997568e-04\n",
      " -2.63493113e-04 -2.17120344e-04 -1.70859075e-04 -1.31982425e-04\n",
      " -1.14463156e-04 -9.88732995e-05 -8.11099585e-05 -3.59020019e-05\n",
      "  0.00000000e+00  3.64963770e-05  8.21634821e-05  1.00763585e-04\n",
      "  1.16739640e-04  1.36996681e-04  1.79861144e-04  2.23464891e-04\n",
      "  2.69550152e-04  3.42472537e-04  4.24547092e-04  5.79961325e-04\n",
      "  7.60022434e-04  8.85546258e-04  1.03700197e-03  1.21962110e-03\n",
      "  1.37621731e-03  1.50364512e-03  1.60531424e-03  1.68532191e-03\n",
      "  1.82536403e-03  2.07615185e-03  2.31833429e-03  2.54345183e-03\n",
      "  2.85865003e-03  3.17679995e-03  3.51304165e-03  3.96893889e-03\n",
      "  4.42796121e-03  4.79283706e-03  5.14048730e-03  5.63933322e-03\n",
      "  6.24249260e-03  6.72493495e-03  7.17403043e-03  7.75098829e-03\n",
      "  8.36192077e-03  8.90780544e-03  9.46273741e-03  1.00825566e-02\n",
      "  1.07277462e-02  1.13753193e-02  1.20102736e-02  1.27325394e-02\n",
      "  1.34888362e-02  1.42151708e-02  1.50028325e-02  1.58237365e-02\n",
      "  1.66173041e-02  1.74032953e-02  1.82522611e-02  1.91193890e-02\n",
      "  1.99874243e-02  2.08809262e-02  2.17755347e-02  2.26579229e-02\n",
      "  2.35789224e-02  2.45218634e-02  2.55064277e-02  2.64923191e-02\n",
      "  2.75191232e-02  2.86204665e-02  2.97061794e-02  3.08158656e-02\n",
      "  3.20034633e-02  3.32080794e-02  3.43795567e-02  3.55514482e-02\n",
      "  3.67277877e-02  3.79229397e-02  3.91716448e-02  4.04879036e-02\n",
      "  4.18355971e-02  4.32009524e-02  4.45972459e-02  4.60317816e-02\n",
      "  4.74888386e-02  4.89896200e-02  5.05733660e-02  5.22346278e-02\n",
      "  5.39361288e-02  5.56708881e-02  5.74040342e-02  5.92121802e-02\n",
      "  6.11287446e-02  6.31371165e-02  6.52669209e-02  6.74387071e-02\n",
      "  6.96024488e-02  7.18361090e-02  7.42243298e-02  7.67380291e-02\n",
      "  7.93190090e-02  8.19581718e-02  8.47352037e-02  8.76579845e-02\n",
      "  9.06567752e-02  9.38258100e-02  9.71831570e-02  1.00629997e-01\n",
      "  1.04298080e-01  1.08120973e-01  1.12017933e-01  1.16176512e-01\n",
      "  1.20591974e-01  1.25319697e-01  1.30335365e-01  1.35710083e-01\n",
      "  1.41379329e-01  1.47072741e-01  1.53022090e-01  1.59663839e-01\n",
      "  1.67066704e-01  1.74909870e-01  1.83408599e-01  1.92819799e-01\n",
      "  2.03235359e-01  2.14772154e-01  2.27741946e-01  2.42362684e-01\n",
      "  2.58774853e-01  2.77856344e-01  3.00345511e-01  3.27714118e-01\n",
      "  3.62461374e-01  4.09763016e-01  4.77387091e-01  5.96748367e-01\n",
      "  4.58876654e+00]\n",
      "mean squared error equivalent in test set is 1630073343740.3203\n",
      "n is  9\n",
      "nBuckets: 513\n",
      "Train on 560376 samples, validate on 140095 samples\n",
      "Epoch 1/4\n",
      "560376/560376 [==============================] - 768s 1ms/step - loss: 5.7094 - acc: 0.1847 - val_loss: 5.7730 - val_acc: 0.1455\n",
      "Epoch 2/4\n",
      "560376/560376 [==============================] - 765s 1ms/step - loss: 5.5527 - acc: 0.1872 - val_loss: 5.7886 - val_acc: 0.1458\n",
      "Epoch 3/4\n",
      "560376/560376 [==============================] - 765s 1ms/step - loss: 5.5462 - acc: 0.1872 - val_loss: 5.8025 - val_acc: 0.1459\n",
      "Epoch 4/4\n",
      "560376/560376 [==============================] - 765s 1ms/step - loss: 5.5422 - acc: 0.1873 - val_loss: 5.8136 - val_acc: 0.1460\n",
      "77831/77831 [==============================] - 172s 2ms/step\n",
      "[5.5950449156657935, 0.1772943942656449]\n",
      "[-6.16512967e+00 -8.10887671e-01 -6.71258386e-01 -5.91588499e-01\n",
      " -5.37463730e-01 -4.95914572e-01 -4.62437008e-01 -4.34367907e-01\n",
      " -4.10356029e-01 -3.89598456e-01 -3.71466055e-01 -3.55264353e-01\n",
      " -3.40560035e-01 -3.27508364e-01 -3.15843174e-01 -3.04799148e-01\n",
      " -2.94528259e-01 -2.85013091e-01 -2.76086705e-01 -2.67524236e-01\n",
      " -2.59209383e-01 -2.51422659e-01 -2.44212835e-01 -2.37571036e-01\n",
      " -2.31254336e-01 -2.25149410e-01 -2.19423468e-01 -2.14036875e-01\n",
      " -2.08802510e-01 -2.03812533e-01 -1.99121012e-01 -1.94581804e-01\n",
      " -1.90157415e-01 -1.85794684e-01 -1.81604168e-01 -1.77656236e-01\n",
      " -1.73832992e-01 -1.70052870e-01 -1.66416517e-01 -1.62954538e-01\n",
      " -1.59534027e-01 -1.56129003e-01 -1.52829285e-01 -1.49763041e-01\n",
      " -1.46779449e-01 -1.43829661e-01 -1.40980109e-01 -1.38226607e-01\n",
      " -1.35519990e-01 -1.32849783e-01 -1.30272735e-01 -1.27753109e-01\n",
      " -1.25384196e-01 -1.23087412e-01 -1.20832185e-01 -1.18658431e-01\n",
      " -1.16516855e-01 -1.14437200e-01 -1.12394284e-01 -1.10374365e-01\n",
      " -1.08405944e-01 -1.06482335e-01 -1.04561757e-01 -1.02697729e-01\n",
      " -1.00934413e-01 -9.92256035e-02 -9.74894293e-02 -9.57248598e-02\n",
      " -9.40337614e-02 -9.24241300e-02 -9.08407615e-02 -8.92620215e-02\n",
      " -8.77240642e-02 -8.62200927e-02 -8.47008652e-02 -8.31961041e-02\n",
      " -8.17713238e-02 -8.03886604e-02 -7.89949502e-02 -7.76256064e-02\n",
      " -7.62689792e-02 -7.49390093e-02 -7.37055205e-02 -7.25134321e-02\n",
      " -7.13326548e-02 -7.01817371e-02 -6.90367018e-02 -6.78962727e-02\n",
      " -6.67362147e-02 -6.55937339e-02 -6.45113207e-02 -6.34277248e-02\n",
      " -6.23671956e-02 -6.13483649e-02 -6.03347180e-02 -5.93469916e-02\n",
      " -5.83479878e-02 -5.73583814e-02 -5.64136586e-02 -5.54782034e-02\n",
      " -5.45455909e-02 -5.36253863e-02 -5.27318870e-02 -5.18333261e-02\n",
      " -5.09551525e-02 -5.01320253e-02 -4.93077881e-02 -4.84741710e-02\n",
      " -4.76632601e-02 -4.68565676e-02 -4.60777024e-02 -4.53484826e-02\n",
      " -4.45969622e-02 -4.38253274e-02 -4.30640961e-02 -4.23145824e-02\n",
      " -4.15978375e-02 -4.08796803e-02 -4.01618883e-02 -3.95036570e-02\n",
      " -3.88549917e-02 -3.81986399e-02 -3.75406274e-02 -3.68521503e-02\n",
      " -3.61995536e-02 -3.55808735e-02 -3.49637907e-02 -3.43414711e-02\n",
      " -3.37131580e-02 -3.31165454e-02 -3.24951061e-02 -3.18234535e-02\n",
      " -3.12183824e-02 -3.06188646e-02 -2.99756095e-02 -2.93808042e-02\n",
      " -2.88125355e-02 -2.82585180e-02 -2.77037637e-02 -2.71345755e-02\n",
      " -2.65849066e-02 -2.60892803e-02 -2.55777511e-02 -2.49726955e-02\n",
      " -2.44132132e-02 -2.39506987e-02 -2.34516297e-02 -2.29315194e-02\n",
      " -2.24287634e-02 -2.19177864e-02 -2.14267294e-02 -2.09559515e-02\n",
      " -2.04482894e-02 -1.99380591e-02 -1.94499771e-02 -1.89747622e-02\n",
      " -1.84896236e-02 -1.80060717e-02 -1.75601640e-02 -1.71003491e-02\n",
      " -1.66532358e-02 -1.62279231e-02 -1.57903390e-02 -1.53555301e-02\n",
      " -1.49208984e-02 -1.44830761e-02 -1.40657034e-02 -1.36680274e-02\n",
      " -1.33076038e-02 -1.29508334e-02 -1.25260074e-02 -1.21442584e-02\n",
      " -1.18241477e-02 -1.14884630e-02 -1.11422113e-02 -1.07710102e-02\n",
      " -1.04206936e-02 -1.00819228e-02 -9.73953975e-03 -9.42537550e-03\n",
      " -9.11082702e-03 -8.81360574e-03 -8.51286117e-03 -8.20595314e-03\n",
      " -7.89608220e-03 -7.55564822e-03 -7.23417034e-03 -6.97662729e-03\n",
      " -6.75358326e-03 -6.55490510e-03 -6.31371716e-03 -5.98573271e-03\n",
      " -5.65719294e-03 -5.36552644e-03 -5.15197894e-03 -4.98447303e-03\n",
      " -4.81484756e-03 -4.62776497e-03 -4.44507605e-03 -4.24547482e-03\n",
      " -4.00799873e-03 -3.74390321e-03 -3.50645851e-03 -3.35109566e-03\n",
      " -3.21835943e-03 -3.06606299e-03 -2.88306807e-03 -2.71576342e-03\n",
      " -2.59067594e-03 -2.47240241e-03 -2.37043871e-03 -2.28247002e-03\n",
      " -2.20193533e-03 -2.06236722e-03 -1.87629537e-03 -1.76609153e-03\n",
      " -1.71607628e-03 -1.67520917e-03 -1.64272824e-03 -1.61179034e-03\n",
      " -1.56068968e-03 -1.50723428e-03 -1.45188860e-03 -1.39084544e-03\n",
      " -1.33420840e-03 -1.28631720e-03 -1.20566379e-03 -1.09657633e-03\n",
      " -1.00212578e-03 -9.19389642e-04 -8.60102913e-04 -8.19085514e-04\n",
      " -7.69744854e-04 -6.84123844e-04 -5.74379845e-04 -4.79792272e-04\n",
      " -4.18211830e-04 -3.84284331e-04 -3.64585563e-04 -3.24682459e-04\n",
      " -2.71596352e-04 -2.42995001e-04 -2.30708991e-04 -2.12937573e-04\n",
      " -1.84042856e-04 -1.55553004e-04 -1.35476672e-04 -1.25089874e-04\n",
      " -1.18715107e-04 -1.11582637e-04 -1.01673024e-04 -9.32156378e-05\n",
      " -8.88008478e-05 -7.94948931e-05 -6.64384789e-05 -3.05364770e-05\n",
      "  0.00000000e+00  3.12500195e-05  6.77463965e-05  8.02562996e-05\n",
      "  8.94270276e-05  9.53872799e-05  1.04816654e-04  1.13855926e-04\n",
      "  1.20402607e-04  1.29357228e-04  1.43067589e-04  1.62476865e-04\n",
      "  1.91630967e-04  2.18924884e-04  2.33374530e-04  2.47064181e-04\n",
      "  2.78699797e-04  3.31229375e-04  3.72516144e-04  3.97030583e-04\n",
      "  4.37818369e-04  5.11868626e-04  6.26495073e-04  7.38701702e-04\n",
      "  8.04136364e-04  8.45224569e-04  9.05313731e-04  9.91523987e-04\n",
      "  1.08289053e-03  1.19659132e-03  1.28784392e-03  1.33994452e-03\n",
      "  1.40528814e-03  1.47273649e-03  1.53482068e-03  1.59312581e-03\n",
      "  1.63271073e-03  1.66223527e-03  1.70265802e-03  1.73855974e-03\n",
      "  1.83817911e-03  2.04158957e-03  2.19275803e-03  2.27810726e-03\n",
      "  2.36912124e-03  2.47901403e-03  2.61311759e-03  2.75625232e-03\n",
      "  2.93734696e-03  3.12832637e-03  3.26538165e-03  3.38909324e-03\n",
      "  3.58827966e-03  3.84223108e-03  4.09894189e-03  4.33060603e-03\n",
      "  4.53291755e-03  4.72089093e-03  4.88345526e-03  5.05528601e-03\n",
      "  5.24037192e-03  5.47704552e-03  5.79080553e-03  6.12120070e-03\n",
      "  6.41060007e-03  6.60765587e-03  6.80069886e-03  7.02113053e-03\n",
      "  7.27718302e-03  7.59638233e-03  7.91728770e-03  8.19997628e-03\n",
      "  8.49000338e-03  8.76627410e-03  9.02213168e-03  9.31724053e-03\n",
      "  9.61631492e-03  9.91129785e-03  1.02320426e-02  1.05633653e-02\n",
      "  1.08878103e-02  1.12280000e-02  1.15511282e-02  1.18602841e-02\n",
      "  1.21721102e-02  1.25486988e-02  1.29591386e-02  1.32995159e-02\n",
      "  1.36453728e-02  1.40049653e-02  1.43854429e-02  1.47968868e-02\n",
      "  1.52040709e-02  1.55816427e-02  1.59953627e-02  1.64139431e-02\n",
      "  1.67937906e-02  1.72111987e-02  1.76173423e-02  1.80082335e-02\n",
      "  1.84510556e-02  1.88999670e-02  1.93242729e-02  1.97504647e-02\n",
      "  2.01941941e-02  2.06588292e-02  2.11086018e-02  2.15456372e-02\n",
      "  2.19904731e-02  2.24428025e-02  2.28803548e-02  2.33548527e-02\n",
      "  2.38382999e-02  2.42669684e-02  2.47264621e-02  2.52680778e-02\n",
      "  2.57931482e-02  2.62432623e-02  2.67040832e-02  2.72400109e-02\n",
      "  2.78059941e-02  2.83543748e-02  2.88897349e-02  2.94238864e-02\n",
      "  2.99742391e-02  3.05703987e-02  3.11297322e-02  3.16875996e-02\n",
      "  3.23158637e-02  3.29143536e-02  3.34907056e-02  3.41042159e-02\n",
      "  3.46993412e-02  3.52583752e-02  3.58351415e-02  3.64051225e-02\n",
      "  3.70046958e-02  3.76290358e-02  3.82246146e-02  3.88693454e-02\n",
      "  3.95224716e-02  4.01491441e-02  4.08122768e-02  4.14904932e-02\n",
      "  4.21750542e-02  4.28569423e-02  4.35377367e-02  4.42271446e-02\n",
      "  4.49426437e-02  4.56700712e-02  4.63891077e-02  4.71171220e-02\n",
      "  4.78551425e-02  4.85985542e-02  4.93613150e-02  5.01464537e-02\n",
      "  5.09674389e-02  5.18200689e-02  5.26603456e-02  5.34986629e-02\n",
      "  5.43598872e-02  5.52388763e-02  5.61124113e-02  5.69756185e-02\n",
      "  5.78352297e-02  5.87323233e-02  5.96808581e-02  6.06433602e-02\n",
      "  6.16113898e-02  6.26314161e-02  6.36717584e-02  6.47118403e-02\n",
      "  6.58013024e-02  6.69042734e-02  6.79865975e-02  6.90397134e-02\n",
      "  7.01211310e-02  7.12367842e-02  7.23890268e-02  7.35944847e-02\n",
      "  7.48304628e-02  7.60916216e-02  7.73693428e-02  7.86579393e-02\n",
      "  7.99611979e-02  8.12561803e-02  8.25920845e-02  8.39894728e-02\n",
      "  8.54306006e-02  8.68901807e-02  8.83718336e-02  8.98971386e-02\n",
      "  9.14142764e-02  9.29942053e-02  9.46461022e-02  9.63452719e-02\n",
      "  9.80507220e-02  9.97205971e-02  1.01461988e-01  1.03325569e-01\n",
      "  1.05252261e-01  1.07164915e-01  1.09061116e-01  1.10996545e-01\n",
      "  1.12997304e-01  1.15046022e-01  1.17203842e-01  1.19460961e-01\n",
      "  1.21718603e-01  1.24074370e-01  1.26544450e-01  1.29038972e-01\n",
      "  1.31584560e-01  1.34271577e-01  1.37100707e-01  1.39875910e-01\n",
      "  1.42716027e-01  1.45684809e-01  1.48538105e-01  1.51368704e-01\n",
      "  1.54464757e-01  1.57912036e-01  1.61457732e-01  1.65062423e-01\n",
      "  1.68919593e-01  1.72925200e-01  1.76911196e-01  1.81127105e-01\n",
      "  1.85639838e-01  1.90319549e-01  1.95218017e-01  2.00392923e-01\n",
      "  2.05910015e-01  2.11642100e-01  2.17661803e-01  2.24201726e-01\n",
      "  2.31151815e-01  2.38384750e-01  2.46055399e-01  2.54213761e-01\n",
      "  2.62955281e-01  2.72368015e-01  2.82707986e-01  2.93878217e-01\n",
      "  3.06027413e-01  3.19728013e-01  3.34947424e-01  3.52223397e-01\n",
      "  3.71751241e-01  3.94669957e-01  4.22443755e-01  4.54684806e-01\n",
      "  4.94535083e-01  5.47903939e-01  6.27414938e-01  7.72576569e-01\n",
      "  4.68508374e+00]\n",
      "mean squared error equivalent in test set is 328552352920.21606\n"
     ]
    }
   ],
   "source": [
    "for n in range(1,10):\n",
    "    print(\"n is \", n)\n",
    "    nBuckets = 2 ** n + 1\n",
    "    \n",
    "    # get buckets separating values in a list \n",
    "    print(\"nBuckets:\", nBuckets)\n",
    "    if nBuckets == 3:\n",
    "        bucket_list = [-Inf,0,Inf]\n",
    "    else:\n",
    "        bucket_list = bucket_separate((nBuckets-1)/2,[-Inf,0,Inf],y_neg)\n",
    "        bucket_list = bucket_separate((nBuckets-1)/2,bucket_list,y_pos)\n",
    "    #print(bucket_list)\n",
    "\n",
    "    train_X, train_Y, test_X, test_Y, de_norm_train, de_norm_test, true_Y_test = \\\n",
    "    lstm_preprocess(data, close_price, time, bucket_list, 0.1)\n",
    "    \n",
    "    \n",
    "    model = build_model(train_X,nBuckets)\n",
    "    #model.summary()\n",
    "    res = model.fit(train_X, train_Y,batch_size=512, epochs=4,verbose=1, shuffle=True,validation_split = 0.2)\n",
    "    \n",
    "    # evaluate on test set, print [loss, accuracy]\n",
    "    #test_result = model.evaluate(test_X, test_Y)\n",
    "    \n",
    "    # evaluate on test set, print [loss, accuracy]\n",
    "    test_result = model.evaluate(test_X, test_Y)\n",
    "    print(test_result)\n",
    "    \n",
    "    # evaluate on test set again, print equivalent\n",
    "    Y_pred = model.predict(test_X)\n",
    "    mse,pred = mse_pred(true_Y_test,Y_pred,de_norm_test,True,bucket_list)\n",
    "    #print(mse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(\"lstm_1_delta.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "t = np.arange(0,pred.shape[0])\n",
    "plt.plot(t, pred, true_Y_test)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mse,pred = mse_pred(true_Y_test,Y_pred,de_norm_test,False,bucket_list)\n",
    "print(mse)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = build_model(train_X,nBuckets)\n",
    "model.summary()\n",
    "res = model.fit(train_X, train_Y,batch_size=512, epochs=5,verbose=1, shuffle=True,validation_split = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "model = build_model(train_X,nBuckets)\n",
    "model.summary()\n",
    "res = model.fit(train_X, train_Y,batch_size=512, epochs=5,verbose=1, shuffle=True,validation_split = 0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(1000):\n",
    "    print(train_Y[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}