tabular-nn-keras-guide-cat-and-num.ipynb

{"cells":[{"metadata":{"papermill":{"duration":0.012437,"end_time":"2020-12-21T10:40:29.455899","exception":false,"start_time":"2020-12-21T10:40:29.443462","status":"completed"},"tags":[]},"cell_type":"markdown","source":"# Tabular NN - Cat and Num\nhttps://keras.io/examples/structured_data/structured_data_classification_from_scratch/"},{"metadata":{"execution":{"iopub.execute_input":"2020-12-21T10:40:29.483627Z","iopub.status.busy":"2020-12-21T10:40:29.483091Z","iopub.status.idle":"2020-12-21T10:40:34.874975Z","shell.execute_reply":"2020-12-21T10:40:34.873737Z"},"papermill":{"duration":5.409323,"end_time":"2020-12-21T10:40:34.875113","exception":false,"start_time":"2020-12-21T10:40:29.46579","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nimport numpy as np\nimport pandas as pd\nimport pprint as pp\nimport matplotlib.pyplot as plt\nimport time\n\nimport itertools\nfrom sklearn.metrics import confusion_matrix\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nfrom keras.utils import to_categorical\nfrom tensorflow.keras import callbacks\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.layers.experimental.preprocessing import Normalization\nfrom tensorflow.keras.layers.experimental.preprocessing import CategoryEncoding\nfrom tensorflow.keras.layers.experimental.preprocessing import StringLookup\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nnotebookstart = time.time()  ","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-21T10:40:34.901142Z","iopub.status.busy":"2020-12-21T10:40:34.900474Z","iopub.status.idle":"2020-12-21T10:40:34.903395Z","shell.execute_reply":"2020-12-21T10:40:34.903805Z"},"papermill":{"duration":0.018182,"end_time":"2020-12-21T10:40:34.903954","exception":false,"start_time":"2020-12-21T10:40:34.885772","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"TARGETVAR = \"target\"\nIDVAR = 'id'\nNCHANNELS = 3\nBATCHSIZE = 64\nEPOCHS = 300\n\ncontinuous_cols = [\"age\", \"trestbps\", \"chol\", \"thalach\", \"oldpeak\", \"slope\"]\nstring_cols = [\"thal\"]\ncategorical_cols = [\"sex\",\"cp\",\"fbs\",\"restecg\",\"exang\",\"ca\"]\n\nall_cols = continuous_cols + string_cols + categorical_cols","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-21T10:40:34.949476Z","iopub.status.busy":"2020-12-21T10:40:34.938119Z","iopub.status.idle":"2020-12-21T10:40:34.951907Z","shell.execute_reply":"2020-12-21T10:40:34.951464Z"},"papermill":{"duration":0.037149,"end_time":"2020-12-21T10:40:34.952","exception":false,"start_time":"2020-12-21T10:40:34.914851","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"def encode_numerical_feature(feature, name, dataset):\n    # Create a Normalization layer for our feature\n    normalizer = Normalization()\n\n    # Prepare a Dataset that only yields our feature\n    feature_ds = dataset.map(lambda x, y: x[name])\n    feature_ds = feature_ds.map(lambda x: tf.expand_dims(x, -1))\n\n    # Learn the statistics of the data\n    normalizer.adapt(feature_ds)\n\n    # Normalize the input feature\n    encoded_feature = normalizer(feature)\n    return encoded_feature\n\n\ndef encode_string_categorical_feature(feature, name, dataset):\n    # Create a StringLookup layer which will turn strings into integer indices\n    index = StringLookup()\n\n    # Prepare a Dataset that only yields our feature\n    feature_ds = dataset.map(lambda x, y: x[name])\n    feature_ds = feature_ds.map(lambda x: tf.expand_dims(x, -1))\n\n    # Learn the set of possible string values and assign them a fixed integer index\n    index.adapt(feature_ds)\n\n    # Turn the string input into integer indices\n    encoded_feature = index(feature)\n\n    # Create a CategoryEncoding for our integer indices\n    encoder = CategoryEncoding(output_mode=\"binary\")\n\n    # Prepare a dataset of indices\n    feature_ds = feature_ds.map(index)\n\n    # Learn the space of possible indices\n    encoder.adapt(feature_ds)\n\n    # Apply one-hot encoding to our indices\n    encoded_feature = encoder(encoded_feature)\n    return encoded_feature\n\n\ndef encode_integer_categorical_feature(feature, name, dataset):\n    # Create a CategoryEncoding for our integer indices\n    encoder = CategoryEncoding(output_mode=\"binary\")\n\n    # Prepare a Dataset that only yields our feature\n    feature_ds = dataset.map(lambda x, y: x[name])\n    feature_ds = feature_ds.map(lambda x: tf.expand_dims(x, -1))\n\n    # Learn the space of possible indices\n    encoder.adapt(feature_ds)\n\n    # Apply one-hot encoding to our indices\n    encoded_feature = encoder(feature)\n    return encoded_feature\n\n\ndef dataframe_to_dataset(dataframe, labels, role, BATCHSIZE):\n    dataframe = dataframe.copy()\n    if role != \"test\":\n        ds = tf.data.Dataset.from_tensor_slices((dict(dataframe), labels))\n    else: \n        ds = tf.data.Dataset.from_tensor_slices((dict(dataframe), labels))\n    if role == \"train\":\n        ds = ds.shuffle(buffer_size=len(dataframe))\n    ds = ds.batch(BATCHSIZE)\n    return ds\n\ndef ohe_target(arr):\n    label_mapper = {name: i for i,name in enumerate(set(arr))}\n    num_label = np.vectorize(label_mapper.get)(arr)\n    train_labels = to_categorical(num_label)\n    \n    return train_labels, num_label, label_mapper\n\n### From http://scikit-learn.org/stable/auto_examples/model_selection/plot_confusion_matrix.html#sphx-glr-auto-examples-model-selection-plot-confusion-matrix-py #\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    \"\"\"\n    This function prints and plots the confusion matrix.\n    Normalization can be applied by setting `normalize=True`.\n    \"\"\"\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n        #print(\"Normalized confusion matrix\")\n    #else:\n    #    print('Confusion matrix, without normalization')\n\n    #print(cm)\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-21T10:40:34.980619Z","iopub.status.busy":"2020-12-21T10:40:34.9801Z","iopub.status.idle":"2020-12-21T10:40:35.095704Z","shell.execute_reply":"2020-12-21T10:40:35.094717Z"},"papermill":{"duration":0.133499,"end_time":"2020-12-21T10:40:35.095818","exception":false,"start_time":"2020-12-21T10:40:34.962319","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"file_url = \"http://storage.googleapis.com/download.tensorflow.org/data/heart.csv\"\ntrain = pd.read_csv(file_url)\ntrain[IDVAR] = train.index\n\ndisplay(train.sample(4))\n\n# test = pd.read_csv(\"/kaggle/input/ghouls-goblins-and-ghosts-boo/test.csv.zip\")\n# test_index = test[IDVAR]\n# submission = pd.read_csv(\"/kaggle/input/ghouls-goblins-and-ghosts-boo/sample_submission.csv.zip\")\n\ntrain_labels, num_label, label_mapper = ohe_target(train[TARGETVAR].values)\nval_dataframe = train[all_cols].sample(frac=0.2, random_state=1337)\ntrain_dataframe = train[all_cols].drop(val_dataframe.index)\n\nprint(\n    \"Using %d samples for training and %d for validation\"\n    % (len(train_dataframe), len(val_dataframe))\n)\n\ntrain_ds = dataframe_to_dataset(train_dataframe, train.loc[train_dataframe.index, TARGETVAR],\n                                \"train\", BATCHSIZE)\nval_ds = dataframe_to_dataset(val_dataframe, train.loc[val_dataframe.index, TARGETVAR],\n                              \"val\", BATCHSIZE)\n# test_ds = dataframe_to_dataset(test[all_cols], np.zeros((test.shape[0],3)), \"test\", BATCHSIZE)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-21T10:40:35.123232Z","iopub.status.busy":"2020-12-21T10:40:35.122688Z","iopub.status.idle":"2020-12-21T10:40:35.175141Z","shell.execute_reply":"2020-12-21T10:40:35.174607Z"},"papermill":{"duration":0.068128,"end_time":"2020-12-21T10:40:35.175255","exception":false,"start_time":"2020-12-21T10:40:35.107127","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"for x, y in train_ds.take(1):\n    pp.pprint(x)\n    pp.pprint(y)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-21T10:40:35.211812Z","iopub.status.busy":"2020-12-21T10:40:35.211299Z","iopub.status.idle":"2020-12-21T10:40:35.551078Z","shell.execute_reply":"2020-12-21T10:40:35.550365Z"},"papermill":{"duration":0.363157,"end_time":"2020-12-21T10:40:35.551229","exception":false,"start_time":"2020-12-21T10:40:35.188072","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"continuous_inputs = [keras.Input(shape=(1,), name=x) for x in continuous_cols]\nstring_inputs = [keras.Input(shape=(1,), name=x, dtype=\"string\") for x in string_cols]\ncategorical_inputs = [keras.Input(shape=(1,), name=x, dtype=\"int64\") for x in categorical_cols]\n\nall_inputs = continuous_inputs + string_inputs + categorical_inputs\nprint(\"All Input Len: {}\".format(len(all_inputs)))\n\nencoded_strings = [encode_string_categorical_feature(var_input, var_name, train_ds)\n                   for var_input, var_name in zip(string_inputs, string_cols)]\n\nencoded_nums = [encode_numerical_feature(var_input, var_name, train_ds)\n                   for var_input, var_name in zip(continuous_inputs, continuous_cols)]\n\nencoded_cats = [encode_integer_categorical_feature(var_input, var_name, train_ds)\n                   for var_input, var_name in zip(categorical_inputs, categorical_cols)]\n\nall_features = layers.concatenate(encoded_nums + encoded_strings + encoded_cats)\nprint(\"All Feature Len: {}\".format(all_features.shape))","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-21T10:40:35.592546Z","iopub.status.busy":"2020-12-21T10:40:35.590645Z","iopub.status.idle":"2020-12-21T10:40:35.660632Z","shell.execute_reply":"2020-12-21T10:40:35.660046Z"},"papermill":{"duration":0.089419,"end_time":"2020-12-21T10:40:35.660735","exception":false,"start_time":"2020-12-21T10:40:35.571316","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"x = layers.Dense(64, activation=\"relu\")(all_features)\nx = layers.Dropout(0.3)(x)\nx = layers.Dense(3, activation=\"relu\")(x)\nx = layers.Dropout(0.3)(x)\noutput = layers.Dense(1, activation=\"sigmoid\")(x)\nmodel = keras.Model(all_inputs, output)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-21T10:40:35.691745Z","iopub.status.busy":"2020-12-21T10:40:35.691001Z","iopub.status.idle":"2020-12-21T10:40:36.251232Z","shell.execute_reply":"2020-12-21T10:40:36.251617Z"},"papermill":{"duration":0.578076,"end_time":"2020-12-21T10:40:36.251738","exception":false,"start_time":"2020-12-21T10:40:35.673662","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# `rankdir='LR'` is to make the graph horizontal.\nkeras.utils.plot_model(model, show_shapes=True, rankdir=\"LR\")","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-output":true,"execution":{"iopub.execute_input":"2020-12-21T10:40:36.281743Z","iopub.status.busy":"2020-12-21T10:40:36.281238Z","iopub.status.idle":"2020-12-21T10:40:41.40001Z","shell.execute_reply":"2020-12-21T10:40:41.399516Z"},"papermill":{"duration":5.134305,"end_time":"2020-12-21T10:40:41.40011","exception":false,"start_time":"2020-12-21T10:40:36.265805","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"# checkpoint = callbacks.ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)\nes = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001,\n                             patience=15, verbose=1, mode='min', baseline=None,\n                             restore_best_weights=True)\n\nreduce_lr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.2,\n                              patience=5, min_lr=0.00001, verbose=1)\n\n\nmodel.compile(Adam(lr=1e-3), \"binary_crossentropy\", metrics=[\"accuracy\"])\nhistory = model.fit(train_ds, batch_size=BATCHSIZE, epochs=EPOCHS,\n                    validation_data=val_ds, verbose=1, callbacks = [es, reduce_lr]\n)","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-21T10:40:41.665963Z","iopub.status.busy":"2020-12-21T10:40:41.665419Z","iopub.status.idle":"2020-12-21T10:40:41.988856Z","shell.execute_reply":"2020-12-21T10:40:41.988414Z"},"papermill":{"duration":0.459353,"end_time":"2020-12-21T10:40:41.988954","exception":false,"start_time":"2020-12-21T10:40:41.529601","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"plot_metrics = ['loss', 'accuracy']\n\nf, ax = plt.subplots(1,2,figsize = [12,4])\nfor p_i,metric in enumerate(plot_metrics):\n    ax[p_i].plot(history.history[metric], label='Train ' + metric, )\n    ax[p_i].plot(history.history['val_' + metric], label='Val ' + metric)\n    ax[p_i].set_title(\"Loss Curve - {}\".format(metric))\n    ax[p_i].set_ylabel(metric.title())\n    ax[p_i].legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-21T10:40:42.257462Z","iopub.status.busy":"2020-12-21T10:40:42.256941Z","iopub.status.idle":"2020-12-21T10:40:42.565068Z","shell.execute_reply":"2020-12-21T10:40:42.564188Z"},"papermill":{"duration":0.4464,"end_time":"2020-12-21T10:40:42.565189","exception":false,"start_time":"2020-12-21T10:40:42.118789","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"val_pred = model.predict(val_ds)\nval_pred_int = (val_pred.flatten() > 0.5).astype(int)\ncnf_matrix = confusion_matrix(train.loc[val_dataframe.index, TARGETVAR], val_pred_int)\nnp.set_printoptions(precision=2)\n\n# Plot non-normalized confusion matrix\nplt.figure(figsize=(5,5))\nplot_confusion_matrix(cnf_matrix, classes=label_mapper.keys(),\n                      title='Confusion matrix, without normalization')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"execution":{"iopub.execute_input":"2020-12-21T10:40:44.61249Z","iopub.status.busy":"2020-12-21T10:40:44.611869Z","iopub.status.idle":"2020-12-21T10:40:44.615297Z","shell.execute_reply":"2020-12-21T10:40:44.614743Z"},"papermill":{"duration":0.14028,"end_time":"2020-12-21T10:40:44.6154","exception":false,"start_time":"2020-12-21T10:40:44.47512","status":"completed"},"tags":[],"trusted":true},"cell_type":"code","source":"print(\"Notebook Runtime: %0.2f Minutes\"%((time.time() - notebookstart)/60))","execution_count":null,"outputs":[]},{"metadata":{"papermill":{"duration":0.13352,"end_time":"2020-12-21T10:40:44.882008","exception":false,"start_time":"2020-12-21T10:40:44.748488","status":"completed"},"tags":[],"trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}