disaster-text-numeric-embeddings-densenet-cv.ipynb

{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Disaster Text - Numeric & Embeddings DenseNet CV\n_by Nick Brooks, January 2020_\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\nfrom keras.models import Model\nfrom keras.layers import Input, Flatten, Dense, Embedding, SpatialDropout1D, concatenate, Dropout, BatchNormalization, Activation\nfrom keras.layers import LSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.preprocessing import text, sequence\nfrom keras import optimizers\nfrom gensim.models import KeyedVectors\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer\n\nfrom sklearn.model_selection import KFold\nfrom tensorflow.keras import callbacks\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nfrom sklearn.metrics import classification_report, confusion_matrix\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import StandardScaler\nfrom scipy.sparse import hstack, csr_matrix\n\nimport time\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport pprint\nimport re\nimport nltk\n\nimport seaborn as sns\nsns.set_style(\"whitegrid\")\nnotebookstart = time.time()\npd.options.display.max_colwidth = 500\n\nimport keras\nprint(\"Keras Version: \",keras.__version__)\nimport tensorflow\nprint(\"Tensorflow Version: \", tensorflow.__version__)\n\nEMBEDDING_FILES = [\n    '../input/gensim-embeddings-dataset/crawl-300d-2M.gensim',\n    '../input/gensim-embeddings-dataset/glove.840B.300d.gensim'\n]\n\nseed = 25\n\nN_ROWS = None\nBATCH_SIZE = 64\nEPOCHS = 100\nN_CLASSES = 1\nf1_strategy = 'macro'\n\nMAX_LEN = 34\nAUX_COLUMNS = ['target']\nTEXT_COLUMN = 'text'\nTARGET_COLUMN = 'target'\nCHARS_TO_REMOVE = '!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n“”’\\'∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bef8d43d-7ca0-4548-9807-fa2263515a63","_cell_guid":"b5f992b0-8986-47d9-9f14-1955dca74931","trusted":true},"cell_type":"code","source":"def build_matrix(word_index, path):\n    embedding_index = KeyedVectors.load(path, mmap='r')\n    embedding_matrix = np.zeros((len(word_index) + 1, 300))\n    for word, i in word_index.items():\n        for candidate in [word, word.lower()]:\n            if candidate in embedding_index:\n                embedding_matrix[i] = embedding_index[candidate]\n                break\n    return embedding_matrix\n\ndef text_processing(df):\n    df['keyword'] = df['keyword'].str.replace(\"%20\", \" \")\n    df['hashtags'] = df['text'].apply(lambda x: \" \".join(re.findall(r\"#(\\w+)\", x)))\n    df['hash_loc_key'] = df[['hashtags', 'location','keyword']].astype(str).apply(lambda x: \" \".join(x), axis=1)\n    df['hash_loc_key'] = df[\"hash_loc_key\"].astype(str).str.lower().str.strip().fillna('nan')\n    \n    textfeats = ['hash_loc_key', 'text']\n    for cols in textfeats:\n        df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.split())) # Count number of Words\n        df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))\n        df[cols + '_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words'] * 100 # Count Unique Words\n        if cols == \"text\":\n            df[cols+\"_vader_Compound\"]= df[cols].apply(lambda x:SIA.polarity_scores(x)['compound'])\n\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Read Data\")\ntrain_df = pd.read_csv('../input/nlp-getting-started/train.csv', nrows = N_ROWS)\ntest_df = pd.read_csv('../input/nlp-getting-started/test.csv', nrows = N_ROWS)\n\nX = train_df[TEXT_COLUMN].astype(str)\ny = train_df[TARGET_COLUMN].values\ntest = test_df[TEXT_COLUMN].astype(str)\n\nprint(\"Train Shape: {} Rows\".format(X.shape[0]))\nprint(\"Test Shape: {} Rows\".format(test.shape[0]))\nprint('Dependent Variable Factor Ratio: ',train_df[TARGET_COLUMN].value_counts(normalize=True).to_dict())\n\ntokenizer = text.Tokenizer(filters=CHARS_TO_REMOVE, lower=False)\ntokenizer.fit_on_texts(list(X) + list(test))\n\nX = tokenizer.texts_to_sequences(X)\ntest = tokenizer.texts_to_sequences(test)\n\nlength_info = [len(x) for x in X]\nprint(\"Train Sequence Length - Mean {:.1f} +/- {:.1f}, Max {:.1f}, Min {:.1f}\".format(\n    np.mean(length_info), np.std(length_info), np.max(length_info), np.min(length_info)))\n\nX = sequence.pad_sequences(X, maxlen=MAX_LEN)\ntest = sequence.pad_sequences(test, maxlen=MAX_LEN)\n\nembedding_matrix = np.concatenate(\n    [build_matrix(tokenizer.word_index, f) for f in EMBEDDING_FILES], axis=-1)\n\nprint(\"Embeddings Matrix Shape:\", embedding_matrix.shape)\n\ncheckpoint_predictions = []\nweights = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Text Processing\nSIA = SentimentIntensityAnalyzer()\ntrain_df = text_processing(train_df)\ntest_df = text_processing(test_df)\n\n# TF-IDF\ncount_vectorizer = TfidfVectorizer(\n    analyzer=\"word\",\n    tokenizer=nltk.word_tokenize,\n    preprocessor=None,\n    stop_words='english',\n    ngram_range=(1, 1),\n    max_features=None)    \n\nhash_loc_tfidf = count_vectorizer.fit(train_df['hash_loc_key'])\ntfvocab = hash_loc_tfidf.get_feature_names()\nprint(\"Number of TF-IDF Features: {}\".format(len(tfvocab)))\n\ntrain_tfidf = count_vectorizer.transform(train_df['hash_loc_key'])\ntest_tfidf = count_vectorizer.transform(test_df['hash_loc_key'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Sparse Stack Numerical and TFIDF\ndense_vars = [\n    'hash_loc_key_num_words',\n    'hash_loc_key_num_unique_words',\n    'hash_loc_key_words_vs_unique',\n    'text_num_words',\n    'text_num_unique_words',\n    'text_words_vs_unique',\n    'text_vader_Compound']\n\n# Normalisation - Standard Scaler\nfor d_i in dense_vars:\n    scaler = StandardScaler()\n    scaler.fit(train_df.loc[:,d_i].values.reshape(-1, 1))\n    train_df.loc[:,d_i] = scaler.transform(train_df.loc[:,d_i].values.reshape(-1, 1))\n    test_df.loc[:,d_i] = scaler.transform(test_df.loc[:,d_i].values.reshape(-1, 1))\n    \n# Sparse Stack\ntrain_num = hstack([csr_matrix(train_df.loc[:,dense_vars].values),train_tfidf]).tocsr()\ntest_num = hstack([csr_matrix(test_df.loc[:,dense_vars].values),test_tfidf]).tocsr()\nnum_cols = train_df[dense_vars].columns.tolist() + tfvocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(embedding_matrix, n_classes):\n    words_inputs = Input(shape=(None,))\n    numeric_inputs = Input(shape=(len(num_cols),))\n    \n    # Dense Inputs\n    numeric_x = Dense(512, activation='relu')(numeric_inputs)\n    numeric_x = Dropout(.4)(numeric_x)\n    numeric_x = Dense(64, activation='relu')(numeric_x)\n    \n    # Embeddings Inputs\n    words_x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix],\n                  trainable=False, input_length=MAX_LEN)(words_inputs)\n    words_x = Flatten()(words_x)\n    \n    # Concat\n    concat_x = concatenate([words_x, numeric_x])\n    concat_x = Dropout(.4)(concat_x)\n    output = Dense(n_classes, activation='sigmoid')(concat_x)\n    model = Model(inputs=[words_inputs,numeric_inputs], outputs=output)\n    opt = optimizers.Adam(learning_rate=0.00004, beta_1=0.9, beta_2=0.999, amsgrad=True)\n    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['acc'])\n    \n    return model\n\nmodel = build_model(embedding_matrix, N_CLASSES)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_preds = np.zeros(X.shape[0])\ntest_preds = np.zeros(test.shape[0])\n\nn_splits = 6\nfolds = KFold(n_splits=n_splits, shuffle=True, random_state=seed)\nplot_metrics = ['loss','acc']\n\nfold_hist = {}\nfor i, (trn_idx, val_idx) in enumerate(folds.split(X)):\n    modelstart = time.time()\n    model = build_model(embedding_matrix, N_CLASSES)\n    \n    es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=4, verbose=1,\n                                 mode='min', baseline=None, restore_best_weights=True)\n    rlr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=2, min_lr=1e-7,\n                                      mode='min', verbose=1)\n    \n    history = model.fit(\n            [X[trn_idx], train_num[trn_idx]],\n            y[trn_idx],\n            validation_data=([X[val_idx], train_num[val_idx]], y[val_idx]),\n            batch_size=BATCH_SIZE,\n            epochs=EPOCHS,\n            verbose=0,\n            callbacks=[es, rlr]\n        )\n\n    best_index = np.argmin(history.history['val_loss'])\n    fold_hist[i] = history\n    \n    oof_preds[val_idx] = model.predict([X[val_idx], train_num[val_idx]]).ravel()\n    test_preds += model.predict([test, test_num]).ravel()\n    f1_sc = f1_score(y[val_idx], (oof_preds[val_idx] > 0.5).astype(int), average=f1_strategy)\n    print(\"\\nFOLD {} COMPLETE in {:.1f} Minutes - Avg F1 {:.5f} - Best Epoch {}\".format(i, (time.time() - modelstart)/60, f1_sc, best_index + 1))\n    best_metrics = {metric: scores[best_index] for metric, scores in history.history.items()}\n    pprint.pprint(best_metrics)\n    \n    f, ax = plt.subplots(1,len(plot_metrics),figsize = [12,4])\n    for p_i,metric in enumerate(plot_metrics):\n        ax[p_i].plot(history.history[metric], label='Train ' + metric)\n        ax[p_i].plot(history.history['val_' + metric], label='Val ' + metric)\n        ax[p_i].set_title(\"{} Fold Loss Curve - {}\\nBest Epoch {}\".format(i, metric, best_index))\n        ax[p_i].legend()\n        ax[p_i].axvline(x=best_index, c='black')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# OOF F1 Cutoff\nsave_f1_opt = []\nfor cutoff in np.arange(.38,.62, .01):\n    save_f1_opt.append([cutoff, f1_score(y, (oof_preds > cutoff).astype(int), average=f1_strategy)])\nf1_pd = pd.DataFrame(save_f1_opt, columns = ['cutoff', 'f1_score'])\n\nbest_cutoff = f1_pd.loc[f1_pd['f1_score'].idxmax(),'cutoff']\nprint(\"F1 Score: {:.4f}, Optimised Cufoff: {:.2f}\".format(f1_pd.loc[f1_pd['f1_score'].idxmax(),'f1_score'], best_cutoff))\n\nf,ax = plt.subplots(1,2,figsize = [10,4])\n\nax[0].plot(f1_pd['cutoff'], f1_pd['f1_score'], c = 'red')\nax[0].set_ylabel(\"F1 Score\")\nax[0].set_xlabel(\"Cutoff\")\nax[0].axvline(x=best_cutoff, c='black')\nax[0].set_title(\"F1 Score and Cutoff on OOF\")\n\n\ntrain_df['oof_preds'] = oof_preds\ntrain_df['error'] = train_df['target'] - train_df['oof_preds']\n\nsns.distplot(train_df['error'], ax = ax[1])\nax[1].set_title(\"Classification Errors: Target - Pred Probability\")\nax[1].axvline(x=.5, c='black')\nax[1].axvline(x=-.5, c='black')\nplt.tight_layout(pad=1)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"OOF Classification Report for Optimised Threshold: {:.3f}\".format(best_cutoff))\nprint(classification_report(y, (oof_preds > best_cutoff).astype(int), digits = 4))\nprint(f1_score(y, (oof_preds > cutoff).astype(int), average=f1_strategy))\n\nprint(\"\\nOOF Non-Optimised Cutoff (.5)\")\nprint(classification_report(y, (oof_preds > .5).astype(int), digits = 4))\nprint(f1_score(y, (oof_preds > .5).astype(int), average=f1_strategy))\n\ncnf_matrix = confusion_matrix(y, (oof_preds > .5).astype(int))\nprint(\"OOF Confusion Matrix\")\nprint(cnf_matrix)\nprint(\"OOF Normalised Confusion Matrix\")\nprint((cnf_matrix.astype('float') / cnf_matrix.sum(axis=1)[:, np.newaxis]).round(3))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Values above 0.5 signify false negatives, under -0.5 false positive."},{"metadata":{"trusted":true},"cell_type":"code","source":"show_cols = [\n    'id',\n    'keyword',\n    'location',\n    'text',\n    'target',\n    'oof_preds',\n    'error']\n\nprint(\"Look at False Negative\")\ndisplay(train_df[show_cols].sort_values(by = 'error', ascending=False).iloc[:20])\n\nprint(\"Look at False Positives\")\ndisplay(train_df[show_cols].sort_values(by = 'error', ascending=True).iloc[:20])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test_preds[:5])\nprint(test_preds[:5] / n_splits)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame.from_dict({\n    'id': test_df.id,\n    TARGET_COLUMN: ((test_preds / n_splits) > best_cutoff).astype(int)\n})\nsubmission.to_csv('submission_optimised_cutoff.csv', index=False)\nprint(submission[TARGET_COLUMN].value_counts(normalize = True).to_dict())\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame.from_dict({\n    'id': test_df.id,\n    TARGET_COLUMN: ((test_preds / n_splits) > .5).astype(int)\n})\nsubmission.to_csv('submission_fixed_cutoff.csv', index=False)\nprint(submission[TARGET_COLUMN].value_counts(normalize = True).to_dict())\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_pd = pd.DataFrame(oof_preds, columns = ['dense_oof'])\noof_pd.to_csv(\"oof_dense_nn.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Notebook Runtime: %0.2f Minutes\"%((time.time() - notebookstart)/60))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"}},"nbformat":4,"nbformat_minor":1}