bert-oof-with-dense-features.ipynb

{"cells":[{"metadata":{},"cell_type":"markdown","source":"# BERT for Disaster Text Problem\n_By Nick Brooks_\n\n### **Goal:**\nExperiment with Bert, LSTM, Pooling, and Dense Features <br>\nPiggiebacking off of [xhulu's work](https://www.kaggle.com/xhlulu/disaster-nlp-keras-bert-using-tfhub) \n\n### **References:**\n- Source for `bert_encode` function: https://www.kaggle.com/user123454321/bert-starter-inference\n- All pre-trained BERT models from Tensorflow Hub: https://tfhub.dev/s?q=bert\n- TF Hub Documentation for Bert Model: https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"},{"metadata":{"trusted":true},"cell_type":"code","source":"# We will use the official tokenization script created by the Google team\n!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input, Dropout, LSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate, SpatialDropout1D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nimport tensorflow_hub as hub\nfrom tensorflow.keras import callbacks\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import StandardScaler\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer\nfrom nltk import word_tokenize\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.model_selection import KFold\nfrom sklearn.metrics import f1_score, precision_score, recall_score, confusion_matrix\nfrom sklearn.metrics import classification_report, confusion_matrix\nimport pprint\n\nimport tokenization\n\nimport re\nimport gc\nimport time\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nsns.set_style(\"whitegrid\")\nnotebookstart = time.time()\npd.options.display.max_colwidth = 500\n\nprint(\"Tensorflow Version: \", tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_LEN = 36\nBATCH_SIZE = 36\nEPOCHS = 5\nSEED = 42\nNROWS = None\nf1_strategy = 'macro'\nTARGET_COLUMN = 'target'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def bert_encode(texts, tokenizer, max_len=512):\n    all_tokens = []\n    all_masks = []\n    all_segments = []\n    \n    for text in texts:\n        text = tokenizer.tokenize(text)\n            \n        text = text[:max_len-2]\n        input_sequence = [\"[CLS]\"] + text + [\"[SEP]\"]\n        pad_len = max_len - len(input_sequence)\n        \n        tokens = tokenizer.convert_tokens_to_ids(input_sequence)\n        tokens += [0] * pad_len\n        pad_masks = [1] * len(input_sequence) + [0] * pad_len\n        segment_ids = [0] * max_len\n        \n        all_tokens.append(tokens)\n        all_masks.append(pad_masks)\n        all_segments.append(segment_ids)\n    \n    return np.array(all_tokens), np.array(all_masks), np.array(all_segments)\n\ndef text_processing(df):\n    df['keyword'] = df['keyword'].str.replace(\"%20\", \" \")\n    df['hashtags'] = df['text'].apply(lambda x: \" \".join(re.findall(r\"#(\\w+)\", x)))\n    df['hash_loc_key'] = df[['hashtags', 'location','keyword']].astype(str).apply(lambda x: \" \".join(x), axis=1)\n    df['hash_loc_key'] = df[\"hash_loc_key\"].astype(str).str.lower().str.strip().fillna('nan')\n    \n    textfeats = ['hash_loc_key', 'text']\n    for cols in textfeats:\n        df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.split())) # Count number of Words\n        df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))\n        df[cols + '_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words'] * 100 # Count Unique Words\n        if cols == \"text\":\n            df[cols+\"_vader_Compound\"]= df[cols].apply(lambda x:SIA.polarity_scores(x)['compound'])\n\n    return df\n\ndef build_model(bert_layer, max_len=512, dropout=.2):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    input_mask = Input(shape=(max_len,), dtype=tf.int32, name=\"input_mask\")\n    segment_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"segment_ids\")\n    numeric_inputs = Input(shape=(len(num_cols),), dtype=tf.float32, name=\"numeric_inputs\")\n    \n    # Bert Layer\n    pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])\n    \n    # Sequence Output\n    sequence_output = SpatialDropout1D(dropout)(sequence_output)\n    sequence_output = Bidirectional(LSTM(128, return_sequences=True))(sequence_output)\n    sequence_output = GlobalAveragePooling1D()(sequence_output)\n    \n    # Pooled Output\n    pooled_output = Dense(36, activation='relu')(pooled_output)\n    \n    # Dense Inputs\n    numeric_x = Dense(512, activation='relu')(numeric_inputs)\n    numeric_x = Dropout(dropout)(numeric_x)\n    numeric_x = Dense(64, activation='relu')(numeric_x)\n    \n    # Concatenate\n    cat = concatenate([\n        pooled_output,\n        sequence_output,\n        numeric_x\n    ])\n    cat = Dropout(dropout)(cat)\n    \n    # Output Layer\n    out = Dense(1, activation='sigmoid')(cat)\n    \n    model = Model(inputs=[input_word_ids, input_mask, segment_ids, numeric_inputs], outputs=out)\n    model.compile(Adam(lr=1e-6), loss='binary_crossentropy', metrics=['acc'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load and Preprocess\n\n- Load BERT from the Tensorflow Hub\n- Load CSV files containing training data\n- Load tokenizer from the bert layer\n- Encode the text into tokens, masks, and segment flags"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodule_url = \"https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1\"\nbert_layer = hub.KerasLayer(module_url, trainable=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/nlp-getting-started/train.csv\", nrows=NROWS)\ntest = pd.read_csv(\"/kaggle/input/nlp-getting-started/test.csv\", nrows=NROWS)\ntestdex = test.id\nsubmission = pd.read_csv(\"/kaggle/input/nlp-getting-started/sample_submission.csv\", nrows=NROWS)\n\nprint(\"Train Shape: {} Rows, {} Columns\".format(*train.shape))\nprint(\"Test Shape: {} Rows, {} Columns\".format(*test.shape))\n\nlength_info = [len(x) for x in np.concatenate([train.text.values, test.text.values])]\nprint(\"Train Sequence Length - Mean {:.1f} +/- {:.1f}, Max {:.1f}, Min {:.1f}\".format(\n    np.mean(length_info), np.std(length_info), np.max(length_info), np.min(length_info)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Text Processing\nSIA = SentimentIntensityAnalyzer()\ntrain_df = text_processing(train)\ntest_df = text_processing(test)\n\n# TF-IDF\ncount_vectorizer = TfidfVectorizer(\n    analyzer=\"word\",\n    tokenizer=word_tokenize,\n    preprocessor=None,\n    stop_words='english',\n    sublinear_tf=True,\n    ngram_range=(1, 1),\n    max_features=500)    \n\nhash_loc_tfidf = count_vectorizer.fit(train_df['hash_loc_key'])\ntfvocab = hash_loc_tfidf.get_feature_names()\nprint(\"Number of TF-IDF Features: {}\".format(len(tfvocab)))\n\ntrain_tfidf = count_vectorizer.transform(train_df['hash_loc_key'])\ntest_tfidf = count_vectorizer.transform(test_df['hash_loc_key'])\n\n# Sparse Stack Numerical and TFIDF\ndense_vars = [\n    'hash_loc_key_num_words',\n    'hash_loc_key_num_unique_words',\n    'hash_loc_key_words_vs_unique',\n    'text_num_words',\n    'text_num_unique_words',\n    'text_words_vs_unique',\n    'text_vader_Compound']\n\n# Normalisation - Standard Scaler\nfor d_i in dense_vars:\n    scaler = StandardScaler()\n    scaler.fit(train_df.loc[:,d_i].values.reshape(-1, 1))\n    train_df.loc[:,d_i] = scaler.transform(train_df.loc[:,d_i].values.reshape(-1, 1))\n    test_df.loc[:,d_i] = scaler.transform(test_df.loc[:,d_i].values.reshape(-1, 1))\n    \n# Sparse Stack\ntrain_num = hstack([csr_matrix(train_df.loc[:,dense_vars].values),train_tfidf]).toarray()\ntest_num = hstack([csr_matrix(test_df.loc[:,dense_vars].values),test_tfidf]).toarray()\nnum_cols = train_df[dense_vars].columns.tolist() + tfvocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Bert Pre-Processing\nvocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\ndo_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\ntokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)\n\ntrain_input_word_ids, train_input_mask, train_segment_ids, train_numeric_inputs = *bert_encode(train.text.values, tokenizer, max_len=MAX_LEN), train_num\ntest_input = (*bert_encode(test.text.values, tokenizer, max_len=MAX_LEN), test_num)\ntrain_labels = train.target.values\n\ndel test, train_num, test_num, train_df, test_df\n_ = gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model: Build, Train, Predict, Submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_model(bert_layer, max_len=MAX_LEN)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_preds = np.zeros(train_input_word_ids.shape[0])\ntest_preds = np.zeros(test_input[0].shape[0])\n\nn_splits = 3\nfolds = KFold(n_splits=n_splits, shuffle=True, random_state=SEED)\nplot_metrics = ['loss','acc']\n\nfold_hist = {}\nfor i, (trn_idx, val_idx) in enumerate(folds.split(train_input_word_ids)):\n    modelstart = time.time()\n    model = build_model(bert_layer, max_len=MAX_LEN)\n    \n    es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=4, verbose=1,\n                                 mode='min', baseline=None, restore_best_weights=True)\n    \n    \n    history = model.fit(\n        x=[train_input_word_ids[trn_idx],\n            train_input_mask[trn_idx],\n            train_segment_ids[trn_idx],\n            train_numeric_inputs[trn_idx]],\n        y=train_labels[trn_idx],\n        validation_data=(\n            [train_input_word_ids[val_idx],\n            train_input_mask[val_idx],\n            train_segment_ids[val_idx],\n            train_numeric_inputs[val_idx]],\n            train_labels[val_idx]),\n        epochs=EPOCHS,\n        batch_size=18,\n        callbacks=[es]\n    )\n\n    best_index = np.argmin(history.history['val_loss'])\n    fold_hist[i] = history\n    \n    oof_preds[val_idx] = model.predict(\n        [train_input_word_ids[val_idx],\n        train_input_mask[val_idx],\n        train_segment_ids[val_idx],\n        train_numeric_inputs[val_idx]]).ravel()\n    test_preds += model.predict(test_input).ravel()\n    f1_sc = f1_score(train_labels[val_idx], (oof_preds[val_idx] > 0.5).astype(int), average=f1_strategy)\n    print(\"\\nFOLD {} COMPLETE in {:.1f} Minutes - Avg F1 {:.5f} - Best Epoch {}\".format(i, (time.time() - modelstart)/60, f1_sc, best_index + 1))\n    best_metrics = {metric: scores[best_index] for metric, scores in history.history.items()}\n    pprint.pprint(best_metrics)\n    \n    f, ax = plt.subplots(1,len(plot_metrics),figsize = [12,4])\n    for p_i,metric in enumerate(plot_metrics):\n        ax[p_i].plot(history.history[metric], label='Train ' + metric)\n        ax[p_i].plot(history.history['val_' + metric], label='Val ' + metric)\n        ax[p_i].set_title(\"{} Fold Loss Curve - {}\\nBest Epoch {}\".format(i, metric, best_index))\n        ax[p_i].legend()\n        ax[p_i].axvline(x=best_index, c='black')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# OOF F1 Cutoff\nsave_f1_opt = []\nfor cutoff in np.arange(.38,.62, .01):\n    save_f1_opt.append([cutoff, f1_score(train_labels, (oof_preds > cutoff).astype(int), average=f1_strategy)])\nf1_pd = pd.DataFrame(save_f1_opt, columns = ['cutoff', 'f1_score'])\n\nbest_cutoff = f1_pd.loc[f1_pd['f1_score'].idxmax(),'cutoff']\nprint(\"F1 Score: {:.4f}, Optimised Cufoff: {:.2f}\".format(f1_pd.loc[f1_pd['f1_score'].idxmax(),'f1_score'], best_cutoff))\n\nf,ax = plt.subplots(1,2,figsize = [10,4])\n\nax[0].plot(f1_pd['cutoff'], f1_pd['f1_score'], c = 'red')\nax[0].set_ylabel(\"F1 Score\")\nax[0].set_xlabel(\"Cutoff\")\nax[0].axvline(x=best_cutoff, c='black')\nax[0].set_title(\"F1 Score and Cutoff on OOF\")\n\n\ntrain['oof_preds'] = oof_preds\ntrain['error'] = train['target'] - train['oof_preds']\n\nsns.distplot(train['error'], ax = ax[1])\nax[1].set_title(\"Classification Errors: Target - Pred Probability\")\nax[1].axvline(x=.5, c='black')\nax[1].axvline(x=-.5, c='black')\nplt.tight_layout(pad=1)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"OOF Classification Report for Optimised Threshold: {:.3f}\".format(best_cutoff))\nprint(classification_report(train_labels, (oof_preds > best_cutoff).astype(int), digits = 4))\nprint(f1_score(train_labels, (oof_preds > cutoff).astype(int), average=f1_strategy))\n\nprint(\"\\nOOF Non-Optimised Cutoff (.5)\")\nprint(classification_report(train_labels, (oof_preds > .5).astype(int), digits = 4))\nprint(f1_score(train_labels, (oof_preds > .5).astype(int), average=f1_strategy))\n\ncnf_matrix = confusion_matrix(train_labels, (oof_preds > .5).astype(int))\nprint(\"OOF Confusion Matrix\")\nprint(cnf_matrix)\nprint(\"OOF Normalised Confusion Matrix\")\nprint((cnf_matrix.astype('float') / cnf_matrix.sum(axis=1)[:, np.newaxis]).round(3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_cols = [\n    'id',\n    'keyword',\n    'location',\n    'text',\n    'target',\n    'oof_preds',\n    'error']\n\nprint(\"Look at False Negative\")\ndisplay(train[show_cols].sort_values(by = 'error', ascending=False).iloc[:20])\n\nprint(\"Look at False Positives\")\ndisplay(train[show_cols].sort_values(by = 'error', ascending=True).iloc[:20])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame.from_dict({\n    'id': testdex,\n    TARGET_COLUMN: ((test_preds / n_splits) > .5).astype(int)\n})\nsubmission.to_csv('submission_fixed_cutoff.csv', index=False)\nprint(submission[TARGET_COLUMN].value_counts(normalize = True).to_dict())\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame.from_dict({\n    'id': testdex,\n    TARGET_COLUMN: ((test_preds / n_splits) > best_cutoff).astype(int)\n})\nsubmission.to_csv('submission_optimised_cutoff.csv', index=False)\nprint(submission[TARGET_COLUMN].value_counts(normalize = True).to_dict())\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oof_pd = pd.DataFrame(oof_preds, columns = ['dense_oof'])\noof_pd.to_csv(\"oof_dense_bert.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Notebook Runtime: %0.2f Minutes\"%((time.time() - notebookstart)/60))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}