disaster-nlp-keras-bert-using-tfhub.ipynb

{"cells":[{"metadata":{},"cell_type":"markdown","source":"# BERT for Disaster Text Problem\n_By Nick Brooks_\n\nPiggiebacking off of [xhulu's work](https://www.kaggle.com/xhlulu/disaster-nlp-keras-bert-using-tfhub) \n\n**References:** <br>\n- Source for `bert_encode` function: https://www.kaggle.com/user123454321/bert-starter-inference\n- All pre-trained BERT models from Tensorflow Hub: https://tfhub.dev/s?q=bert\n- TF Hub Documentation for Bert Model: https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1"},{"metadata":{"trusted":true},"cell_type":"code","source":"# We will use the official tokenization script created by the Google team\n!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input, Dropout, LSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate, SpatialDropout1D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nimport tensorflow_hub as hub\nfrom tensorflow.keras import callbacks\nimport matplotlib.pyplot as plt\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import StandardScaler\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer\nimport re\nimport nltk\nfrom scipy.sparse import hstack, csr_matrix\n\nimport tokenization","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def bert_encode(texts, tokenizer, max_len=512):\n    all_tokens = []\n    all_masks = []\n    all_segments = []\n    \n    for text in texts:\n        text = tokenizer.tokenize(text)\n            \n        text = text[:max_len-2]\n        input_sequence = [\"[CLS]\"] + text + [\"[SEP]\"]\n        pad_len = max_len - len(input_sequence)\n        \n        tokens = tokenizer.convert_tokens_to_ids(input_sequence)\n        tokens += [0] * pad_len\n        pad_masks = [1] * len(input_sequence) + [0] * pad_len\n        segment_ids = [0] * max_len\n        \n        all_tokens.append(tokens)\n        all_masks.append(pad_masks)\n        all_segments.append(segment_ids)\n    \n    return np.array(all_tokens), np.array(all_masks), np.array(all_segments)\n\ndef text_processing(df):\n    df['keyword'] = df['keyword'].str.replace(\"%20\", \" \")\n    df['hashtags'] = df['text'].apply(lambda x: \" \".join(re.findall(r\"#(\\w+)\", x)))\n    df['hash_loc_key'] = df[['hashtags', 'location','keyword']].astype(str).apply(lambda x: \" \".join(x), axis=1)\n    df['hash_loc_key'] = df[\"hash_loc_key\"].astype(str).str.lower().str.strip().fillna('nan')\n    \n    textfeats = ['hash_loc_key', 'text']\n    for cols in textfeats:\n        df[cols + '_num_words'] = df[cols].apply(lambda comment: len(comment.split())) # Count number of Words\n        df[cols + '_num_unique_words'] = df[cols].apply(lambda comment: len(set(w for w in comment.split())))\n        df[cols + '_words_vs_unique'] = df[cols+'_num_unique_words'] / df[cols+'_num_words'] * 100 # Count Unique Words\n        if cols == \"text\":\n            df[cols+\"_vader_Compound\"]= df[cols].apply(lambda x:SIA.polarity_scores(x)['compound'])\n\n    return df\n\ndef build_model(bert_layer, max_len=512, dropout=.2):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    input_mask = Input(shape=(max_len,), dtype=tf.int32, name=\"input_mask\")\n    segment_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"segment_ids\")\n    numeric_inputs = Input(shape=(len(num_cols),), dtype=tf.float32, name=\"numeric_inputs\")\n    \n    # Bert Layer\n    pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])\n    \n    # Sequence Output\n    sequence_output = SpatialDropout1D(dropout)(sequence_output)\n    sequence_output = Bidirectional(LSTM(128, return_sequences=True))(sequence_output)\n    sequence_output = GlobalAveragePooling1D()(sequence_output)\n    \n    # Pooled Output\n    pooled_output = Dense(36, activation='relu')(pooled_output)\n    \n    # Dense Inputs\n    numeric_x = Dense(512, activation='relu')(numeric_inputs)\n    numeric_x = Dropout(dropout)(numeric_x)\n    numeric_x = Dense(64, activation='relu')(numeric_x)\n    \n    # Concatenate\n    cat = concatenate([\n        pooled_output,\n        sequence_output,\n        numeric_x\n    ])\n    cat = Dropout(dropout)(cat)\n    \n    # Output Layer\n    out = Dense(1, activation='sigmoid')(cat)\n    \n    model = Model(inputs=[input_word_ids, input_mask, segment_ids, numeric_inputs], outputs=out)\n    model.compile(Adam(lr=1e-6), loss='binary_crossentropy', metrics=['acc'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load and Preprocess\n\n- Load BERT from the Tensorflow Hub\n- Load CSV files containing training data\n- Load tokenizer from the bert layer\n- Encode the text into tokens, masks, and segment flags"},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_LEN = 36\nBATCH_SIZE = 36\nEPOCHS = 12","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmodule_url = \"https://tfhub.dev/tensorflow/bert_en_uncased_L-24_H-1024_A-16/1\"\nbert_layer = hub.KerasLayer(module_url, trainable=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/nlp-getting-started/train.csv\")\ntest = pd.read_csv(\"/kaggle/input/nlp-getting-started/test.csv\")\nsubmission = pd.read_csv(\"/kaggle/input/nlp-getting-started/sample_submission.csv\")\n\nlength_info = [len(x) for x in np.concatenate([train.text.values, test.text.values])]\nprint(\"Train Sequence Length - Mean {:.1f} +/- {:.1f}, Max {:.1f}, Min {:.1f}\".format(\n    np.mean(length_info), np.std(length_info), np.max(length_info), np.min(length_info)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Text Processing\nSIA = SentimentIntensityAnalyzer()\ntrain_df = text_processing(train)\ntest_df = text_processing(test)\n\n# TF-IDF\ncount_vectorizer = TfidfVectorizer(\n    analyzer=\"word\",\n    tokenizer=nltk.word_tokenize,\n    preprocessor=None,\n    stop_words='english',\n    sublinear_tf=True,\n    ngram_range=(1, 1),\n    max_features=500)    \n\nhash_loc_tfidf = count_vectorizer.fit(train_df['hash_loc_key'])\ntfvocab = hash_loc_tfidf.get_feature_names()\nprint(\"Number of TF-IDF Features: {}\".format(len(tfvocab)))\n\ntrain_tfidf = count_vectorizer.transform(train_df['hash_loc_key'])\ntest_tfidf = count_vectorizer.transform(test_df['hash_loc_key'])\n\n# Sparse Stack Numerical and TFIDF\ndense_vars = [\n    'hash_loc_key_num_words',\n    'hash_loc_key_num_unique_words',\n    'hash_loc_key_words_vs_unique',\n    'text_num_words',\n    'text_num_unique_words',\n    'text_words_vs_unique',\n    'text_vader_Compound']\n\n# Normalisation - Standard Scaler\nfor d_i in dense_vars:\n    scaler = StandardScaler()\n    scaler.fit(train_df.loc[:,d_i].values.reshape(-1, 1))\n    train_df.loc[:,d_i] = scaler.transform(train_df.loc[:,d_i].values.reshape(-1, 1))\n    test_df.loc[:,d_i] = scaler.transform(test_df.loc[:,d_i].values.reshape(-1, 1))\n    \n# Sparse Stack\ntrain_num = hstack([csr_matrix(train_df.loc[:,dense_vars].values),train_tfidf]).toarray()\ntest_num = hstack([csr_matrix(test_df.loc[:,dense_vars].values),test_tfidf]).toarray()\nnum_cols = train_df[dense_vars].columns.tolist() + tfvocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Bert Pre-Processing\nvocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\ndo_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\ntokenizer = tokenization.FullTokenizer(vocab_file, do_lower_case)\n\ntrain_input = (*bert_encode(train.text.values, tokenizer, max_len=MAX_LEN), train_num)\ntest_input = (*bert_encode(test.text.values, tokenizer, max_len=MAX_LEN), test_num)\ntrain_labels = train.target.values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model: Build, Train, Predict, Submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_model(bert_layer, max_len=MAX_LEN)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"es = callbacks.EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=4, verbose=1,\n                             mode='min', baseline=None, restore_best_weights=False)\nrlr = callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=2, min_lr=1e-7,\n                                  mode='min', verbose=1)\ncheckpoint_1 = tf.keras.callbacks.ModelCheckpoint('model_loss.h5', monitor='val_loss', save_best_only=True)\n# checkpoint_2 = tf.keras.callbacks.ModelCheckpoint('model_acc.h5', monitor='val_acc', save_best_only=True)\n\n\nhistory = model.fit(\n    train_input,\n    train_labels,\n    validation_split=0.2,\n    epochs=EPOCHS,\n    batch_size=BATCH_SIZE,\n    callbacks=[checkpoint_1,  rlr]\n)\n\nplot_metrics = ['loss','acc']\ni = \"None\"\nf, ax = plt.subplots(1,len(plot_metrics),figsize = [12,4])\nfor p_i,metric in enumerate(plot_metrics):\n    ax[p_i].plot(history.history[metric], label='Train ' + metric)\n    ax[p_i].plot(history.history['val_' + metric], label='Val ' + metric)\n    ax[p_i].set_title(\"{} Fold Loss Curve - {}\".format(i, metric))\n    ax[p_i].legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('model_loss.h5')\ntest_pred = model.predict(test_input, batch_size=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission['target'] = test_pred.round().astype(int)\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}