Merge pull request #279 from JohnSnowLabs/nlu/annos

Added FewShotAssertionClassifier
JohnSnowLabs · Sep 16, 2024 · 9cbec12 · 9cbec12
2 parents 2cdd142 + ea20ceb
commit 9cbec12
Show file tree

Hide file tree

Showing 48 changed files with 52,040 additions and 4,969 deletions.
diff --git a/...g/binary_text_classification/NLU_training_negation_classifier_demo_biological_texts.ipynb b/...g/binary_text_classification/NLU_training_negation_classifier_demo_biological_texts.ipynb
diff --git a/...ing/binary_text_classification/NLU_training_sarcasam_classifier_demo_news_headlines.ipynb b/...ing/binary_text_classification/NLU_training_sarcasam_classifier_demo_news_headlines.ipynb
diff --git a/...es/colab/Training/binary_text_classification/NLU_training_sentiment_classifier_demo.ipynb b/...es/colab/Training/binary_text_classification/NLU_training_sentiment_classifier_demo.ipynb
diff --git a/...lab/Training/binary_text_classification/NLU_training_sentiment_classifier_demo_IMDB.ipynb b/...lab/Training/binary_text_classification/NLU_training_sentiment_classifier_demo_IMDB.ipynb
diff --git a/...ing/binary_text_classification/NLU_training_sentiment_classifier_demo_apple_twitter.ipynb b/...ing/binary_text_classification/NLU_training_sentiment_classifier_demo_apple_twitter.ipynb
diff --git a/...Training/binary_text_classification/NLU_training_sentiment_classifier_demo_covid_19.ipynb b/...Training/binary_text_classification/NLU_training_sentiment_classifier_demo_covid_19.ipynb
diff --git a/...ng/binary_text_classification/NLU_training_sentiment_classifier_demo_finanical_news.ipynb b/...ng/binary_text_classification/NLU_training_sentiment_classifier_demo_finanical_news.ipynb
diff --git a/...binary_text_classification/NLU_training_sentiment_classifier_demo_natural_disasters.ipynb b/...binary_text_classification/NLU_training_sentiment_classifier_demo_natural_disasters.ipynb
diff --git a/...b/Training/binary_text_classification/NLU_training_sentiment_classifier_demo_reddit.ipynb b/...b/Training/binary_text_classification/NLU_training_sentiment_classifier_demo_reddit.ipynb
diff --git a/...ning/binary_text_classification/NLU_training_sentiment_classifier_demo_stock_market.ipynb b/...ning/binary_text_classification/NLU_training_sentiment_classifier_demo_stock_market.ipynb
diff --git a/.../Training/binary_text_classification/NLU_training_sentiment_classifier_demo_twitter.ipynb b/.../Training/binary_text_classification/NLU_training_sentiment_classifier_demo_twitter.ipynb
diff --git a/examples/colab/Training/entity_resolution/sentence_entity_resolution_training.ipynb b/examples/colab/Training/entity_resolution/sentence_entity_resolution_training.ipynb
diff --git a/examples/colab/Training/named_entity_recognition/NLU_training_NER_demo.ipynb b/examples/colab/Training/named_entity_recognition/NLU_training_NER_demo.ipynb
diff --git a/examples/colab/Training/part_of_speech/NLU_training_POS_demo.ipynb b/examples/colab/Training/part_of_speech/NLU_training_POS_demo.ipynb
diff --git a/.../colab/component_examples/text_pre_processing_and_cleaning/NLU_tokenization_example.ipynb b/.../colab/component_examples/text_pre_processing_and_cleaning/NLU_tokenization_example.ipynb
diff --git a/.../colab/component_examples/text_pre_processing_and_cleaning/document_normalizer_demo.ipynb b/.../colab/component_examples/text_pre_processing_and_cleaning/document_normalizer_demo.ipynb
diff --git a/...xamples/word_embeddings/NLU_BIOBERT_word_embeddings_and_t-SNE_visualization_example.ipynb b/...xamples/word_embeddings/NLU_BIOBERT_word_embeddings_and_t-SNE_visualization_example.ipynb
diff --git a/...t_examples/word_embeddings/NLU_ELMo_word_embeddings_and_t-SNE_visualization_example.ipynb b/...t_examples/word_embeddings/NLU_ELMo_word_embeddings_and_t-SNE_visualization_example.ipynb
diff --git a/..._examples/word_embeddings/NLU_XLNET_word_embeddings_and_t-SNE_visualization_example.ipynb b/..._examples/word_embeddings/NLU_XLNET_word_embeddings_and_t-SNE_visualization_example.ipynb
diff --git a/examples/colab/healthcare/assertion/assertion_overview.ipynb b/examples/colab/healthcare/assertion/assertion_overview.ipynb
@@ -32,31 +32,6 @@
     "| English  | [assert.large](https://nlp.johnsnowlabs.com/2020/05/21/assertion_dl_large_en.html) | [assertion_dl_large](https://nlp.johnsnowlabs.com/2020/05/21/assertion_dl_large_en.html)                   |"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {
-    "id": "OMiHjCtQhF_r"
-   },
-   "outputs": [],
-   "source": [
-    "# Install the johnsnowlabs library\n",
-    "! pip install -q johnsnowlabs==5.1.0"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "fLmvgQZoRucE"
-   },
-   "outputs": [],
-   "source": [
-    "from google.colab import files\n",
-    "print('Please Upload your John Snow Labs License using the button below')\n",
-    "license_keys = files.upload()"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -71,29 +46,6 @@
     "nlp.install()"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": 4,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "id": "cPT4lI0x8ZOj",
-    "outputId": "13a9978d-8d0d-4672-b70e-eb574a66f303"
-   },
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "👌 Launched \u001b[92mcpu optimized\u001b[39m session with with: 🚀Spark-NLP==5.1.0, 💊Spark-Healthcare==5.1.0, running on ⚡ PySpark==3.1.2\n"
-     ]
-    }
-   ],
-   "source": [
-    "spark=nlp.start()"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {

diff --git a/examples/colab/healthcare/assertion/few_shot_assertion_classifier.ipynb b/examples/colab/healthcare/assertion/few_shot_assertion_classifier.ipynb
@@ -0,0 +1,171 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "source": [
+    "![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)\n",
+    "\n",
+    "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/nlu/blob/master/examples/colab/healthcare/assertion/few_shot_assertion_classifier.ipynb)\n"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "### Assign assertion status to clinical entities\n",
+    "\n",
+    "Classes are: `absent`, `present`, `conditional`, `associated_with_someone_else`, `hypothetical`, `possible`"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Starting the session"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "from johnsnowlabs import nlp\n",
+    "nlp.install()\n",
+    "nlp.start()"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## Loading the data"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "outputs": [],
+   "source": [
+    "data = (\n",
+    "            \"\"\"Includes hypertension and chronic obstructive pulmonary disease.\"\"\"\n",
+    "        )"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-09-09T14:02:59.655452100Z",
+     "start_time": "2024-09-09T14:02:59.634164800Z"
+    }
+   }
+  },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "## [Assign assertion status to clinical entities](https://nlp.johnsnowlabs.com/2024/04/15/assertion_fewshotclassifier_en.html)"
+   ],
+   "metadata": {
+    "collapsed": false
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Warning::Spark Session already created, some configs may not take.\n",
+      "Warning::Spark Session already created, some configs may not take.\n",
+      "assertion_fewshotclassifier download started this may take some time.\n",
+      "[OK!]\n",
+      "sentence_detector_dl download started this may take some time.\n",
+      "Approximate size to download 354.6 KB\n",
+      "[OK!]\n",
+      "pos_anc download started this may take some time.\n",
+      "Approximate size to download 3.9 MB\n",
+      "[OK!]\n",
+      "Warning::Spark Session already created, some configs may not take.\n"
+     ]
+    }
+   ],
+   "source": [
+    "res = nlp.load(\"en.few_assert_shot_classifier\").predict(data, output_level='chunk')"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-09-09T14:22:02.470834500Z",
+     "start_time": "2024-09-09T14:21:37.585408700Z"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "outputs": [
+    {
+     "data": {
+      "text/plain": "  assertion assertion_confidence  \\\n0   present                  1.0   \n0   present                  1.0   \n\n                                            document   matched_pos  \\\n0  Includes hypertension and chronic obstructive ...  hypertension   \n0  Includes hypertension and chronic obstructive ...       disease   \n\n                                pos  \n0  [VBZ, NN, CC, JJ, JJ, JJ, NN, .]  \n0  [VBZ, NN, CC, JJ, JJ, JJ, NN, .]  ",
+      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>assertion</th>\n      <th>assertion_confidence</th>\n      <th>document</th>\n      <th>matched_pos</th>\n      <th>pos</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>present</td>\n      <td>1.0</td>\n      <td>Includes hypertension and chronic obstructive ...</td>\n      <td>hypertension</td>\n      <td>[VBZ, NN, CC, JJ, JJ, JJ, NN, .]</td>\n    </tr>\n    <tr>\n      <th>0</th>\n      <td>present</td>\n      <td>1.0</td>\n      <td>Includes hypertension and chronic obstructive ...</td>\n      <td>disease</td>\n      <td>[VBZ, NN, CC, JJ, JJ, JJ, NN, .]</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "res"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "ExecuteTime": {
+     "end_time": "2024-09-09T14:22:02.515663Z",
+     "start_time": "2024-09-09T14:22:02.478142Z"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [],
+   "metadata": {
+    "collapsed": false
+   }
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 2
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython2",
+   "version": "2.7.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}
diff --git a/examples/colab/healthcare/de_identification/DeIdentification_model_overview.ipynb b/examples/colab/healthcare/de_identification/DeIdentification_model_overview.ipynb
@@ -54,31 +54,6 @@
     "| English  | classify.pico | classifierdl_pico_biobert          |"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "OMiHjCtQhF_r"
-   },
-   "outputs": [],
-   "source": [
-    "# Install the johnsnowlabs library\n",
-    "! pip install -q johnsnowlabs==5.1.0"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "WzQxxFcyBWJ6"
-   },
-   "outputs": [],
-   "source": [
-    "from google.colab import files\n",
-    "print('Please Upload your John Snow Labs License using the button below')\n",
-    "license_keys = files.upload()"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -93,17 +68,6 @@
     "nlp.install()"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "mxj5X9miBZoX"
-   },
-   "outputs": [],
-   "source": [
-    "spark=nlp.start()"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {

diff --git a/examples/colab/healthcare/drug_normalization/drug_norm.ipynb b/examples/colab/healthcare/drug_normalization/drug_norm.ipynb
@@ -19,31 +19,6 @@
     "See the [install docs](https://nlu.johnsnowlabs.com/docs/en/install#super-quickstart-on-google-colab-or-kaggle)\n"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "M4SGENPXisd7"
-   },
-   "outputs": [],
-   "source": [
-    "# Install the johnsnowlabs library\n",
-    "! pip install -q johnsnowlabs==5.1.0"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "DMHSgj8EDlUt"
-   },
-   "outputs": [],
-   "source": [
-    "from google.colab import files\n",
-    "print('Please Upload your John Snow Labs License using the button below')\n",
-    "license_keys = files.upload()"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -58,17 +33,6 @@
     "nlp.install()"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "id": "IxMbCELVD5jj"
-   },
-   "outputs": [],
-   "source": [
-    "spark=nlp.start()"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {

diff --git a/examples/colab/healthcare/entity_resolution/NLU_atc_resolver_pipeline.ipynb b/examples/colab/healthcare/entity_resolution/NLU_atc_resolver_pipeline.ipynb
diff --git a/examples/colab/healthcare/entity_resolution/NLU_hpo_resolver_pipeline.ipynb b/examples/colab/healthcare/entity_resolution/NLU_hpo_resolver_pipeline.ipynb