From 4a0498c2493c7bfeabf6c07daba396abb3d1753c Mon Sep 17 00:00:00 2001 From: C-K-Loan Date: Thu, 3 Jun 2021 04:19:10 +0200 Subject: [PATCH 1/3] Nav Updated --- docs/_data/navigation.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/_data/navigation.yml b/docs/_data/navigation.yml index 0fa95197..1aecfb9a 100644 --- a/docs/_data/navigation.yml +++ b/docs/_data/navigation.yml @@ -4,6 +4,16 @@ header: - title: Docs url: /docs/en/install key: docs + - title: Visualizations + url: /docs/en/viz_examples + key: viz + - title: Streamlit & NLU + url: /docs/en/streamlit_viz_examples + key: streamlit_viz + - title: Tutorial Notebooks + url: /docs/en/notebooks + key: tutorial_notebooks + # - title: Articles # url: /articles # key: articles From eeb5445add941c1f07b631f18f26cf73a52b493e Mon Sep 17 00:00:00 2001 From: C-K-Loan Date: Thu, 3 Jun 2021 04:19:33 +0200 Subject: [PATCH 2/3] Streamlit docs added --- docs/en/streamlit_viz_examples.md | 329 ++++++++++++++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100644 docs/en/streamlit_viz_examples.md diff --git a/docs/en/streamlit_viz_examples.md b/docs/en/streamlit_viz_examples.md new file mode 100644 index 00000000..2d173958 --- /dev/null +++ b/docs/en/streamlit_viz_examples.md @@ -0,0 +1,329 @@ +--- +layout: docs +header: true +title: NLU& Streamlit visualization Examples +key: streamlit-viz-examples +permalink: /docs/en/streamlit_viz_examples +modify_date: "2019-05-16" +--- + +
+ + +This page contains examples and tutorials on how to visualize the 1000+ state-of-the-art NLP models provided by NLU in *just 1 line of code* in `streamlit`. +It includes simple `1-liners` you can sprinkle into your Streamlit app to for features like **Dependency Trees, Named Entities (NER), text classification results, semantic simmilarity, +embedding visualizations via ELMO, BERT, ALBERT, XLNET and much more** . Additionally, improvements for T5 and various resolvers have been added. + +This is the ultimate NLP research tool. You can visualize and compare the results of hundreds of context aware deep learning embeddings and compare them with classical vanilla embeddings like Glove +and can see with your own eyes how context is encoded by transformer models like `BERT` or `XLNET`and many more ! +Besides that, you can also compare the results of the 200+ NER models John Snow Labs provides and see how peformances changes with varrying ebeddings, like Contextual, Static and Domain Specific Embeddings. + +## Install +[For detailed instructions refer to the NLU install documentation here](https://nlu.johnsnowlabs.com/docs/en/install) +You need `Open JDK 8` installed and the following python packages +```bash +pip install nlu streamlit pyspark==3.0.1 sklearn plotly +``` +Problems? [Connect with us on Slack!](https://join.slack.com/t/spark-nlp/shared_invite/zt-lutct9gm-kuUazcyFKhuGY3_0AMkxqA) + +## Impatient and want some action? +Just run this Streamlit app, you can use it to generate python code for each NLU-Streamlit building block +```shell +streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/01_dashboard.py +``` + +## Quick Starter cheat sheet - All you need to know in 1 picture for NLU + Streamlit +For NLU models to load, see [the NLU Namespace](https://nlu.johnsnowlabs.com/docs/en/namespace) or the [John Snow Labs Modelshub](https://modelshub.johnsnowlabs.com/models) or go [straight to the source](https://github.com/JohnSnowLabs/nlu/blob/3.0.2rc1/nlu/namespace.py). +![NLU Streamlit Cheatsheet](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/img/NLU_Streamlit_Cheetsheet.png) + + +## Examples +Just try out any of these. +You can use the first example to generate python-code snippets which you can +recycle as building blocks in your streamlit apps! +### Example: [`01_dashboard`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/01_dashboard.py) +```shell +streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/01_dashboard.py +``` +### Example: [`02_NER`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/02_NER.py) +```shell +streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/02_NER.py +``` +### Example: [`03_text_similarity_matrix`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/03_text_similarity_matrix.py) +```shell +streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/03_text_similarity_matrix.py +``` + + +### Example: [`04_dependency_tree`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/04_dependency_tree.py) +```shell +streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/04_dependency_tree.py +``` + +### Example: [`05_classifiers`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/05_classifiers.py) +```shell +streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/05_classifiers.py +``` + +### Example: [`06_token_features`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/06_token_features.py) +```shell +streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/06_token_features.py +``` + +## How to use NLU? +All you need to know about NLU is that there is the [`nlu.load()`](https://nlu.johnsnowlabs.com/docs/en/load_api) method which returns a `NLUPipeline` object +which has a [`.predict()`](https://nlu.johnsnowlabs.com/docs/en/predict_api) that works on most [common data types in the pydata stack like Pandas dataframes](https://nlu.johnsnowlabs.com/docs/en/predict_api#supported-data-types) . +Ontop of that, there are various visualization methods a NLUPipeline provides easily integrate in Streamlit as re-usable components. [`viz() method`](https://nlu.johnsnowlabs.com/docs/en/viz_examples) + + + + + +### Overview of NLU + Streamlit buildingblocks + +|Method | Description | +|---------------------------------------------------------------|-------------------------------------------| +| [`nlu.load('').predict(data)`](TODO.com) | Load any of the [1000+ models](https://nlp.johnsnowlabs.com/models) by providing the model name any predict on most Pythontic [data strucutres like Pandas, strings, arrays of strings and more](https://nlu.johnsnowlabs.com/docs/en/predict_api#supported-data-types) | +| [`nlu.load('').viz_streamlit(data)`](TODO.com) | Display full NLU exploration dashboard, that showcases every feature avaiable with dropdown selectors for 1000+ models| +| [`nlu.load('').viz_streamlit_similarity([string1, string2])`](TODO.com) | Display similarity matrix and scalar similarity for every word embedding loaded and 2 strings. | +| [`nlu.load('').viz_streamlit_ner(data)`](TODO.com) | Visualize predicted NER tags from Named Entity Recognizer model| +| [`nlu.load('').viz_streamlit_dep_tree(data)`](TODO.com) | Visualize Dependency Tree together with Part of Speech labels| +| [`nlu.load('').viz_streamlit_classes(data)`](TODO.com) | Display all extracted class features and confidences for every classifier loaded in pipeline| +| [`nlu.load('').viz_streamlit_token(data)`](TODO.com) | Display all detected token features and informations in Streamlit | +| [`nlu.load('').viz(data, write_to_streamlit=True)`](TODO.com) | Display the raw visualization without any UI elements. See [viz docs for more info](https://nlu.johnsnowlabs.com/docs/en/viz_examples). By default all aplicable nlu model references will be shown. | +| [`nlu.enable_streamlit_caching()`](#test) | Enable caching the `nlu.load()` call. Once enabled, the `nlu.load()` method will automatically cached. **This is recommended** to run first and for large peformance gans | +

TODO ANCHORSZZ

+ + +# Detailed visualizer information and API docs + +## function `pipe.viz_streamlit` + + +Display a highly configurable UI that showcases almost every feature available for Streamlit visualization with model selection dropdowns in your applications. +Ths includes : +- `Similarity Matrix` & `Scalars` & `Embedding Information` for any of the [100+ Word Embedding Models]() +- `NER visualizations` for any of the [200+ Named entity recognizers]() +- `Labled` & `Unlabled Dependency Trees visualizations` with `Part of Speech Tags` for any of the [100+ Part of Speech Models]() +- `Token informations` predicted by any of the [1000+ models]() +- `Classification results` predicted by any of the [100+ models classification models]() +- `Pipeline Configuration` & `Model Information` & `Link to John Snow Labs Modelshub` for all loaded pipelines +- `Auto generate Python code` that can be copy pasted to re-create the individual Streamlit visualization blocks. + NlLU takes the first model specified as `nlu.load()` for the first visualization run. + Once the Streamlit app is running, additional models can easily be added via the UI. + It is recommended to run this first, since you can generate Python code snippets `to recreate individual Streamlit visualization blocks` + +```python +nlu.load('ner').viz_streamlit(['I love NLU and Streamlit!','I hate buggy software']) +``` + + + +![NLU Streamlit UI Overview](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/gif/ui.gif) + +### function parameters `pipe.viz_streamlit` + +| Argument | Type | Default |Description | +|-----------------------|--------------------------------------------------|----------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------| +| `text` | `Union [str, List[str], pd.DataFrame, pd.Series]` | `'NLU and Streamlit go together like peanutbutter and jelly'` | Default text for the `Classification`, `Named Entitiy Recognizer`, `Token Information` and `Dependency Tree` visualizations +| `similarity_texts` | `Union[List[str],Tuple[str,str]]` | `('Donald Trump Likes to part', 'Angela Merkel likes to party')` | Default texts for the `Text similarity` visualization. Should contain `exactly 2 strings` which will be compared `token embedding wise`. For each embedding active, a `token wise similarity matrix` and a `similarity scalar` +| `model_selection` | `List[str]` | `[]` | List of nlu references to display in the model selector, see [the NLU Namespace](https://nlu.johnsnowlabs.com/docs/en/namespace) or the [John Snow Labs Modelshub](https://modelshub.johnsnowlabs.com/models) or go [straight to the source](https://github.com/JohnSnowLabs/nlu/blob/3.0.2rc1/nlu/namespace.py) for more info +| `title` | `str` | `'NLU ❀️ Streamlit - Prototype your NLP startup in 0 lines of codeπŸš€'` | Title of the Streamlit app +| `sub_title` | `str` | `'Play with over 1000+ scalable enterprise NLP models'` | Sub title of the Streamlit app +| `visualizers` | `List[str]` | `( "dependency_tree", "ner", "similarity", "token_information", 'classification')` | Define which visualizations should be displayed. By default all visualizations are displayed. +| `show_models_info` | `bool` | `True` | Show information for every model loaded in the bottom of the Streamlit app. +| `show_model_select` | `bool` | `True` | Show a model selection dropdowns that makes any of the 1000+ models avaiable in 1 click +| `show_viz_selection` | `bool` | `False` | Show a selector in the sidebar which lets you configure which visualizations are displayed. +| `show_logo` | `bool` | `True` | Show logo +| `display_infos` | `bool` | `False` | Display additonal information about ISO codes and the NLU namespace structure. +| `set_wide_layout_CSS` | `bool` | `True` | Whether to inject custom CSS or not. +| `key` | `str` | `"NLU_streamlit"` | Key for the Streamlit elements drawn +| `model_select_position` | `str` | `'side'` | [Whether to output the positions of predictions or not, see `pipe.predict(positions=true`) for more info](https://nlu.johnsnowlabs.com/docs/en/predict_api#output-positions-parameter) | +| `show_code_snippets` | `bool` | `False` | Display Python code snippets above visualizations that can be used to re-create the visualization +|`num_similarity_cols` | `int` | `2` | How many columns should for the layout in Streamlit when rendering the similarity matrixes. + + + +## function `pipe.viz_streamlit_classes` + +Visualize the predicted classes and their confidences and additional metadata to streamlit. +Aplicable with [any of the 100+ classifiers](https://nlp.johnsnowlabs.com/models?task=Text+Classification) + +```python +nlu.load('sentiment').viz_streamlit_classes(['I love NLU and Streamlit!','I love buggy software', 'Sign up now get a chance to win 1000$ !', 'I am afraid of Snakes','Unicorns have been sighted on Mars!','Where is the next bus stop?']) +``` +![text_class1](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/gif/class.gif) + + +### function parameters `pipe.viz_streamlit_classes` + +| Argument | Type | Default |Description | +|--------------------------- | ---------- |-----------------------------------------------------------| ------------------------------------------------------- | +| `text` | `Union[str,list,pd.DataFrame, pd.Series, pyspark.sql.DataFrame ]` | `'I love NLU and Streamlit and sunny days!'` | Text to predict classes for. Will predict on each input of the iteratable or dataframe if type is not str.| +| `output_level` | `Optional[str]` | `document` | [Outputlevel of NLU pipeline, see `pipe.predict()` docsmore info](https://nlu.johnsnowlabs.com/docs/en/predict_api#output-level-parameter)| +| `include_text_col` | `bool` |`True` | Whether to include a e text column in the output table or just the prediction data | +| `title` | `Optional[str]` | `Text Classification` | Title of the Streamlit building block that will be visualized to screen | +| `metadata` | `bool` | `False` | [whether to output addition metadata or not, see `pipe.predict(meta=true)` docs for more info](https://nlu.johnsnowlabs.com/docs/en/predict_api#output-metadata) | +| `positions` | `bool` | `False` | [whether to output the positions of predictions or not, see `pipe.predict(positions=true`) for more info](https://nlu.johnsnowlabs.com/docs/en/predict_api#output-positions-parameter) | +| `set_wide_layout_CSS` | `bool` | `True` | Whether to inject custom CSS or not. +| `key` | `str` | `"NLU_streamlit"` | Key for the Streamlit elements drawn +| `model_select_position` | `str` | `'side'` | [Whether to output the positions of predictions or not, see `pipe.predict(positions=true`) for more info](https://nlu.johnsnowlabs.com/docs/en/predict_api#output-positions-parameter) | +| `generate_code_sample` | `bool` | `False` | Display Python code snippets above visualizations that can be used to re-create the visualization +| `show_model_select` | `bool` | `True` | Show a model selection dropdowns that makes any of the 1000+ models avaiable in 1 click +| `show_logo` | `bool` | `True` | Show logo +| `display_infos` | `bool` | `False` | Display additonal information about ISO codes and the NLU namespace structure. + + + +## function `pipe.viz_streamlit_ner` +Visualize the predicted classes and their confidences and additional metadata to Streamlit. +Aplicable with [any of the 250+ NER models](https://nlp.johnsnowlabs.com/models?task=Named+Entity+Recognition). +You can filter which NER tags to highlight via the dropdown in the main window. + +Basic usage +```python +nlu.load('ner').viz_streamlit_ner('Donald Trump from America and Angela Merkel from Germany dont share many views') +``` + +![NER visualization](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/gif/NER.gif) + +Example for coloring +```python +# Color all entities of class GPE black +nlu.load('ner').viz_streamlit_ner('Donald Trump from America and Angela Merkel from Germany dont share many views',colors={'PERSON':'#6e992e', 'GPE':'#000000'}) +``` +![NER coloring](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/img/NER_colored.png) + +### function parameters `pipe.viz_streamlit_ner` + +| Argument | Type | Default |Description | +|--------------------------- | -----------------------|-----------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------ | +| `text` | `str` | `'Donald Trump from America and Anegela Merkel from Germany do not share many views'` | Text to predict classes for.| +| `ner_tags` | `Optional[List[str]]` | `None` |Tags to display. By default all tags will be displayed| +| `show_label_select` | `bool` |`True` | Whether to include the label selector| +| `show_table` | `bool` | `True` | Whether show to predicted pandas table or not| +| `title` | `Optional[str]` | `'Named Entities'` | Title of the Streamlit building block that will be visualized to screen | +| `sub_title` | `Optional[str]` | `'"Recognize various Named Entities (NER) in text entered and filter them. You can select from over 100 languages in the dropdown. On the left side.",'` | Sub-title of the Streamlit building block that will be visualized to screen | +| `colors` | `Dict[str,str]` | `{}` | Dict with `KEY=ENTITY_LABEL` and `VALUE=COLOR_AS_HEX_CODE`,which will change color of highlighted entities.[See custom color labels docs for more info.](https://nlu.johnsnowlabs.com/docs/en/viz_examples#define-custom-colors-for-labels) | +| `set_wide_layout_CSS` | `bool` | `True` | Whether to inject custom CSS or not. +| `key` | `str` | `"NLU_streamlit"` | Key for the Streamlit elements drawn +| `generate_code_sample` | `bool` | `False` | Display Python code snippets above visualizations that can be used to re-create the visualization +| `show_model_select` | `bool` | `True` | Show a model selection dropdowns that makes any of the 1000+ models avaiable in 1 click +| `model_select_position` | `str` | `'side'` | [Whether to output the positions of predictions or not, see `pipe.predict(positions=true`) for more info](https://nlu.johnsnowlabs.com/docs/en/predict_api#output-positions-parameter) | +| `show_text_input` | `bool` | `True` | Show text input field to input text in +| `show_logo` | `bool` | `True` | Show logo +| `display_infos` | `bool` | `False` | Display additonal information about ISO codes and the NLU namespace structure. + + + + +## function `pipe.viz_streamlit_dep_tree` +Visualize a typed dependency tree, the relations between tokens and part of speech tags predicted. +Aplicable with [any of the 100+ Part of Speech(POS) models and dep tree model](https://nlp.johnsnowlabs.com/models?task=Part+of+Speech+Tagging) + +```python +nlu.load('dep.typed').viz_streamlit_dep_tree('POS tags define a grammatical label for each token and the Dependency Tree classifies Relations between the tokens') +``` +![Dependency Tree](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/img/DEP.png) + +### function parameters `pipe.viz_streamlit_dep_tree` + +| Argument | Type | Default |Description | +|--------------------------- | ---------- |-----------------------------------------------------------| ------------------------------------------------------- | +| `text` | `str` | `'Billy likes to swim'` | Text to predict classes for.| +| `title` | `Optional[str]` | `'Dependency Parse Tree & Part-of-speech tags'` | Title of the Streamlit building block that will be visualized to screen | +| `set_wide_layout_CSS` | `bool` | `True` | Whether to inject custom CSS or not. +| `key` | `str` | `"NLU_streamlit"` | Key for the Streamlit elements drawn +| `generate_code_sample` | `bool` | `False` | Display Python code snippets above visualizations that can be used to re-create the visualization +| `set_wide_layout_CSS` | `bool` | `True` | Whether to inject custom CSS or not. +| `key` | `str` | `"NLU_streamlit"` | Key for the Streamlit elements drawn +| `generate_code_sample` | `bool` | `False` | Display Python code snippets above visualizations that can be used to re-create the visualization +| `show_model_select` | `bool` | `True` | Show a model selection dropdowns that makes any of the 1000+ models avaiable in 1 click +| `model_select_position` | `str` | `'side'` | [Whether to output the positions of predictions or not, see `pipe.predict(positions=true`) for more info](https://nlu.johnsnowlabs.com/docs/en/predict_api#output-positions-parameter) | +| `show_logo` | `bool` | `True` | Show logo +| `display_infos` | `bool` | `False` | Display additonal information about ISO codes and the NLU namespace structure. + + + + + +## function `pipe.viz_streamlit_token` +Visualize predicted token and text features for every model loaded. +You can use this with [any of the 1000+ models](https://nlp.johnsnowlabs.com/models) and select them from the left dropdown. + +```python +nlu.load('stemm pos spell').viz_streamlit_token('I liek pentut buttr and jelly !') +``` +![text_class1](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/gif/token.gif) + + +### function parameters `pipe.viz_streamlit_token` + +| Argument | Type | Default |Description | +|--------------------------- | ---------- |-----------------------------------------------------------| ------------------------------------------------------- | +| `text` | `str` | `'NLU and Streamlit are great!'` | Text to predict token information for.| +| `title` | `Optional[str]` | `'Named Entities'` | Title of the Streamlit building block that will be visualized to screen | +| `show_feature_select` | `bool` |`True` | Whether to include the token feature selector| +| `features` | `Optional[List[str]]` | `None` |Features to to display. By default all Features will be displayed| +| `metadata` | `bool` | `False` | [Whether to output addition metadata or not, see `pipe.predict(meta=true)` docs for more info](https://nlu.johnsnowlabs.com/docs/en/predict_api#output-metadata) | +| `output_level` | `Optional[str]` | `'token'` | [Outputlevel of NLU pipeline, see `pipe.predict()` docsmore info](https://nlu.johnsnowlabs.com/docs/en/predict_api#output-level-parameter)| +| `positions` | `bool` | `False` | [Whether to output the positions of predictions or not, see `pipe.predict(positions=true`) for more info](https://nlu.johnsnowlabs.com/docs/en/predict_api#output-positions-parameter) | +| `set_wide_layout_CSS` | `bool` | `True` | Whether to inject custom CSS or not. +| `key` | `str` | `"NLU_streamlit"` | Key for the Streamlit elements drawn +| `generate_code_sample` | `bool` | `False` | Display Python code snippets above visualizations that can be used to re-create the visualization +| `show_model_select` | `bool` | `True` | Show a model selection dropdowns that makes any of the 1000+ models avaiable in 1 click +| `model_select_position` | `str` | `'side'` | [Whether to output the positions of predictions or not, see `pipe.predict(positions=true`) for more info](https://nlu.johnsnowlabs.com/docs/en/predict_api#output-positions-parameter) | +| `show_logo` | `bool` | `True` | Show logo +| `display_infos` | `bool` | `False` | Display additonal information about ISO codes and the NLU namespace structure. + + + + +## function `pipe.viz_streamlit_similarity` + +- Displays a `similarity matrix`, where `x-axis` is every token in the first text and `y-axis` is every token in the second text. +- Index `i,j` in the matrix describes the similarity of `token-i` to `token-j` based on the loaded embeddings and distance metrics, based on [Sklearns Pariwise Metrics.](https://scikit-learn.org/stable/modules/classes.html#module-sklearn.metrics.pairwise). [See this article for more elaboration on similarities](https://medium.com/spark-nlp/easy-sentence-similarity-with-bert-sentence-embeddings-using-john-snow-labs-nlu-ea078deb6ebf) +- Displays a dropdown selectors from which various similarity metrics and over 100 embeddings can be selected. + -There will be one similarity matrix per `metric` and `embedding` pair selected. `num_plots = num_metric*num_embeddings` + Also displays embedding vector information. + Applicable with [any of the 100+ Word Embedding models](https://nlp.johnsnowlabs.com/models?task=Embeddings) + + + +```python +nlu.load('bert').viz_streamlit_word_similarity(['I love love loooove NLU! <3','I also love love looove Streamlit! <3']) +``` +![text_class1](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/gif/SIM.gif) + +### function parameters `pipe.viz_streamlit_similarity` + +| Argument | Type | Default |Description | +|--------------------------- | ---------- |-----------------------------------------------------------| ------------------------------------------------------- | +| `texts` | `str` | `'Donald Trump from America and Anegela Merkel from Germany do not share many views.'` | Text to predict token information for.| +| `title` | `Optional[str]` | `'Named Entities'` | Title of the Streamlit building block that will be visualized to screen | +| `similarity_matrix` | `bool` | `None` |Whether to display similarity matrix or not| +| `show_algo_select` | `bool` |`True` | Whether to show dist algo select or not | +| `show_table` | `bool` | `True` | Whether show to predicted pandas table or not| +| `threshold` | `float` | `0.5` | Threshold for displaying result red on screen | +| `set_wide_layout_CSS` | `bool` | `True` | Whether to inject custom CSS or not. +| `key` | `str` | `"NLU_streamlit"` | Key for the Streamlit elements drawn +| `generate_code_sample` | `bool` | `False` | Display Python code snippets above visualizations that can be used to re-create the visualization +| `show_model_select` | `bool` | `True` | Show a model selection dropdowns that makes any of the 1000+ models avaiable in 1 click +| `model_select_position` | `str` | `'side'` | [Whether to output the positions of predictions or not, see `pipe.predict(positions=true`) for more info](https://nlu.johnsnowlabs.com/docs/en/predict_api#output-positions-parameter) | +|`write_raw_pandas` | `bool` | `False` | Write the raw pandas similarity df to streamlit +|`display_embed_information` | `bool` | `True` | Show additional embedding information like `dimension`, `nlu_reference`, `spark_nlp_reference`, `sotrage_reference`, `modelhub link` and more. +|`dist_metrics` | `List[str]` | `['cosine']` | Which distance metrics to apply. If multiple are selected, there will be multiple plots for each embedding and metric. `num_plots = num_metric*num_embeddings`. Can use multiple at the same time, any of of `cityblock`,`cosine`,`euclidean`,`l2`,`l1`,`manhattan`,`nan_euclidean`. Provided via [Sklearn `metrics.pairwise` package](https://scikit-learn.org/stable/modules/classes.html#module-sklearn.metrics.pairwise) +|`num_cols` | `int` | `2` | How many columns should for the layout in streamlit when rendering the similarity matrixes. +|`display_scalar_similarities` | `bool` | `False` | Display scalar simmilarities in an additional field. +|`display_similarity_summary` | `bool` | `False` | Display summary of all similarities for all embeddings and metrics. +| `show_logo` | `bool` | `True` | Show logo +| `display_infos` | `bool` | `False` | Display additonal information about ISO codes and the NLU namespace structure. + + + + + + + +
+ From 2a08bf04e040ae43b45b5e54bc3311bca3dfffde Mon Sep 17 00:00:00 2001 From: C-K-Loan Date: Thu, 3 Jun 2021 04:24:45 +0200 Subject: [PATCH 3/3] NLU 3.0.2 release --- docs/Gemfile.lock | 24 +- docs/README.md | 3 + docs/en/streamlit_viz_examples.md | 46 +- .../entity_resolvers_overview.ipynb | 1635 ++++++++++++++++- nlu/namespace.py | 2 - setup.py | 2 +- 6 files changed, 1675 insertions(+), 37 deletions(-) diff --git a/docs/Gemfile.lock b/docs/Gemfile.lock index a3040ed2..a46adf43 100644 --- a/docs/Gemfile.lock +++ b/docs/Gemfile.lock @@ -1,7 +1,7 @@ GEM remote: https://rubygems.org/ specs: - activesupport (6.0.3.6) + activesupport (6.0.3.7) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (>= 0.7, < 2) minitest (~> 5.1) @@ -26,18 +26,22 @@ GEM ffi (>= 1.15.0) eventmachine (1.2.7) eventmachine (1.2.7-x64-mingw32) - execjs (2.7.0) - faraday (1.4.1) + execjs (2.8.1) + faraday (1.4.2) + faraday-em_http (~> 1.0) + faraday-em_synchrony (~> 1.0) faraday-excon (~> 1.1) faraday-net_http (~> 1.0) faraday-net_http_persistent (~> 1.1) multipart-post (>= 1.2, < 3) ruby2_keywords (>= 0.0.4) + faraday-em_http (1.0.0) + faraday-em_synchrony (1.0.0) faraday-excon (1.1.0) faraday-net_http (1.0.1) faraday-net_http_persistent (1.1.0) - ffi (1.15.0) - ffi (1.15.0-x64-mingw32) + ffi (1.15.1) + ffi (1.15.1-x64-mingw32) forwardable-extended (2.6.0) gemoji (3.0.1) github-pages (214) @@ -211,17 +215,17 @@ GEM rb-fsevent (~> 0.10, >= 0.10.3) rb-inotify (~> 0.9, >= 0.9.10) mercenary (0.3.6) - mini_portile2 (2.5.1) + mini_portile2 (2.5.3) minima (2.5.1) jekyll (>= 3.5, < 5.0) jekyll-feed (~> 0.9) jekyll-seo-tag (~> 2.1) minitest (5.14.4) multipart-post (2.1.1) - nokogiri (1.11.3) + nokogiri (1.11.7) mini_portile2 (~> 2.5.0) racc (~> 1.4) - nokogiri (1.11.3-x64-mingw32) + nokogiri (1.11.7-x64-mingw32) racc (~> 1.4) octokit (4.21.0) faraday (>= 0.9) @@ -230,7 +234,7 @@ GEM forwardable-extended (~> 2.6) public_suffix (4.0.6) racc (1.5.2) - rb-fsevent (0.10.4) + rb-fsevent (0.11.0) rb-inotify (0.10.1) ffi (~> 1.0) rexml (3.2.5) @@ -272,4 +276,4 @@ DEPENDENCIES github-pages (= 214) BUNDLED WITH - 2.1.4 \ No newline at end of file + 2.2.19 diff --git a/docs/README.md b/docs/README.md index 405ba909..96af99fb 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,6 +2,9 @@ We welcome you to contribute to NLU documentation hosted inside `en/` directory. All the files are in Markdown format. +## Install +[You need to install Jekyll](https://jekyllrb.com/docs/installation/ubuntu/) + ## Development For development purposes, you need to have `bundle` and `Gem` installed on your system. Please run these commands: diff --git a/docs/en/streamlit_viz_examples.md b/docs/en/streamlit_viz_examples.md index 2d173958..01401331 100644 --- a/docs/en/streamlit_viz_examples.md +++ b/docs/en/streamlit_viz_examples.md @@ -29,45 +29,45 @@ Problems? [Connect with us on Slack!](https://join.slack.com/t/spark-nlp/shared_ ## Impatient and want some action? Just run this Streamlit app, you can use it to generate python code for each NLU-Streamlit building block ```shell -streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/01_dashboard.py +streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/examples/streamlit/01_dashboard.py ``` ## Quick Starter cheat sheet - All you need to know in 1 picture for NLU + Streamlit -For NLU models to load, see [the NLU Namespace](https://nlu.johnsnowlabs.com/docs/en/namespace) or the [John Snow Labs Modelshub](https://modelshub.johnsnowlabs.com/models) or go [straight to the source](https://github.com/JohnSnowLabs/nlu/blob/3.0.2rc1/nlu/namespace.py). -![NLU Streamlit Cheatsheet](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/img/NLU_Streamlit_Cheetsheet.png) +For NLU models to load, see [the NLU Namespace](https://nlu.johnsnowlabs.com/docs/en/namespace) or the [John Snow Labs Modelshub](https://modelshub.johnsnowlabs.com/models) or go [straight to the source](https://github.com/JohnSnowLabs/nlu/blob/master/nlu/namespace.py). +![NLU Streamlit Cheatsheet](https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/docs/assets/streamlit_docs_assets/img/NLU_Streamlit_Cheetsheet.png) ## Examples Just try out any of these. You can use the first example to generate python-code snippets which you can recycle as building blocks in your streamlit apps! -### Example: [`01_dashboard`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/01_dashboard.py) +### Example: [`01_dashboard`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/examples/streamlit/01_dashboard.py) ```shell -streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/01_dashboard.py +streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/examples/streamlit/01_dashboard.py ``` -### Example: [`02_NER`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/02_NER.py) +### Example: [`02_NER`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/examples/streamlit/02_NER.py) ```shell -streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/02_NER.py +streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/examples/streamlit/02_NER.py ``` -### Example: [`03_text_similarity_matrix`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/03_text_similarity_matrix.py) +### Example: [`03_text_similarity_matrix`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/examples/streamlit/03_text_similarity_matrix.py) ```shell -streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/03_text_similarity_matrix.py +streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/examples/streamlit/03_text_similarity_matrix.py ``` -### Example: [`04_dependency_tree`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/04_dependency_tree.py) +### Example: [`04_dependency_tree`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/examples/streamlit/04_dependency_tree.py) ```shell -streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/04_dependency_tree.py +streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/examples/streamlit/04_dependency_tree.py ``` -### Example: [`05_classifiers`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/05_classifiers.py) +### Example: [`05_classifiers`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/examples/streamlit/05_classifiers.py) ```shell -streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/05_classifiers.py +streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/examples/streamlit/05_classifiers.py ``` -### Example: [`06_token_features`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/06_token_features.py) +### Example: [`06_token_features`](https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/examples/streamlit/06_token_features.py) ```shell -streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/examples/streamlit/06_token_features.py +streamlit run https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/examples/streamlit/06_token_features.py ``` ## How to use NLU? @@ -119,7 +119,7 @@ nlu.load('ner').viz_streamlit(['I love NLU and Streamlit!','I hate buggy softwar -![NLU Streamlit UI Overview](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/gif/ui.gif) +![NLU Streamlit UI Overview](https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/docs/assets/streamlit_docs_assets/gif/ui.gif) ### function parameters `pipe.viz_streamlit` @@ -127,7 +127,7 @@ nlu.load('ner').viz_streamlit(['I love NLU and Streamlit!','I hate buggy softwar |-----------------------|--------------------------------------------------|----------------------------------------------------------------------------------------|--------------------------------------------------------------------------------------------------------------------------------| | `text` | `Union [str, List[str], pd.DataFrame, pd.Series]` | `'NLU and Streamlit go together like peanutbutter and jelly'` | Default text for the `Classification`, `Named Entitiy Recognizer`, `Token Information` and `Dependency Tree` visualizations | `similarity_texts` | `Union[List[str],Tuple[str,str]]` | `('Donald Trump Likes to part', 'Angela Merkel likes to party')` | Default texts for the `Text similarity` visualization. Should contain `exactly 2 strings` which will be compared `token embedding wise`. For each embedding active, a `token wise similarity matrix` and a `similarity scalar` -| `model_selection` | `List[str]` | `[]` | List of nlu references to display in the model selector, see [the NLU Namespace](https://nlu.johnsnowlabs.com/docs/en/namespace) or the [John Snow Labs Modelshub](https://modelshub.johnsnowlabs.com/models) or go [straight to the source](https://github.com/JohnSnowLabs/nlu/blob/3.0.2rc1/nlu/namespace.py) for more info +| `model_selection` | `List[str]` | `[]` | List of nlu references to display in the model selector, see [the NLU Namespace](https://nlu.johnsnowlabs.com/docs/en/namespace) or the [John Snow Labs Modelshub](https://modelshub.johnsnowlabs.com/models) or go [straight to the source](https://github.com/JohnSnowLabs/nlu/blob/master/nlu/namespace.py) for more info | `title` | `str` | `'NLU ❀️ Streamlit - Prototype your NLP startup in 0 lines of codeπŸš€'` | Title of the Streamlit app | `sub_title` | `str` | `'Play with over 1000+ scalable enterprise NLP models'` | Sub title of the Streamlit app | `visualizers` | `List[str]` | `( "dependency_tree", "ner", "similarity", "token_information", 'classification')` | Define which visualizations should be displayed. By default all visualizations are displayed. @@ -152,7 +152,7 @@ Aplicable with [any of the 100+ classifiers](https://nlp.johnsnowlabs.com/models ```python nlu.load('sentiment').viz_streamlit_classes(['I love NLU and Streamlit!','I love buggy software', 'Sign up now get a chance to win 1000$ !', 'I am afraid of Snakes','Unicorns have been sighted on Mars!','Where is the next bus stop?']) ``` -![text_class1](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/gif/class.gif) +![text_class1](https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/docs/assets/streamlit_docs_assets/gif/class.gif) ### function parameters `pipe.viz_streamlit_classes` @@ -185,14 +185,14 @@ Basic usage nlu.load('ner').viz_streamlit_ner('Donald Trump from America and Angela Merkel from Germany dont share many views') ``` -![NER visualization](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/gif/NER.gif) +![NER visualization](https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/docs/assets/streamlit_docs_assets/gif/NER.gif) Example for coloring ```python # Color all entities of class GPE black nlu.load('ner').viz_streamlit_ner('Donald Trump from America and Angela Merkel from Germany dont share many views',colors={'PERSON':'#6e992e', 'GPE':'#000000'}) ``` -![NER coloring](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/img/NER_colored.png) +![NER coloring](https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/docs/assets/streamlit_docs_assets/img/NER_colored.png) ### function parameters `pipe.viz_streamlit_ner` @@ -224,7 +224,7 @@ Aplicable with [any of the 100+ Part of Speech(POS) models and dep tree model](h ```python nlu.load('dep.typed').viz_streamlit_dep_tree('POS tags define a grammatical label for each token and the Dependency Tree classifies Relations between the tokens') ``` -![Dependency Tree](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/img/DEP.png) +![Dependency Tree](https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/docs/assets/streamlit_docs_assets/img/DEP.png) ### function parameters `pipe.viz_streamlit_dep_tree` @@ -254,7 +254,7 @@ You can use this with [any of the 1000+ models](https://nlp.johnsnowlabs.com/mod ```python nlu.load('stemm pos spell').viz_streamlit_token('I liek pentut buttr and jelly !') ``` -![text_class1](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/gif/token.gif) +![text_class1](https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/docs/assets/streamlit_docs_assets/gif/token.gif) ### function parameters `pipe.viz_streamlit_token` @@ -293,7 +293,7 @@ nlu.load('stemm pos spell').viz_streamlit_token('I liek pentut buttr and jelly ! ```python nlu.load('bert').viz_streamlit_word_similarity(['I love love loooove NLU! <3','I also love love looove Streamlit! <3']) ``` -![text_class1](https://raw.githubusercontent.com/JohnSnowLabs/nlu/3.0.2rc1/docs/assets/streamlit_docs_assets/gif/SIM.gif) +![text_class1](https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/docs/assets/streamlit_docs_assets/gif/SIM.gif) ### function parameters `pipe.viz_streamlit_similarity` diff --git a/examples/colab/healthcare/entity_resolution/entity_resolvers_overview.ipynb b/examples/colab/healthcare/entity_resolution/entity_resolvers_overview.ipynb index 752264b9..3e2fe848 100644 --- a/examples/colab/healthcare/entity_resolution/entity_resolvers_overview.ipynb +++ b/examples/colab/healthcare/entity_resolution/entity_resolvers_overview.ipynb @@ -1 +1,1634 @@ -{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"entity_resolvers_overview.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","metadata":{"id":"8ZelAXba3D05"},"source":["![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)\n","\n","[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/nlu/blob/master/examples/colab/healthcare/entity_resolution/entity_resolvers_overview.ipynb)\n","\n","All the models avaiable are :\n","\n","\n","| Language | nlu.load() reference | Spark NLP Model reference |\n","| -------- | ------------------------------------------------------------ | ------------------------------------------------------------ |\n","| English | embed_sentence.biobert.mli | sbiobert_base_cased_mli |\n","| English | resolve | sbiobertresolve_cpt |\n","| English | resolve.cpt | sbiobertresolve_cpt |\n","| English | resolve.cpt.augmented | sbiobertresolve_cpt_augmented |\n","| English | resolve.cpt.procedures_augmented | sbiobertresolve_cpt_procedures_augmented |\n","| English | resolve.hcc.augmented | sbiobertresolve_hcc_augmented |\n","| English | [resolve.icd10cm](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icd10cm_en.html) | [sbiobertresolve_icd10cm](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icd10cm_en.html) |\n","| English | [resolve.icd10cm.augmented](https://nlp.johnsnowlabs.com/2020/12/13/sbiobertresolve_icd10cm_augmented_en.html) | [sbiobertresolve_icd10cm_augmented](https://nlp.johnsnowlabs.com/2020/12/13/sbiobertresolve_icd10cm_augmented_en.html) |\n","| English | [resolve.icd10cm.augmented_billable](https://nlp.johnsnowlabs.com/2021/02/06/sbiobertresolve_icd10cm_augmented_billable_hcc_en.html) | [sbiobertresolve_icd10cm_augmented_billable_hcc](https://nlp.johnsnowlabs.com/2021/02/06/sbiobertresolve_icd10cm_augmented_billable_hcc_en.html) |\n","| English | [resolve.icd10pcs](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icd10pcs_en.html) | [sbiobertresolve_icd10pcs](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icd10pcs_en.html) |\n","| English | [resolve.icdo](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icdo_en.html) | [sbiobertresolve_icdo](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icdo_en.html) |\n","| English | [resolve.rxcui](https://nlp.johnsnowlabs.com/2020/12/11/sbiobertresolve_rxcui_en.html) | [sbiobertresolve_rxcui](https://nlp.johnsnowlabs.com/2020/12/11/sbiobertresolve_rxcui_en.html) |\n","| English | [resolve.rxnorm](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_rxnorm_en.html) | [sbiobertresolve_rxnorm](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_rxnorm_en.html) |\n","| English | [resolve.snomed](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_auxConcepts_en.html) | [sbiobertresolve_snomed_auxConcepts](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_auxConcepts_en.html) |\n","| English | [resolve.snomed.aux_concepts](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_auxConcepts_en.html) | [sbiobertresolve_snomed_auxConcepts](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_auxConcepts_en.html) |\n","| English | [resolve.snomed.aux_concepts_int](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_auxConcepts_int_en.html) | [sbiobertresolve_snomed_auxConcepts_int](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_auxConcepts_int_en.html) |\n","| English | [resolve.snomed.findings](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_findings_en.html) | [sbiobertresolve_snomed_findings](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_findings_en.html) |\n","| English | [resolve.snomed.findings_int](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_findings_int_en.html) | [sbiobertresolve_snomed_findings_int](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_findings_int_en.html) |\n"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"OMiHjCtQhF_r","executionInfo":{"status":"ok","timestamp":1620194990296,"user_tz":-300,"elapsed":128547,"user":{"displayName":"ahmed lone","photoUrl":"","userId":"02458088882398909889"}},"outputId":"d699d432-db14-48d6-98c0-7c593a9d0fb1"},"source":["# # Install NLU\n","# # Upload add your spark_nlp_fo\"r_healthcare.json\n","!wget https://setup.johnsnowlabs.com/nlu/colab.sh | bash\n","import nlu"],"execution_count":2,"outputs":[{"output_type":"stream","text":["--2021-05-05 06:07:42-- https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/scripts/colab_setup.sh\n","Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n","Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n","HTTP request sent, awaiting response... 200 OK\n","Length: 1671 (1.6K) [text/plain]\n","Saving to: β€˜STDOUT’\n","\n","\r- 0%[ ] 0 --.-KB/s \r- 100%[===================>] 1.63K --.-KB/s in 0s \n","\n","2021-05-05 06:07:42 (21.3 MB/s) - written to stdout [1671/1671]\n","\n","Installing NLU 3.0.0 with PySpark 3.0.2 and Spark NLP 3.0.1 for Google Colab ...\n","\u001b[K |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 204.8MB 66kB/s \n","\u001b[K |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 153kB 37.2MB/s \n","\u001b[K |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 204kB 20.3MB/s \n","\u001b[K |β–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆβ–ˆ| 204kB 49.1MB/s \n","\u001b[?25h Building wheel for pyspark (setup.py) ... \u001b[?25l\u001b[?25hdone\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"kbnlNAw_W3Qj"},"source":["#### [Athena Conditions Entity Resolver (Healthcare)](https://nlp.johnsnowlabs.com/2020/09/16/chunkresolve_athena_conditions_healthcare_en.html)"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000},"id":"n7zbAt-SW3Eh","executionInfo":{"elapsed":190050,"status":"ok","timestamp":1620101576257,"user":{"displayName":"Christian Kasim Loan","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjqAD-ircKP-s5Eh6JSdkDggDczfqQbJGU_IRb4Hw=s64","userId":"14469489166467359317"},"user_tz":-120},"outputId":"ed79275c-dd0e-4b2d-e721-59129ec67ecc"},"source":["import nlu\n","\n","data =\"\"\"The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion.\"\"\"\n","nlu.load('med_ner.jsl.wip.clinical en.resolve_chunk.cpt_clinical').predict(data, output_level='chunk')"],"execution_count":null,"outputs":[{"output_type":"stream","text":["Spark NLP Healthcare could not be imported. Installing latest spark-nlp-jsl PyPI package via pip...\n","jsl_ner_wip_clinical download started this may take some time.\n","Approximate size to download 14.5 MB\n","[OK!]\n","chunkresolve_cpt_clinical download started this may take some time.\n","Approximate size to download 16.1 MB\n","[OK!]\n","embeddings_clinical download started this may take some time.\n","Approximate size to download 1.6 GB\n","[OK!]\n","sentence_detector_dl download started this may take some time.\n","Approximate size to download 354.6 KB\n","[OK!]\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
entities_confidenceentity_resolution_cpt_clinicaldocumententities_classentity_resolution_cpt_clinical_confidencetextsentencetokenchunk_embedding_gloveentity_resolution_cpt_clinical_codeword_embedding_gloveentitiesorigin_index
00.9988Repair, initial inguinal hernia, full term inf...The patient is a 5-month-old infant who presen...Age0.6608The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.18568818271160126, -0.5141453742980957, -0....49495[[-0.0823686420917511, -0.3178570866584778, 0....5-month-old8589934592
00.9999Repair, initial inguinal hernia, preterm infan...The patient is a 5-month-old infant who presen...Age1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.0033501186408102512, -0.8093816637992859, -...49492[[-0.0823686420917511, -0.3178570866584778, 0....infant8589934592
00.998Induced abortion, by 1 or more vaginal supposi...The patient is a 5-month-old infant who presen...RelativeDate1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.1907067447900772, 0.04115135222673416, -0.1...59857[[-0.0823686420917511, -0.3178570866584778, 0....Monday8589934592
00.712Donor nephrectomy (including cold preservation...The patient is a 5-month-old infant who presen...Symptom0.5539The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.324726939201355, -0.137750506401062, 0.2801...50320[[-0.0823686420917511, -0.3178570866584778, 0....cold8589934592
00.9966Pleural scarification for repeat pneumothoraxThe patient is a 5-month-old infant who presen...Symptom0.6608The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.39911380410194397, -0.5716836452484131, 0.2...32215[[-0.0823686420917511, -0.3178570866584778, 0....cough8589934592
00.4894Excision of thyroglossal duct cyst or sinus; r...The patient is a 5-month-old infant who presen...Symptom1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.19262725114822388, -0.619391918182373, 0.26...60281[[-0.0823686420917511, -0.3178570866584778, 0....runny nose8589934592
00.50416666Reoperation, carotid, thromboendarterectomy, m...The patient is a 5-month-old infant who presen...Duration1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.16546504199504852, -0.016397878527641296, ...35390[[-0.0823686420917511, -0.3178570866584778, 0....for 2 days8589934592
00.5929Cesarean delivery only; including postpartum careThe patient is a 5-month-old infant who presen...Gender0.4715The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.13232460618019104, 0.04228536784648895, -0....59515[[-0.0823686420917511, -0.3178570866584778, 0....Mom8589934592
00.9992Repair, initial inguinal hernia, full term inf...The patient is a 5-month-old infant who presen...Gender0.6608The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.19785617291927338, -0.6325990557670593, -0....49495[[-0.0823686420917511, -0.3178570866584778, 0....she8589934592
00.9978Exploration for postoperative hemorrhage, thro...The patient is a 5-month-old infant who presen...VS_Finding1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.19038061797618866, -0.46617722511291504, -0...35840[[-0.0823686420917511, -0.3178570866584778, 0....fever8589934592
00.9973Exploration for postoperative hemorrhage, thro...The patient is a 5-month-old infant who presen...Gender1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.3410295248031616, -0.3789218068122864, 0.2...35840[[-0.0823686420917511, -0.3178570866584778, 0....Her8589934592
00.998Repair, initial inguinal hernia, full term inf...The patient is a 5-month-old infant who presen...Gender0.6608The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.19785617291927338, -0.6325990557670593, -0....49495[[-0.0823686420917511, -0.3178570866584778, 0....she8589934592
00.24122499Laparoscopy, abdomen, peritoneum, and omentum,...The patient is a 5-month-old infant who presen...Symptom1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.04143378883600235, -0.4098125100135803, 0.1...49320[[-0.0823686420917511, -0.3178570866584778, 0....spitting up a lot8589934592
00.9996Pericardiectomy, subtotal or complete; with ca...The patient is a 5-month-old infant who presen...Gender1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.21964989602565765, -0.2844458520412445, -0...33031[[-0.0823686420917511, -0.3178570866584778, 0....She8589934592
00.86275Repair, laceration of diaphragm, any approachThe patient is a 5-month-old infant who presen...Symptom0.9213The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.0559418648481369, -0.18557831645011902, 0.0...39501[[-0.0823686420917511, -0.3178570866584778, 0....difficulty breathing8589934592
00.9981Repair, initial inguinal hernia, full term inf...The patient is a 5-month-old infant who presen...Gender1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.16458719968795776, -0.5421502590179443, -0...49496[[-0.0823686420917511, -0.3178570866584778, 0....her8589934592
00.996Pleural scarification for repeat pneumothoraxThe patient is a 5-month-old infant who presen...Symptom0.6608The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.39911380410194397, -0.5716836452484131, 0.2...32215[[-0.0823686420917511, -0.3178570866584778, 0....cough8589934592
00.878Vaginal hysterectomy, for uterus 250 g or less;The patient is a 5-month-old infant who presen...Modifier0.8570The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.09050256758928299, -0.5477138757705688, 0....58275[[-0.0823686420917511, -0.3178570866584778, 0....dry8589934592
00.0834Right ventricular resection for infundibular s...The patient is a 5-month-old infant who presen...Direction0.8843The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.14579641819000244, -0.06464755535125732, -0...33476[[-0.0823686420917511, -0.3178570866584778, 0....right8589934592
00.3906Lysis of adhesions (salpingolysis, ovariolysis)The patient is a 5-month-old infant who presen...Internal_organ_or_component1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.09020363539457321, 0.13196095824241638, 0.2...58740[[-0.0823686420917511, -0.3178570866584778, 0....TM8589934592
00.0918Repair of double outlet right ventricle with i...The patient is a 5-month-old infant who presen...Direction0.5406The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.11810600757598877, 0.6391716599464417, -0....33612[[-0.0823686420917511, -0.3178570866584778, 0....Left8589934592
00.372Lysis of adhesions (salpingolysis, ovariolysis)The patient is a 5-month-old infant who presen...Internal_organ_or_component1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.09020363539457321, 0.13196095824241638, 0.2...58740[[-0.0823686420917511, -0.3178570866584778, 0....TM8589934592
00.9998Pericardiectomy, subtotal or complete; with ca...The patient is a 5-month-old infant who presen...Gender1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.21964989602565765, -0.2844458520412445, -0...33031[[-0.0823686420917511, -0.3178570866584778, 0....She8589934592
00.4025Reoperation, carotid, thromboendarterectomy, m...The patient is a 5-month-old infant who presen...Modifier0.9405The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.01206454448401928, 0.1365838199853897, 0.16...35390[[-0.0823686420917511, -0.3178570866584778, 0....fairly8589934592
00.7996Closure of enterostomy, large or small intestineThe patient is a 5-month-old infant who presen...Symptom1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.3624162971973419, -0.38189926743507385, 0....44620[[-0.0823686420917511, -0.3178570866584778, 0....congested8589934592
00.9996Pericardiectomy, subtotal or complete; with ca...The patient is a 5-month-old infant who presen...Gender1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.21964989602565765, -0.2844458520412445, -0...33031[[-0.0823686420917511, -0.3178570866584778, 0....She8589934592
00.9898Vaginal hysterectomy, for uterus 250 g or lessThe patient is a 5-month-old infant who presen...Drug_BrandName0.5551The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.04134025424718857, -0.06713572889566422, -...58260[[-0.0823686420917511, -0.3178570866584778, 0....Amoxil8589934592
00.9416Cholecystoenterostomy; with gastroenterostomyThe patient is a 5-month-old infant who presen...Drug_BrandName0.3333The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...47721[[-0.0823686420917511, -0.3178570866584778, 0....Aldex8589934592
00.5745333Repair, initial inguinal hernia, preterm infan...The patient is a 5-month-old infant who presen...RelativeDate1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.17584700882434845, -0.20433296263217926, -...49491[[-0.0823686420917511, -0.3178570866584778, 0....in 2 weeks8589934592
00.999Repair, initial inguinal hernia, full term inf...The patient is a 5-month-old infant who presen...Gender1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.16458719968795776, -0.5421502590179443, -0...49496[[-0.0823686420917511, -0.3178570866584778, 0....her8589934592
00.9324Cesarean delivery only; including postpartum careThe patient is a 5-month-old infant who presen...Gender0.4715The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.13232460618019104, 0.04228536784648895, -0....59515[[-0.0823686420917511, -0.3178570866584778, 0....Mom8589934592
01.0Laparoscopy, surgical, ablation of 1 or more l...The patient is a 5-month-old infant who presen...RelativeDate0.9361The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.21045373380184174, -0.4043411910533905, 0.2...47371[[-0.0823686420917511, -0.3178570866584778, 0....today8589934592
00.9996Repair, initial inguinal hernia, full term inf...The patient is a 5-month-old infant who presen...Gender0.6608The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.19785617291927338, -0.6325990557670593, -0....49495[[-0.0823686420917511, -0.3178570866584778, 0....she8589934592
00.9646Replacement or irrigation, subarachnoid/subdur...The patient is a 5-month-old infant who presen...RelativeTime0.9975The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.37081244587898254, -0.4993651509284973, 0....62194[[-0.0823686420917511, -0.3178570866584778, 0....overnight8589934592
00.9996Pericardiectomy, subtotal or complete; with ca...The patient is a 5-month-old infant who presen...Gender1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.21964989602565765, -0.2844458520412445, -0...33031[[-0.0823686420917511, -0.3178570866584778, 0....She8589934592
00.9042Repair, laceration of diaphragm, any approachThe patient is a 5-month-old infant who presen...Symptom0.9213The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.0559418648481369, -0.18557831645011902, 0.0...39501[[-0.0823686420917511, -0.3178570866584778, 0....difficulty breathing8589934592
00.9985Pericardiectomy, subtotal or complete; with ca...The patient is a 5-month-old infant who presen...Gender1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.21964989602565765, -0.2844458520412445, -0...33031[[-0.0823686420917511, -0.3178570866584778, 0....She8589934592
00.8923Closure of enterostomy, large or small intestineThe patient is a 5-month-old infant who presen...Symptom1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.3624162971973419, -0.38189926743507385, 0....44620[[-0.0823686420917511, -0.3178570866584778, 0....congested8589934592
00.9932Repair, initial inguinal hernia, full term inf...The patient is a 5-month-old infant who presen...Gender1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.16458719968795776, -0.5421502590179443, -0...49496[[-0.0823686420917511, -0.3178570866584778, 0....her8589934592
00.21180001Vaginal hysterectomy, for uterus greater than ...The patient is a 5-month-old infant who presen...Symptom0.5463The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.02755250409245491, 0.016924332827329636, -0...58293[[-0.0823686420917511, -0.3178570866584778, 0....appetite had decreased significantly8589934592
01.0Laparoscopy, surgical, ablation of 1 or more l...The patient is a 5-month-old infant who presen...RelativeDate0.9361The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.21045373380184174, -0.4043411910533905, 0.2...47371[[-0.0823686420917511, -0.3178570866584778, 0....today8589934592
00.9995Pericardiectomy, subtotal or complete; with ca...The patient is a 5-month-old infant who presen...Gender1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.21964989602565765, -0.2844458520412445, -0...33031[[-0.0823686420917511, -0.3178570866584778, 0....She8589934592
01.0Laparoscopy, surgical; cholecystoenterostomyThe patient is a 5-month-old infant who presen...RelativeDate0.8784The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.3847889304161072, -0.07220222800970078, 0.3...47570[[-0.0823686420917511, -0.3178570866584778, 0....yesterday8589934592
00.9522Vaginal hysterectomy, for uterus greater than ...The patient is a 5-month-old infant who presen...Temperature1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.1621697098016739, 0.044322118163108826, -0....58291[[-0.0823686420917511, -0.3178570866584778, 0....102.68589934592
00.66429996Exploration for postoperative hemorrhage, thro...The patient is a 5-month-old infant who presen...Symptom1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[0.17440977692604065, -0.4206063151359558, -0....35840[[-0.0823686420917511, -0.3178570866584778, 0....trouble sleeping8589934592
00.9749Exploration for postoperative hemorrhage, thro...The patient is a 5-month-old infant who presen...Symptom1.0000The patient is a 5-month-old infant who presen...[The patient is a 5-month-old infant who prese...[The, patient, is, a, 5-month-old, infant, who...[-0.33774805068969727, -0.18630369007587433, 0...35840[[-0.0823686420917511, -0.3178570866584778, 0....congestion8589934592
\n","
"],"text/plain":[" entities_confidence ... origin_index\n","0 0.9988 ... 8589934592\n","0 0.9999 ... 8589934592\n","0 0.998 ... 8589934592\n","0 0.712 ... 8589934592\n","0 0.9966 ... 8589934592\n","0 0.4894 ... 8589934592\n","0 0.50416666 ... 8589934592\n","0 0.5929 ... 8589934592\n","0 0.9992 ... 8589934592\n","0 0.9978 ... 8589934592\n","0 0.9973 ... 8589934592\n","0 0.998 ... 8589934592\n","0 0.24122499 ... 8589934592\n","0 0.9996 ... 8589934592\n","0 0.86275 ... 8589934592\n","0 0.9981 ... 8589934592\n","0 0.996 ... 8589934592\n","0 0.878 ... 8589934592\n","0 0.0834 ... 8589934592\n","0 0.3906 ... 8589934592\n","0 0.0918 ... 8589934592\n","0 0.372 ... 8589934592\n","0 0.9998 ... 8589934592\n","0 0.4025 ... 8589934592\n","0 0.7996 ... 8589934592\n","0 0.9996 ... 8589934592\n","0 0.9898 ... 8589934592\n","0 0.9416 ... 8589934592\n","0 0.5745333 ... 8589934592\n","0 0.999 ... 8589934592\n","0 0.9324 ... 8589934592\n","0 1.0 ... 8589934592\n","0 0.9996 ... 8589934592\n","0 0.9646 ... 8589934592\n","0 0.9996 ... 8589934592\n","0 0.9042 ... 8589934592\n","0 0.9985 ... 8589934592\n","0 0.8923 ... 8589934592\n","0 0.9932 ... 8589934592\n","0 0.21180001 ... 8589934592\n","0 1.0 ... 8589934592\n","0 0.9995 ... 8589934592\n","0 1.0 ... 8589934592\n","0 0.9522 ... 8589934592\n","0 0.66429996 ... 8589934592\n","0 0.9749 ... 8589934592\n","\n","[46 rows x 13 columns]"]},"metadata":{"tags":[]},"execution_count":1}]},{"cell_type":"markdown","metadata":{"id":"rSUTBWKi6Hb-"},"source":["#### [Sentence Entity Resolver for ICD10-CM (sbiobert_base_cased_mli embeddings)](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icd10cm_en.html)"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":481},"id":"emvKNnXE6HBv","executionInfo":{"elapsed":299508,"status":"ok","timestamp":1620101685722,"user":{"displayName":"Christian Kasim Loan","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjqAD-ircKP-s5Eh6JSdkDggDczfqQbJGU_IRb4Hw=s64","userId":"14469489166467359317"},"user_tz":-120},"outputId":"add87762-5b6c-4314-f888-45b8f77a7f1e"},"source":["nlu.load(\"en.embed_sentence.biobert.mli en.resolve.icd10cm\").predict(\"\"\"This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD ,\n","gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\\'s Center for Women & Infants for cardiac\n","catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , \n","subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .\"\"\",output_level = \"sentence\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["sbiobert_base_cased_mli download started this may take some time.\n","Approximate size to download 384.3 MB\n","[OK!]\n","sbiobertresolve_icd10cm download started this may take some time.\n","Approximate size to download 201 MB\n","[OK!]\n","sentence_detector_dl download started this may take some time.\n","Approximate size to download 354.6 KB\n","[OK!]\n","ner_dl_bert download started this may take some time.\n","Approximate size to download 15.4 MB\n","[OK!]\n","bert_base_cased download started this may take some time.\n","Approximate size to download 389.1 MB\n","[OK!]\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
entities_confidencetextword_embedding_bertdocumententities_classsentencesentence_resolution_icd10cmtokensentence_resolution_icd10cm_codesentence_embedding_biobertentitiesorigin_indexsentence_resolution_icd10cm_confidence
0[0.8038, 0.9959, 0.6357, 0.9798, 0.5822, 0.8133]This is an 82 - year-old male with a history o...[[0.465839684009552, -0.5797176361083984, 0.84...This is an 82 - year-old male with a history o...[MISC, LOC, ORG, LOC, MISC, ORG]This is an 82 - year-old male with a history o...Other injury due to other accident to sailboat...[This, is, an, 82, -, year-old, male, with, a,...V9184XA[0.2428887039422989, -0.23971685767173767, -0....[TIA, Braintree, Guaiac, St . Margaret's Cente...85899345920.3355
\n","
"],"text/plain":[" entities_confidence ... sentence_resolution_icd10cm_confidence\n","0 [0.8038, 0.9959, 0.6357, 0.9798, 0.5822, 0.8133] ... 0.3355\n","\n","[1 rows x 13 columns]"]},"metadata":{"tags":[]},"execution_count":2}]},{"cell_type":"markdown","metadata":{"id":"GlKDEOCrG13g"},"source":["#### [Sentence Entity Resolver for ICD10-PCS (sbiobert_base_cased_mli embeddings)](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icd10pcs_en.html)"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":481},"id":"AQjiCfFQG13h","executionInfo":{"elapsed":351462,"status":"ok","timestamp":1620101737681,"user":{"displayName":"Christian Kasim Loan","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjqAD-ircKP-s5Eh6JSdkDggDczfqQbJGU_IRb4Hw=s64","userId":"14469489166467359317"},"user_tz":-120},"outputId":"48d95631-94ec-4803-c76e-78b5123e2059"},"source":["nlu.load(\"en.embed_sentence.biobert.mli en.resolve.icd10pcs\").predict(\"\"\"This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD ,\n","gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\\'s Center for Women & Infants for cardiac\n","catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , \n","subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .\"\"\",output_level = \"sentence\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["sbiobert_base_cased_mli download started this may take some time.\n","Approximate size to download 384.3 MB\n","[OK!]\n","sbiobertresolve_icd10pcs download started this may take some time.\n","Approximate size to download 220.1 MB\n","[OK!]\n","sentence_detector_dl download started this may take some time.\n","Approximate size to download 354.6 KB\n","[OK!]\n","ner_dl_bert download started this may take some time.\n","Approximate size to download 15.4 MB\n","[OK!]\n","bert_base_cased download started this may take some time.\n","Approximate size to download 389.1 MB\n","[OK!]\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
entities_confidencetextword_embedding_bertdocumententities_classsentencesentence_resolution_icd10pcs_codesentence_resolution_icd10pcs_confidencetokensentence_resolution_icd10pcssentence_embedding_biobertentitiesorigin_index
0[0.8038, 0.9959, 0.6357, 0.9798, 0.5822, 0.8133]This is an 82 - year-old male with a history o...[[0.465839684009552, -0.5797176361083984, 0.84...This is an 82 - year-old male with a history o...[MISC, LOC, ORG, LOC, MISC, ORG]This is an 82 - year-old male with a history o...02703660.3338[This, is, an, 82, -, year-old, male, with, a,...Dilation of Coronary Artery, One Artery, Bifur...[0.2428887039422989, -0.23971685767173767, -0....[TIA, Braintree, Guaiac, St . Margaret's Cente...8589934592
\n","
"],"text/plain":[" entities_confidence ... origin_index\n","0 [0.8038, 0.9959, 0.6357, 0.9798, 0.5822, 0.8133] ... 8589934592\n","\n","[1 rows x 13 columns]"]},"metadata":{"tags":[]},"execution_count":3}]},{"cell_type":"markdown","metadata":{"id":"lMep62x_G2AS"},"source":["#### [Sentence Entity Resolver for RxCUI (sbiobert_base_cased_mli embeddings)](https://nlp.johnsnowlabs.com/2020/12/11/sbiobertresolve_rxcui_en.html)"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":431},"id":"i4Jaj97KG2AT","executionInfo":{"elapsed":375461,"status":"ok","timestamp":1620101761685,"user":{"displayName":"Christian Kasim Loan","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GjqAD-ircKP-s5Eh6JSdkDggDczfqQbJGU_IRb4Hw=s64","userId":"14469489166467359317"},"user_tz":-120},"outputId":"883c299a-7977-479a-8ed8-7779f67bc081"},"source":["nlu.load(\"en.embed_sentence.biobert.mli en.resolve.rxcui\").predict(\"He was seen by the endocrinology service and she was discharged on 50 mg of eltrombopag oral at night, 5 mg amlodipine with meals, and metformin 1000 mg two times a day\",output_level = \"sentence\")"],"execution_count":null,"outputs":[{"output_type":"stream","text":["sbiobert_base_cased_mli download started this may take some time.\n","Approximate size to download 384.3 MB\n","[OK!]\n","sbiobertresolve_rxcui download started this may take some time.\n","Approximate size to download 53.2 MB\n","[OK!]\n","sentence_detector_dl download started this may take some time.\n","Approximate size to download 354.6 KB\n","[OK!]\n","ner_dl_bert download started this may take some time.\n","Approximate size to download 15.4 MB\n","[OK!]\n","bert_base_cased download started this may take some time.\n","Approximate size to download 389.1 MB\n","[OK!]\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
sentence_resolution_rxcui_codetextword_embedding_bertdocumentsentencetokensentence_embedding_biobertsentence_resolution_rxcui_confidenceorigin_indexsentence_resolution_rxcui
02167676He was seen by the endocrinology service and s...[[0.4401422142982483, -0.7511107921600342, 0.5...He was seen by the endocrinology service and s...He was seen by the endocrinology service and s...[He, was, seen, by, the, endocrinology, servic...[0.11727594584226608, 0.17300474643707275, -0....0.3339858993459212 HR dextromethorphan hydrobromide 40 MG / gu...
\n","
"],"text/plain":[" sentence_resolution_rxcui_code ... sentence_resolution_rxcui\n","0 2167676 ... 12 HR dextromethorphan hydrobromide 40 MG / gu...\n","\n","[1 rows x 10 columns]"]},"metadata":{"tags":[]},"execution_count":4}]},{"cell_type":"markdown","metadata":{"id":"vifO7xR_LOhj"},"source":["#### [Sentence Entity Resolver for RxNorm (sbiobert_base_cased_mli embeddings)](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_rxnorm_en.html)"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":474},"id":"m8cBuMRsLOhu","executionInfo":{"status":"ok","timestamp":1620196148110,"user_tz":-300,"elapsed":278219,"user":{"displayName":"ahmed lone","photoUrl":"","userId":"02458088882398909889"}},"outputId":"1f758b49-4739-4d28-eb6e-e8c022884dc1"},"source":["nlu.load(\"en.embed_sentence.biobert.mli en.resolve.rxnorm\").predict(\"\"\"This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD ,\n","gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\\'s Center for Women & Infants for cardiac\n","catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , \n","subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .\"\"\",output_level = \"sentence\")"],"execution_count":1,"outputs":[{"output_type":"stream","text":["sbiobert_base_cased_mli download started this may take some time.\n","Approximate size to download 384.3 MB\n","[OK!]\n","sbiobertresolve_rxnorm download started this may take some time.\n","Approximate size to download 810.7 MB\n","[OK!]\n","sentence_detector_dl download started this may take some time.\n","Approximate size to download 354.6 KB\n","[OK!]\n","ner_dl_bert download started this may take some time.\n","Approximate size to download 15.4 MB\n","[OK!]\n","bert_base_cased download started this may take some time.\n","Approximate size to download 389.1 MB\n","[OK!]\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
word_embedding_bertsentence_resolution_rxnorm_codedocumenttextsentenceorigin_indexentities_classentitiessentence_resolution_rxnorm_confidencesentence_embedding_biobertsentence_resolution_rxnormentities_confidencetoken
0[[0.46583977341651917, -0.5797176957130432, 0....213388This is an 82 - year-old male with a history o...This is an 82 - year-old male with a history o...This is an 82 - year-old male with a history o...8589934592[MISC, LOC, ORG, LOC, MISC, ORG][TIA, Braintree, Guaiac, St . Margaret's Cente...0.3342[0.2428889274597168, -0.2397168129682541, -0.7...coal tar 100 mg/ml / salicylic acid 50 mg/ml m...[0.8038, 0.9959, 0.6357, 0.9798, 0.5822, 0.8133][This, is, an, 82, -, year-old, male, with, a,...
\n","
"],"text/plain":[" word_embedding_bert ... token\n","0 [[0.46583977341651917, -0.5797176957130432, 0.... ... [This, is, an, 82, -, year-old, male, with, a,...\n","\n","[1 rows x 13 columns]"]},"metadata":{"tags":[]},"execution_count":1}]},{"cell_type":"markdown","metadata":{"id":"c4y2ZMaeLOro"},"source":["#### [Sentence Entity Resolver for Snomed Concepts, INT version (sbiobert_base_cased_mli embeddings)](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_findings_int_en.html)"]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":474},"id":"XRmbCMNULOrq","executionInfo":{"status":"ok","timestamp":1620196510437,"user_tz":-300,"elapsed":209018,"user":{"displayName":"ahmed lone","photoUrl":"","userId":"02458088882398909889"}},"outputId":"a0e2ed67-5aaa-4836-9735-2481a8c79480"},"source":["nlu.load(\"en.embed_sentence.biobert.mli en.resolve.snomed.findings_int\").predict(\"\"\"This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD ,\n","gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\\'s Center for Women & Infants for cardiac\n","catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , \n","subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .\"\"\",output_level = \"sentence\")"],"execution_count":1,"outputs":[{"output_type":"stream","text":["sbiobert_base_cased_mli download started this may take some time.\n","Approximate size to download 384.3 MB\n","[OK!]\n","sbiobertresolve_snomed_findings_int download started this may take some time.\n","Approximate size to download 517.4 MB\n","[OK!]\n","sentence_detector_dl download started this may take some time.\n","Approximate size to download 354.6 KB\n","[OK!]\n","ner_dl_bert download started this may take some time.\n","Approximate size to download 15.4 MB\n","[OK!]\n","bert_base_cased download started this may take some time.\n","Approximate size to download 389.1 MB\n","[OK!]\n"],"name":"stdout"},{"output_type":"execute_result","data":{"text/html":["
\n","\n","\n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n"," \n","
sentencesentence_resolution_snomed_confidenceentitiesentities_confidenceword_embedding_bertsentence_resolution_snomed_codesentence_embedding_biobertdocumentsentence_resolution_snomedtextorigin_indexentities_classtoken
0This is an 82 - year-old male with a history o...0.3356[TIA, Braintree, Guaiac, St . Margaret's Cente...[0.8038, 0.9959, 0.6357, 0.9798, 0.5822, 0.8133][[0.46583977341651917, -0.5797176957130432, 0....15963181000119104[0.2428889274597168, -0.2397168129682541, -0.7...This is an 82 - year-old male with a history o...acute st segment elevation myocardial infarcti...This is an 82 - year-old male with a history o...8589934592[MISC, LOC, ORG, LOC, MISC, ORG][This, is, an, 82, -, year-old, male, with, a,...
\n","
"],"text/plain":[" sentence ... token\n","0 This is an 82 - year-old male with a history o... ... [This, is, an, 82, -, year-old, male, with, a,...\n","\n","[1 rows x 13 columns]"]},"metadata":{"tags":[]},"execution_count":1}]},{"cell_type":"code","metadata":{"id":"f_mvF_EQj86x","executionInfo":{"status":"aborted","timestamp":1620196177465,"user_tz":-300,"elapsed":305044,"user":{"displayName":"ahmed lone","photoUrl":"","userId":"02458088882398909889"}}},"source":[""],"execution_count":null,"outputs":[]}]} \ No newline at end of file +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "name": "entity_resolvers_overview.ipynb", + "provenance": [], + "collapsed_sections": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "8ZelAXba3D05" + }, + "source": [ + "![JohnSnowLabs](https://nlp.johnsnowlabs.com/assets/images/logo.png)\n", + "\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/JohnSnowLabs/nlu/blob/master/examples/colab/healthcare/entity_resolution/entity_resolvers_overview.ipynb)\n", + "\n", + "\n", + "\n", + "# Entity Resolution\n", + "**Named entities** are sub-strings in a text that can be classified into catogires. For example, in the String \n", + "`\"Tesla is a great stock to invest in \"` , the sub-string `\"Tesla\"` is a named entity, it can be classified with the label `company` by an ML algorithm. \n", + "**Named entities** can easily be extracted by the various pre-trained Deep Learning based NER algorithms provided by NLU. \n", + "\n", + "\n", + "\n", + "After extracting **named entities** an **entity resolution algorithm** can be applied to the extracted named entities. The resolution algorithm classifies each extracted entitiy into a class, which reduces dimensionality of the data and has many useful applications. \n", + "For example : \n", + "- \"**Tesla** is a great stock to invest in \"\n", + "- \"**TSLA** is a great stock to invest in \"\n", + "- \"**Tesla, Inc** is a great company to invest in\" \n", + "\n", + "The sub-strings `Tesla` , `TSLA` and `Tesla, Inc` are all named entities, that are classified with the labeld `company` by the NER algorithm. It tells us, all these 3 sub-strings are of type `company`, but we cannot yet infer that these 3 strings are actually referring to literally the same company. \n", + "\n", + "This exact problem is solved by the resolver algorithms, it would resolve all these 3 entities to a common name, like a company ID. This maps every reference of Tesla, regardless of how the string is represented, to the same ID.\n", + "\n", + "This example can analogusly be expanded to healthcare any any other text problems. In medical documents, the same disease can be referenced in many different ways. \n", + "\n", + "With NLU Healthcare you can leverage state of the art pre-trained NER models to extract **Medical Named Entities** (Diseases, Treatments, Posology, etc..) and **resolve these** to common **healthcare disease codes**.\n", + "\n", + "\n", + "These algorithms are based provided by **Spark NLP for Healthcare's** [SentenceEntitiyResolver](https://nlp.johnsnowlabs.com/docs/en/licensed_annotators#sentenceentityresolver) and [ChunkEntityResolvers](https://nlp.johnsnowlabs.com/docs/en/licensed_annotators#chunkentityresolver)\n", + "\n", + "\n", + "## Avaiable models\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "All the models avaiable are :\n", + "\n", + "\n", + "| Language | nlu.load() reference | Spark NLP Model reference |\n", + "| -------- | ------------------------------------------------------------ | ------------------------------------------------------------ |\n", + "| English | embed_sentence.biobert.mli | sbiobert_base_cased_mli |\n", + "| English | resolve | sbiobertresolve_cpt |\n", + "| English | resolve.cpt | sbiobertresolve_cpt |\n", + "| English | resolve.cpt.augmented | sbiobertresolve_cpt_augmented |\n", + "| English | resolve.cpt.procedures_augmented | sbiobertresolve_cpt_procedures_augmented |\n", + "| English | resolve.hcc.augmented | sbiobertresolve_hcc_augmented |\n", + "| English | [resolve.icd10cm](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icd10cm_en.html) | [sbiobertresolve_icd10cm](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icd10cm_en.html) |\n", + "| English | [resolve.icd10cm.augmented](https://nlp.johnsnowlabs.com/2020/12/13/sbiobertresolve_icd10cm_augmented_en.html) | [sbiobertresolve_icd10cm_augmented](https://nlp.johnsnowlabs.com/2020/12/13/sbiobertresolve_icd10cm_augmented_en.html) |\n", + "| English | [resolve.icd10cm.augmented_billable](https://nlp.johnsnowlabs.com/2021/02/06/sbiobertresolve_icd10cm_augmented_billable_hcc_en.html) | [sbiobertresolve_icd10cm_augmented_billable_hcc](https://nlp.johnsnowlabs.com/2021/02/06/sbiobertresolve_icd10cm_augmented_billable_hcc_en.html) |\n", + "| English | [resolve.icd10pcs](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icd10pcs_en.html) | [sbiobertresolve_icd10pcs](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icd10pcs_en.html) |\n", + "| English | [resolve.icdo](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icdo_en.html) | [sbiobertresolve_icdo](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icdo_en.html) |\n", + "| English | [resolve.rxcui](https://nlp.johnsnowlabs.com/2020/12/11/sbiobertresolve_rxcui_en.html) | [sbiobertresolve_rxcui](https://nlp.johnsnowlabs.com/2020/12/11/sbiobertresolve_rxcui_en.html) |\n", + "| English | [resolve.rxnorm](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_rxnorm_en.html) | [sbiobertresolve_rxnorm](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_rxnorm_en.html) |\n", + "| English | [resolve.snomed](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_auxConcepts_en.html) | [sbiobertresolve_snomed_auxConcepts](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_auxConcepts_en.html) |\n", + "| English | [resolve.snomed.aux_concepts](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_auxConcepts_en.html) | [sbiobertresolve_snomed_auxConcepts](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_auxConcepts_en.html) |\n", + "| English | [resolve.snomed.aux_concepts_int](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_auxConcepts_int_en.html) | [sbiobertresolve_snomed_auxConcepts_int](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_auxConcepts_int_en.html) |\n", + "| English | [resolve.snomed.findings](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_findings_en.html) | [sbiobertresolve_snomed_findings](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_findings_en.html) |\n", + "| English | [resolve.snomed.findings_int](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_findings_int_en.html) | [sbiobertresolve_snomed_findings_int](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_findings_int_en.html) |\n" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "OMiHjCtQhF_r", + "outputId": "c5a8f38d-5fc1-4553-9911-a3293b2e3f08" + }, + "source": [ + "# # Install NLU\n", + "# # Upload add your spark_nlp_fo\"r_healthcare.json\n", + "!wget http://setup.johnsnowlabs.com/nlu/colab.sh -O - | bash\n", + "import nlu" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "--2021-05-11 11:49:03-- http://setup.johnsnowlabs.com/nlu/colab.sh\n", + "Resolving setup.johnsnowlabs.com (setup.johnsnowlabs.com)... 51.158.130.125\n", + "Connecting to setup.johnsnowlabs.com (setup.johnsnowlabs.com)|51.158.130.125|:80... connected.\n", + "HTTP request sent, awaiting response... 302 Moved Temporarily\n", + "Location: https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/scripts/colab_setup.sh [following]\n", + "--2021-05-11 11:49:03-- https://raw.githubusercontent.com/JohnSnowLabs/nlu/master/scripts/colab_setup.sh\n", + "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.109.133, 185.199.110.133, ...\n", + "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n", + "HTTP request sent, awaiting response... 200 OK\n", + "Length: 1662 (1.6K) [text/plain]\n", + "Saving to: β€˜STDOUT’\n", + "\n", + "- 0%[ ] 0 --.-KB/s Installing NLU 3.0.1 with PySpark 3.0.2 and Spark NLP 3.0.1 for Google Colab ...\n", + "- 100%[===================>] 1.62K --.-KB/s in 0.001s \n", + "\n", + "2021-05-11 11:49:03 (2.14 MB/s) - written to stdout [1662/1662]\n", + "\n", + "Hit:1 https://cloud.r-project.org/bin/linux/ubuntu bionic-cran40/ InRelease\n", + "Ign:2 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 InRelease\n", + "Hit:3 http://ppa.launchpad.net/c2d4u.team/c2d4u4.0+/ubuntu bionic InRelease\n", + "Ign:4 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 InRelease\n", + "Hit:5 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 Release\n", + "Hit:6 http://security.ubuntu.com/ubuntu bionic-security InRelease\n", + "Hit:7 https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 Release\n", + "Hit:8 http://archive.ubuntu.com/ubuntu bionic InRelease\n", + "Hit:9 http://archive.ubuntu.com/ubuntu bionic-updates InRelease\n", + "Hit:10 http://ppa.launchpad.net/cran/libgit2/ubuntu bionic InRelease\n", + "Hit:11 http://archive.ubuntu.com/ubuntu bionic-backports InRelease\n", + "Hit:12 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease\n", + "Hit:13 http://ppa.launchpad.net/graphics-drivers/ppa/ubuntu bionic InRelease\n", + "Reading package lists... Done\n" + ], + "name": "stdout" + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kbnlNAw_W3Qj" + }, + "source": [ + "#### [Athena Conditions Entity Resolver (Healthcare)](https://nlp.johnsnowlabs.com/2020/09/16/chunkresolve_athena_conditions_healthcare_en.html)" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "n7zbAt-SW3Eh", + "outputId": "3a191cc7-6f77-448f-9d4b-1a831ff0ec5c" + }, + "source": [ + "data =\"\"\"The patient is a 5-month-old infant who presented initially on Monday with a cold, cough, and runny nose for 2 days. Mom states she had no fever. Her appetite was good but she was spitting up a lot. She had no difficulty breathing and her cough was described as dry and hacky. At that time, physical exam showed a right TM, which was red. Left TM was okay. She was fairly congested but looked happy and playful. She was started on Amoxil and Aldex and we told to recheck in 2 weeks to recheck her ear. Mom returned to clinic again today because she got much worse overnight. She was having difficulty breathing. She was much more congested and her appetite had decreased significantly today. She also spiked a temperature yesterday of 102.6 and always having trouble sleeping secondary to congestion.\"\"\"\n", + "nlu.load('med_ner.jsl.wip.clinical en.resolve_chunk.cpt_clinical').predict(data, output_level='chunk')" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "Spark NLP Healthcare could not be imported. Installing latest spark-nlp-jsl PyPI package via pip...\n", + "jsl_ner_wip_clinical download started this may take some time.\n", + "Approximate size to download 14.5 MB\n", + "[OK!]\n", + "chunkresolve_cpt_clinical download started this may take some time.\n", + "Approximate size to download 16.1 MB\n", + "[OK!]\n", + "embeddings_clinical download started this may take some time.\n", + "Approximate size to download 1.6 GB\n", + "[OK!]\n", + "sentence_detector_dl download started this may take some time.\n", + "Approximate size to download 354.6 KB\n", + "[OK!]\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
documententities_classentity_resolution_cpt_clinicalorigin_indexchunk_embedding_gloveword_embedding_glovesentenceentity_resolution_cpt_clinical_codeentitiesentity_resolution_cpt_clinical_confidencetexttokenentities_confidence
0The patient is a 5-month-old infant who presen...AgeRepair, initial inguinal hernia, full term inf...8589934592[0.18568818271160126, -0.5141453742980957, -0....[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...494955-month-old0.6608The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9988
0The patient is a 5-month-old infant who presen...AgeRepair, initial inguinal hernia, preterm infan...8589934592[0.0033501186408102512, -0.8093816637992859, -...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...49492infant1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9999
0The patient is a 5-month-old infant who presen...RelativeDateInduced abortion, by 1 or more vaginal supposi...8589934592[0.1907067447900772, 0.04115135222673416, -0.1...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...59857Monday1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.998
0The patient is a 5-month-old infant who presen...SymptomDonor nephrectomy (including cold preservation...8589934592[0.324726939201355, -0.137750506401062, 0.2801...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...50320cold0.5539The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.712
0The patient is a 5-month-old infant who presen...SymptomPleural scarification for repeat pneumothorax8589934592[0.39911380410194397, -0.5716836452484131, 0.2...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...32215cough0.6608The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9966
0The patient is a 5-month-old infant who presen...SymptomExcision of thyroglossal duct cyst or sinus; r...8589934592[0.19262725114822388, -0.619391918182373, 0.26...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...60281runny nose1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.4894
0The patient is a 5-month-old infant who presen...DurationReoperation, carotid, thromboendarterectomy, m...8589934592[-0.16546504199504852, -0.016397878527641296, ...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...35390for 2 days1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.50416666
0The patient is a 5-month-old infant who presen...GenderCesarean delivery only; including postpartum care8589934592[0.13232460618019104, 0.04228536784648895, -0....[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...59515Mom0.4715The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.5929
0The patient is a 5-month-old infant who presen...GenderRepair, initial inguinal hernia, full term inf...8589934592[0.19785617291927338, -0.6325990557670593, -0....[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...49495she0.6608The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9992
0The patient is a 5-month-old infant who presen...VS_FindingExploration for postoperative hemorrhage, thro...8589934592[0.19038061797618866, -0.46617722511291504, -0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...35840fever1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9978
0The patient is a 5-month-old infant who presen...GenderExploration for postoperative hemorrhage, thro...8589934592[-0.3410295248031616, -0.3789218068122864, 0.2...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...35840Her1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9973
0The patient is a 5-month-old infant who presen...GenderRepair, initial inguinal hernia, full term inf...8589934592[0.19785617291927338, -0.6325990557670593, -0....[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...49495she0.6608The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.998
0The patient is a 5-month-old infant who presen...SymptomLaparoscopy, abdomen, peritoneum, and omentum,...8589934592[0.04143378883600235, -0.4098125100135803, 0.1...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...49320spitting up a lot1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.24122499
0The patient is a 5-month-old infant who presen...GenderPericardiectomy, subtotal or complete; with ca...8589934592[-0.21964989602565765, -0.2844458520412445, -0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...33031She1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9996
0The patient is a 5-month-old infant who presen...SymptomRepair, laceration of diaphragm, any approach8589934592[0.0559418648481369, -0.18557831645011902, 0.0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...39501difficulty breathing0.9213The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.86275
0The patient is a 5-month-old infant who presen...GenderRepair, initial inguinal hernia, full term inf...8589934592[-0.16458719968795776, -0.5421502590179443, -0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...49496her1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9981
0The patient is a 5-month-old infant who presen...SymptomPleural scarification for repeat pneumothorax8589934592[0.39911380410194397, -0.5716836452484131, 0.2...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...32215cough0.6608The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.996
0The patient is a 5-month-old infant who presen...ModifierVaginal hysterectomy, for uterus 250 g or less;8589934592[-0.09050256758928299, -0.5477138757705688, 0....[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...58275dry0.8570The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.878
0The patient is a 5-month-old infant who presen...DirectionRight ventricular resection for infundibular s...8589934592[0.14579641819000244, -0.06464755535125732, -0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...33476right0.8843The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.0834
0The patient is a 5-month-old infant who presen...Internal_organ_or_componentLysis of adhesions (salpingolysis, ovariolysis)8589934592[0.09020363539457321, 0.13196095824241638, 0.2...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...58740TM1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.3906
0The patient is a 5-month-old infant who presen...DirectionRepair of double outlet right ventricle with i...8589934592[-0.11810600757598877, 0.6391716599464417, -0....[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...33612Left0.5406The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.0918
0The patient is a 5-month-old infant who presen...Internal_organ_or_componentLysis of adhesions (salpingolysis, ovariolysis)8589934592[0.09020363539457321, 0.13196095824241638, 0.2...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...58740TM1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.372
0The patient is a 5-month-old infant who presen...GenderPericardiectomy, subtotal or complete; with ca...8589934592[-0.21964989602565765, -0.2844458520412445, -0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...33031She1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9998
0The patient is a 5-month-old infant who presen...ModifierReoperation, carotid, thromboendarterectomy, m...8589934592[0.01206454448401928, 0.1365838199853897, 0.16...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...35390fairly0.9405The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.4025
0The patient is a 5-month-old infant who presen...SymptomClosure of enterostomy, large or small intestine8589934592[-0.3624162971973419, -0.38189926743507385, 0....[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...44620congested1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.7996
0The patient is a 5-month-old infant who presen...GenderPericardiectomy, subtotal or complete; with ca...8589934592[-0.21964989602565765, -0.2844458520412445, -0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...33031She1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9996
0The patient is a 5-month-old infant who presen...Drug_BrandNameVaginal hysterectomy, for uterus 250 g or less8589934592[-0.04134025424718857, -0.06713572889566422, -...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...58260Amoxil0.5551The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9898
0The patient is a 5-month-old infant who presen...Drug_BrandNameCholecystoenterostomy; with gastroenterostomy8589934592[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, ...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...47721Aldex0.3333The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9416
0The patient is a 5-month-old infant who presen...RelativeDateRepair, initial inguinal hernia, preterm infan...8589934592[-0.17584700882434845, -0.20433296263217926, -...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...49491in 2 weeks1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.5745333
0The patient is a 5-month-old infant who presen...GenderRepair, initial inguinal hernia, full term inf...8589934592[-0.16458719968795776, -0.5421502590179443, -0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...49496her1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.999
0The patient is a 5-month-old infant who presen...GenderCesarean delivery only; including postpartum care8589934592[0.13232460618019104, 0.04228536784648895, -0....[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...59515Mom0.4715The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9324
0The patient is a 5-month-old infant who presen...RelativeDateLaparoscopy, surgical, ablation of 1 or more l...8589934592[0.21045373380184174, -0.4043411910533905, 0.2...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...47371today0.9361The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...1.0
0The patient is a 5-month-old infant who presen...GenderRepair, initial inguinal hernia, full term inf...8589934592[0.19785617291927338, -0.6325990557670593, -0....[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...49495she0.6608The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9996
0The patient is a 5-month-old infant who presen...RelativeTimeReplacement or irrigation, subarachnoid/subdur...8589934592[-0.37081244587898254, -0.4993651509284973, 0....[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...62194overnight0.9975The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9646
0The patient is a 5-month-old infant who presen...GenderPericardiectomy, subtotal or complete; with ca...8589934592[-0.21964989602565765, -0.2844458520412445, -0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...33031She1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9996
0The patient is a 5-month-old infant who presen...SymptomRepair, laceration of diaphragm, any approach8589934592[0.0559418648481369, -0.18557831645011902, 0.0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...39501difficulty breathing0.9213The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9042
0The patient is a 5-month-old infant who presen...GenderPericardiectomy, subtotal or complete; with ca...8589934592[-0.21964989602565765, -0.2844458520412445, -0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...33031She1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9985
0The patient is a 5-month-old infant who presen...SymptomClosure of enterostomy, large or small intestine8589934592[-0.3624162971973419, -0.38189926743507385, 0....[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...44620congested1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.8923
0The patient is a 5-month-old infant who presen...GenderRepair, initial inguinal hernia, full term inf...8589934592[-0.16458719968795776, -0.5421502590179443, -0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...49496her1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9932
0The patient is a 5-month-old infant who presen...SymptomVaginal hysterectomy, for uterus greater than ...8589934592[0.02755250409245491, 0.016924332827329636, -0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...58293appetite had decreased significantly0.5463The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.21180001
0The patient is a 5-month-old infant who presen...RelativeDateLaparoscopy, surgical, ablation of 1 or more l...8589934592[0.21045373380184174, -0.4043411910533905, 0.2...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...47371today0.9361The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...1.0
0The patient is a 5-month-old infant who presen...GenderPericardiectomy, subtotal or complete; with ca...8589934592[-0.21964989602565765, -0.2844458520412445, -0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...33031She1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9995
0The patient is a 5-month-old infant who presen...RelativeDateLaparoscopy, surgical; cholecystoenterostomy8589934592[0.3847889304161072, -0.07220222800970078, 0.3...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...47570yesterday0.8784The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...1.0
0The patient is a 5-month-old infant who presen...TemperatureVaginal hysterectomy, for uterus greater than ...8589934592[0.1621697098016739, 0.044322118163108826, -0....[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...58291102.61.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9522
0The patient is a 5-month-old infant who presen...SymptomExploration for postoperative hemorrhage, thro...8589934592[0.17440977692604065, -0.4206063151359558, -0....[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...35840trouble sleeping1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.66429996
0The patient is a 5-month-old infant who presen...SymptomExploration for postoperative hemorrhage, thro...8589934592[-0.33774805068969727, -0.18630369007587433, 0...[[-0.0823686420917511, -0.3178570866584778, 0....[The patient is a 5-month-old infant who prese...35840congestion1.0000The patient is a 5-month-old infant who presen...[The, patient, is, a, 5-month-old, infant, who...0.9749
\n", + "
" + ], + "text/plain": [ + " document ... entities_confidence\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9988\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9999\n", + "0 The patient is a 5-month-old infant who presen... ... 0.998\n", + "0 The patient is a 5-month-old infant who presen... ... 0.712\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9966\n", + "0 The patient is a 5-month-old infant who presen... ... 0.4894\n", + "0 The patient is a 5-month-old infant who presen... ... 0.50416666\n", + "0 The patient is a 5-month-old infant who presen... ... 0.5929\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9992\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9978\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9973\n", + "0 The patient is a 5-month-old infant who presen... ... 0.998\n", + "0 The patient is a 5-month-old infant who presen... ... 0.24122499\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9996\n", + "0 The patient is a 5-month-old infant who presen... ... 0.86275\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9981\n", + "0 The patient is a 5-month-old infant who presen... ... 0.996\n", + "0 The patient is a 5-month-old infant who presen... ... 0.878\n", + "0 The patient is a 5-month-old infant who presen... ... 0.0834\n", + "0 The patient is a 5-month-old infant who presen... ... 0.3906\n", + "0 The patient is a 5-month-old infant who presen... ... 0.0918\n", + "0 The patient is a 5-month-old infant who presen... ... 0.372\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9998\n", + "0 The patient is a 5-month-old infant who presen... ... 0.4025\n", + "0 The patient is a 5-month-old infant who presen... ... 0.7996\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9996\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9898\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9416\n", + "0 The patient is a 5-month-old infant who presen... ... 0.5745333\n", + "0 The patient is a 5-month-old infant who presen... ... 0.999\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9324\n", + "0 The patient is a 5-month-old infant who presen... ... 1.0\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9996\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9646\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9996\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9042\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9985\n", + "0 The patient is a 5-month-old infant who presen... ... 0.8923\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9932\n", + "0 The patient is a 5-month-old infant who presen... ... 0.21180001\n", + "0 The patient is a 5-month-old infant who presen... ... 1.0\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9995\n", + "0 The patient is a 5-month-old infant who presen... ... 1.0\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9522\n", + "0 The patient is a 5-month-old infant who presen... ... 0.66429996\n", + "0 The patient is a 5-month-old infant who presen... ... 0.9749\n", + "\n", + "[46 rows x 13 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 3 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rSUTBWKi6Hb-" + }, + "source": [ + "#### [Sentence Entity Resolver for ICD10-CM (sbiobert_base_cased_mli embeddings)](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icd10cm_en.html)" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 474 + }, + "id": "emvKNnXE6HBv", + "outputId": "6a0d8142-ca0b-41da-94a6-9aed933c783e" + }, + "source": [ + "nlu.load(\"med_ner.jsl.wip.clinical en.resolve.icd10cm\").predict(\"\"\"This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD ,\n", + "gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\\'s Center for Women & Infants for cardiac\n", + "catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , \n", + "subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .\"\"\",output_level = \"sentence\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "jsl_ner_wip_clinical download started this may take some time.\n", + "Approximate size to download 14.5 MB\n", + "[OK!]\n", + "sbiobertresolve_icd10cm download started this may take some time.\n", + "Approximate size to download 201 MB\n", + "[OK!]\n", + "sbiobert_base_cased_mli download started this may take some time.\n", + "Approximate size to download 384.3 MB\n", + "[OK!]\n", + "embeddings_clinical download started this may take some time.\n", + "Approximate size to download 1.6 GB\n", + "[OK!]\n", + "sentence_detector_dl download started this may take some time.\n", + "Approximate size to download 354.6 KB\n", + "[OK!]\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
entities_confidencesentence_resolution_icd10cm_codesentence_resolution_icd10cm_confidencedocumententities_classorigin_indexword_embedding_glovesentenceentitiestextsentence_resolution_icd10cmtokensentence_embedding_biobert
0[0.2792667, 0.9984, 0.9998, 1.0, 0.6011, 0.998...V9184XA0.3355This is an 82 - year-old male with a history o...[Age, Gender, Smoking, Hypertension, Kidney_Di...8589934592[[0.4546172618865967, -0.3298020362854004, 0.7...This is an 82 - year-old male with a history o...[82 - year-old, male, tobacco, hypertension, c...This is an 82 - year-old male with a history o...Other injury due to other accident to sailboat...[This, is, an, 82, -, year-old, male, with, a,...[0.2428887039422989, -0.23971685767173767, -0....
\n", + "
" + ], + "text/plain": [ + " entities_confidence ... sentence_embedding_biobert\n", + "0 [0.2792667, 0.9984, 0.9998, 1.0, 0.6011, 0.998... ... [0.2428887039422989, -0.23971685767173767, -0....\n", + "\n", + "[1 rows x 13 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 4 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "GlKDEOCrG13g" + }, + "source": [ + "#### [Sentence Entity Resolver for ICD10-PCS (sbiobert_base_cased_mli embeddings)](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_icd10pcs_en.html)" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "background_save": true, + "base_uri": "https://localhost:8080/" + }, + "id": "AQjiCfFQG13h", + "outputId": "249e6af6-d7d2-43a2-e31c-2694edf86671" + }, + "source": [ + "nlu.load(\"med_ner.jsl.wip.clinical en.resolve.icd10pcs\").predict(\"\"\"This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD ,\n", + "gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\\'s Center for Women & Infants for cardiac\n", + "catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , \n", + "subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .\"\"\",output_level = \"sentence\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "jsl_ner_wip_clinical download started this may take some time.\n", + "Approximate size to download 14.5 MB\n", + "[OK!]\n", + "sbiobertresolve_icd10pcs download started this may take some time.\n", + "Approximate size to download 220.1 MB\n", + "[OK!]\n", + "sbiobert_base_cased_mli download started this may take some time.\n", + "Approximate size to download 384.3 MB\n", + "[OK!]\n", + "embeddings_clinical download started this may take some time.\n", + "Approximate size to download 1.6 GB\n", + "[OK!]\n", + "sentence_detector_dl download started this may take some time.\n", + "Approximate size to download 354.6 KB\n", + "[OK!]\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
entities_confidencedocumententities_classsentence_resolution_icd10pcssentence_resolution_icd10pcs_confidencesentence_resolution_icd10pcs_codeorigin_indexword_embedding_glovesentenceentitiestexttokensentence_embedding_biobert
0[0.2792667, 0.9984, 0.9998, 1.0, 0.6011, 0.998...This is an 82 - year-old male with a history o...[Age, Gender, Smoking, Hypertension, Kidney_Di...Dilation of Coronary Artery, One Artery, Bifur...0.333802703668589934592[[0.4546172618865967, -0.3298020362854004, 0.7...This is an 82 - year-old male with a history o...[82 - year-old, male, tobacco, hypertension, c...This is an 82 - year-old male with a history o...[This, is, an, 82, -, year-old, male, with, a,...[0.2428887039422989, -0.23971685767173767, -0....
\n", + "
" + ], + "text/plain": [ + " entities_confidence ... sentence_embedding_biobert\n", + "0 [0.2792667, 0.9984, 0.9998, 1.0, 0.6011, 0.998... ... [0.2428887039422989, -0.23971685767173767, -0....\n", + "\n", + "[1 rows x 13 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 0 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "lMep62x_G2AS" + }, + "source": [ + "#### [Sentence Entity Resolver for RxCUI (sbiobert_base_cased_mli embeddings)](https://nlp.johnsnowlabs.com/2020/12/11/sbiobertresolve_rxcui_en.html)" + ] + }, + { + "cell_type": "code", + "metadata": { + "colab": { + "background_save": true + }, + "id": "i4Jaj97KG2AT", + "outputId": "2a5b6324-db03-4756-cbeb-112c746b2bbc" + }, + "source": [ + "nlu.load(\"med_ner.jsl.wip.clinical en.resolve.rxcui\").predict(\"He was seen by the endocrinology service and she was discharged on 50 mg of eltrombopag oral at night, 5 mg amlodipine with meals, and metformin 1000 mg two times a day\",output_level = \"sentence\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "jsl_ner_wip_clinical download started this may take some time.\n", + "Approximate size to download 14.5 MB\n", + "[OK!]\n", + "sbiobertresolve_rxcui download started this may take some time.\n", + "Approximate size to download 53.2 MB\n", + "[OK!]\n", + "sbiobert_base_cased_mli download started this may take some time.\n", + "Approximate size to download 384.3 MB\n", + "[OK!]\n", + "embeddings_clinical download started this may take some time.\n", + "Approximate size to download 1.6 GB\n", + "[OK!]\n", + "sentence_detector_dl download started this may take some time.\n", + "Approximate size to download 354.6 KB\n", + "[OK!]\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
sentence_resolution_rxcui_codeentities_confidencedocumententities_classorigin_indexword_embedding_glovesentence_resolution_rxcuisentencesentence_resolution_rxcui_confidenceentitiestexttokensentence_embedding_biobert
02167676[0.9999, 0.9046, 0.9998, 0.9874, 0.8478, 0.665...He was seen by the endocrinology service and s...[Gender, Clinical_Dept, Gender, Admission_Disc...8589934592[[-0.00917550828307867, -0.16478723287582397, ...12 HR dextromethorphan hydrobromide 40 MG / gu...He was seen by the endocrinology service and s...0.3339[He, endocrinology service, she, discharged, 5...He was seen by the endocrinology service and s...[He, was, seen, by, the, endocrinology, servic...[0.11727594584226608, 0.17300474643707275, -0....
\n", + "
" + ], + "text/plain": [ + " sentence_resolution_rxcui_code ... sentence_embedding_biobert\n", + "0 2167676 ... [0.11727594584226608, 0.17300474643707275, -0....\n", + "\n", + "[1 rows x 13 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 0 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vifO7xR_LOhj" + }, + "source": [ + "#### [Sentence Entity Resolver for RxNorm (sbiobert_base_cased_mli embeddings)](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_rxnorm_en.html)" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "m8cBuMRsLOhu", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 474 + }, + "outputId": "801d6639-9625-4cbb-a41f-2a878fd4d1a7" + }, + "source": [ + "import nlu\n", + "nlu.load(\"med_ner.jsl.wip.clinical en.resolve.rxnorm\").predict(\"\"\"This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD ,\n", + "gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\\'s Center for Women & Infants for cardiac\n", + "catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , \n", + "subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .\"\"\",output_level = \"sentence\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "jsl_ner_wip_clinical download started this may take some time.\n", + "Approximate size to download 14.5 MB\n", + "[OK!]\n", + "sbiobertresolve_rxnorm download started this may take some time.\n", + "Approximate size to download 810.7 MB\n", + "[OK!]\n", + "embeddings_clinical download started this may take some time.\n", + "Approximate size to download 1.6 GB\n", + "[OK!]\n", + "sbiobert_base_cased_mli download started this may take some time.\n", + "Approximate size to download 384.3 MB\n", + "[OK!]\n", + "sentence_detector_dl download started this may take some time.\n", + "Approximate size to download 354.6 KB\n", + "[OK!]\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
sentencesentence_resolution_rxnorm_confidencetokenentities_classsentence_resolution_rxnorm_codeorigin_indexentities_confidencesentence_resolution_rxnormentitiesdocumentsentence_embedding_bioberttextword_embedding_glove
0This is an 82 - year-old male with a history o...0.3342[This, is, an, 82, -, year-old, male, with, a,...[Age, Gender, Smoking, Hypertension, Kidney_Di...2133888589934592[0.2792667, 0.9984, 0.9998, 1.0, 0.6011, 0.998...coal tar 100 mg/ml / salicylic acid 50 mg/ml m...[82 - year-old, male, tobacco, hypertension, c...This is an 82 - year-old male with a history o...[0.2428887039422989, -0.23971685767173767, -0....This is an 82 - year-old male with a history o...[[0.4546172618865967, -0.3298020362854004, 0.7...
\n", + "
" + ], + "text/plain": [ + " sentence ... word_embedding_glove\n", + "0 This is an 82 - year-old male with a history o... ... [[0.4546172618865967, -0.3298020362854004, 0.7...\n", + "\n", + "[1 rows x 13 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 1 + } + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "c4y2ZMaeLOro" + }, + "source": [ + "#### [Sentence Entity Resolver for Snomed Concepts, INT version (sbiobert_base_cased_mli embeddings)](https://nlp.johnsnowlabs.com/2020/11/27/sbiobertresolve_snomed_findings_int_en.html)" + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "XRmbCMNULOrq", + "colab": { + "base_uri": "https://localhost:8080/", + "height": 474 + }, + "outputId": "62b3f206-80fd-4322-bbb7-a8835f20940a" + }, + "source": [ + "nlu.load(\"med_ner.jsl.wip.clinical en.resolve.snomed.findings_int\").predict(\"\"\"This is an 82 - year-old male with a history of prior tobacco use , hypertension , chronic renal insufficiency , COPD ,\n", + "gastritis , and TIA who initially presented to Braintree with a non-ST elevation MI and Guaiac positive stools , transferred to St . Margaret\\'s Center for Women & Infants for cardiac\n", + "catheterization with PTCA to mid LAD lesion complicated by hypotension and bradycardia requiring Atropine , IV fluids and transient dopamine possibly secondary to vagal reaction , \n", + "subsequently transferred to CCU for close monitoring , hemodynamically stable at the time of admission to the CCU .\"\"\",output_level = \"sentence\")" + ], + "execution_count": null, + "outputs": [ + { + "output_type": "stream", + "text": [ + "jsl_ner_wip_clinical download started this may take some time.\n", + "Approximate size to download 14.5 MB\n", + "[OK!]\n", + "sbiobertresolve_snomed_findings_int download started this may take some time.\n", + "Approximate size to download 517.4 MB\n", + "[OK!]\n", + "embeddings_clinical download started this may take some time.\n", + "Approximate size to download 1.6 GB\n", + "[OK!]\n", + "sbiobert_base_cased_mli download started this may take some time.\n", + "Approximate size to download 384.3 MB\n", + "[OK!]\n", + "sentence_detector_dl download started this may take some time.\n", + "Approximate size to download 354.6 KB\n", + "[OK!]\n" + ], + "name": "stdout" + }, + { + "output_type": "execute_result", + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
entities_classsentence_resolution_snomedword_embedding_glovetextentitiessentence_resolution_snomed_confidencesentencetokensentence_resolution_snomed_codeentities_confidenceorigin_indexdocumentsentence_embedding_biobert
0[Age, Gender, Smoking, Hypertension, Kidney_Di...acute st segment elevation myocardial infarcti...[[0.4546172618865967, -0.3298020362854004, 0.7...This is an 82 - year-old male with a history o...[82 - year-old, male, tobacco, hypertension, c...0.3356This is an 82 - year-old male with a history o...[This, is, an, 82, -, year-old, male, with, a,...15963181000119104[0.2792667, 0.9984, 0.9998, 1.0, 0.6011, 0.998...8589934592This is an 82 - year-old male with a history o...[0.2428887039422989, -0.23971685767173767, -0....
\n", + "
" + ], + "text/plain": [ + " entities_class ... sentence_embedding_biobert\n", + "0 [Age, Gender, Smoking, Hypertension, Kidney_Di... ... [0.2428887039422989, -0.23971685767173767, -0....\n", + "\n", + "[1 rows x 13 columns]" + ] + }, + "metadata": { + "tags": [] + }, + "execution_count": 1 + } + ] + }, + { + "cell_type": "code", + "metadata": { + "id": "f_mvF_EQj86x" + }, + "source": [ + " " + ], + "execution_count": null, + "outputs": [] + } + ] +} \ No newline at end of file diff --git a/nlu/namespace.py b/nlu/namespace.py index 4ccc7081..3ece57b2 100644 --- a/nlu/namespace.py +++ b/nlu/namespace.py @@ -2073,8 +2073,6 @@ class NameSpace(): 'en.classify.pico':'classifierdl_pico_biobert', # 'en.classify.icd10.use':'useclassifier_icd10cm_hcc', buggy - # 3.0.3 - # 3.0.3 # 3.0.3 'en.resolve.umls':'sbiobertresolve_umls_major_concepts', 'en.resolve.umls.findings':'sbiobertresolve_umls_findings', diff --git a/setup.py b/setup.py index c085aa53..fee77bbf 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,7 @@ # For a discussion on single-sourcing the version across setup.py and the # project code, see # https://packaging.python.org/en/latest/single_source_version.html - version='3.0.1', # Required + version='3.0.2', # Required # This is a one-line description or tagline of what your project does. This # corresponds to the "Summary" metadata field: