From 214cdef86e71ae6e07bb1185aea28d8f31dc0b56 Mon Sep 17 00:00:00 2001
From: Vildan <64216738+Meryem1425@users.noreply.github.com>
Date: Mon, 25 Dec 2023 09:50:53 -0500
Subject: [PATCH] Added release notes for 520 (#839)

---
 .../_includes/docs-healthcare-pagination.html |   1 +
 docs/en/licensed_version_compatibility.md     |   3 +-
 .../licensed_release_notes.md                 | 357 +++++++++++-------
 .../release_notes_5_2_0.md                    | 332 ++++++++++++++++
 4 files changed, 552 insertions(+), 141 deletions(-)
 create mode 100644 docs/en/spark_nlp_healthcare_versions/release_notes_5_2_0.md

diff --git a/docs/_includes/docs-healthcare-pagination.html b/docs/_includes/docs-healthcare-pagination.html
index f8035c4d03..a16e65b2fd 100644
--- a/docs/_includes/docs-healthcare-pagination.html
+++ b/docs/_includes/docs-healthcare-pagination.html
@@ -10,6 +10,7 @@
     </li>
 </ul>
 <ul class="pagination owl-carousel pagination_big">
+    <li><a href="release_notes_5_2_0">5.2.0</a></li>
     <li><a href="release_notes_5_1_4">5.1.4</a></li>
     <li><a href="release_notes_5_1_3">5.1.3</a></li>
     <li><a href="release_notes_5_1_2">5.1.2</a></li>
diff --git a/docs/en/licensed_version_compatibility.md b/docs/en/licensed_version_compatibility.md
index 21e4e1bd94..dcb634154d 100644
--- a/docs/en/licensed_version_compatibility.md
+++ b/docs/en/licensed_version_compatibility.md
@@ -5,7 +5,7 @@ seotitle: Spark OCR | John Snow Labs
 title: Version Compatibility
 permalink: /docs/en/licensed_version_compatibility
 key: docs-licensed-version-compatibility
-modify_date: 2023-11-10
+modify_date: 2023-12-25
 show_nav: true
 sidebar:
     nav: sparknlp-healthcare
@@ -16,6 +16,7 @@ sidebar:
 
 | Spark NLP for Healthcare	| Spark NLP (Public) |
 |---------------------------|--------------------|
+| 5.2.0                     | 5.2.0              |
 | 5.1.4                     | 5.1.4              |
 | 5.1.3                     | 5.1.4              |
 | 5.1.2                     | 5.1.2              |
diff --git a/docs/en/spark_nlp_healthcare_versions/licensed_release_notes.md b/docs/en/spark_nlp_healthcare_versions/licensed_release_notes.md
index e3662a4320..cd48ef17a3 100644
--- a/docs/en/spark_nlp_healthcare_versions/licensed_release_notes.md
+++ b/docs/en/spark_nlp_healthcare_versions/licensed_release_notes.md
@@ -5,7 +5,7 @@ seotitle: Spark NLP for Healthcare | John Snow Labs
 title: Spark NLP for Healthcare Release Notes
 permalink: /docs/en/spark_nlp_healthcare_versions/licensed_release_notes
 key: docs-licensed-release-notes
-modify_date: 2023-11-10
+modify_date: 2023-12-25
 show_nav: true
 sidebar:
     nav: sparknlp-healthcare
@@ -13,196 +13,214 @@ sidebar:
 
 <div class="h3-box" markdown="1">
 
-## 5.1.4
+## 5.2.0
 
 #### Highlights
 
-We are delighted to announce a suite of remarkable enhancements and updates in our latest release of Spark NLP for Healthcare. **This release comes with the advanced Document Splitter annotator specifically designed for medical context and more flexibility in Deidentification as well as robust exception handling in `MedicalNerModel` for corrupted inputs**. 
+We are delighted to announce remarkable enhancements and updates in our latest release of Spark NLP for Healthcare. **This release comes with the first DocumentFiltererByClassifier annotator, RAG with SparkNLP tutorials and well as 5 new clinical pretrained models and pipelines**. Here are the newies in this release:
 
-+ Introducing the Advanced `Medical Document Splitter` annotator with more flexibility and customization for RAG pipelines
-+ Enhancing `ChunkFiltererApproach` by introducing json-based entity confidence configuration
-+ Advanced data privacy with `DeIdentification` unleashing custom regex patterns 
-+ Robust exception handling in `MedicalNerModel` for corrupted inputs 
-+ Various core improvements; bug fixes, enhanced overall robustness and reliability of Spark NLP for Healthcare
-  - Enhanced the `ChunkSentenceSplitter` annotator and revised documentation
-  - Updated the `training_log_parser` and its utility script to align with the latest NumPy version
-  - Transitioned the SecureRandom algorithm from Spark configuration to the system environment. Please check the previous [Random Seed Algorithm](https://nlp.johnsnowlabs.com/docs/en/spark_nlp_healthcare_versions/release_notes_5_1_3) implementation for details.
-  - Revised some imports for improved functionality in Deidentification
-+ New and updated demos
-  - New [Drug Resolver Demo](https://demo.johnsnowlabs.com/healthcare/ER_DRUG/)
-  - Updated [Multi Language Clinical NER Demo](https://demo.johnsnowlabs.com/healthcare/NER_CLINICAL_MULTI/) 
-  - New [Medical Document Splitter Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/38.InternalDocumentSplitter.ipynb)
-  - Updated [Clinical Named Entity Recognition Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb) with `setDoExceptionHandling` information
-  - Updated [Clinical DeIdentification Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/4.Clinical_DeIdentification.ipynb) with `setRegexPatternsDictionaryAsJsonString` and `setCombineRegexPatterns` examples
-  - Updated [Calculate Medicare Risk Adjustment Score Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.1.Calculate_Medicare_Risk_Adjustment_Score.ipynb) with the latest improvement
++ A tutorial to build a Retrieval Augmented Generation (RAG) based clinical chatbot using LLMs with John Snow Labs in Databricks.
++ Clinical document section classifier models to classify the sections within clinical documents.
++ Introducing `DocumentFiltererByClassifier` to filter the documents or document sections using classifier outcomes.
++ New Social Determinants of Health (SDOH) models that are categorizing text, identifying health risks, and recognizing mental health concerns in clinical documents.
++ Robust exception handling to allow skipping the corrupted records processed via `NERConverter`, `ChunkMapperModel`, `DocMapperModel`, `SentenceEntityResolverModel`, `RelationExtractonModel`, `RelationExtractonDLModel` and `AssertionDLModel` annotators.
++ Exploring generative AI applications: RAG-based medical chatbot notebooks in Spark NLP workshop repository.
++ Introducing the `enableSentenceIncrement` parameter in `InternalDocumentSplitter`, offering precise sentence indexing control for streamlined text processing with annotators like sentenceDetector.
++ Various core improvements; bug fixes, enhanced overall robustness and reliability of Spark NLP for Healthcare:
+    - `enableSentenceIncrement` parameter in `InternalDocumentSplitter`, offering precise sentence indexing control for streamlined text processing with annotators like SentenceDetector.
+    - Enabling `targetEntities` and `entityWeights` parameters in `ChunkEntityEmbeddings`.
+    - Refactored `ContextualParserApproach` rules instances for proper integration into LightPipelines.
+    - Fixed assertion pre-annotations issue in the NLP Lab module.
+    - Deprecated messages deleted: Removed deprecation warning for `confidenceValue` metadata in `ContextualParserApproach` and "deprecation_notice" message in `ChunkMapperModel`.  
++ Updated notebooks and demonstrations for making Spark NLP for Healthcare easier to navigate and understand
++ The additions and updates of numerous new clinical models and pipelines continue to reinforce our offering in the healthcare domain
 
-These enhancements will elevate your experience with Spark NLP for Healthcare, enabling more efficient, accurate, and streamlined healthcare-related natural language data analysis.
+These enhancements will elevate your experience with Spark NLP for Healthcare, enabling more efficient, accurate, and streamlined analysis of healthcare-related natural language data.
 
 
+</div><div class="h3-box" markdown="1">
+
+#### A tutorial to Build a Retrieval Augmented Generation (RAG) Based Clinical Catbot Using LLMs with John Snow Labs in Databricks
+
+In an era witnessing rapid advancements in Large Language Models (LLMs) and chatbot technologies, this informative session emphasizes the benefits of employing LLM systems based on RAG (Retrieval Augmented Generation). Particularly valuable in scenarios where precision in responses holds significance—such as addressing queries concerning medical patients or clinical protocols—RAG LLMs excel by providing accurate answers while minimizing imaginative outputs. They achieve this by explaining the source of each fact, mitigating errors, and utilizing private documents for information retrieval. Additionally, these systems facilitate near-real-time data updates without necessitating LLM reconfiguration.
+
+This tutorial guides participants through the creation of a RAG Large Language Model (LLM) clinical chatbot system. Leveraging John Snow Labs’ specialized healthcare-oriented LLM and NLP models integrated into the Databricks platform, the system employs LLMs to query a knowledge base via a vector database populated by Healthcare NLP at scale within a Databricks notebook. Utilizing a user-friendly graphical interface, users can engage in productive conversations with the system, thereby enhancing the efficiency and efficacy of healthcare workflows.
+
+Of paramount importance, the system is developed with a focus on data privacy, security, and compliance. It operates entirely within customers’ cloud infrastructure, ensuring zero data sharing and eliminating external API calls, thereby safeguarding sensitive healthcare information.
+
+[Webinar Slides](https://d5ln38p3754yc.cloudfront.net/content_object_shared_files/79064165adfc3626e2872ebb19ab5ed32f1bce8e/Webinar-JSL-Databricks-LangChain_122023.pdf?1702403478)
+
+[Medical Chatbot RAG JohnSnowLabs Databricks notebook link](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/databricks/python/tutorials/healthcare_case_studies/Medical_Chatbot_RAG_JohnSnowLabs_Databricks.ipynb)
+
+[![Building a RAG LLM Clinical Chatbot with John Snow Labs in Databricks](https://i.ytimg.com/vi/Q35kk-9opcw/hqdefault.jpg?sqp=-oaymwE2CPYBEIoBSFXyq4qpAygIARUAAIhCGAFwAcABBvABAfgB_gmAAtAFigIMCAAQARhFIFQoZTAP&rs=AOn4CLAXb9RdAEEa-NJqEw2KeQiFHKngvQ)](https://www.youtube.com/watch?v=Q35kk-9opcw "Click and go to the video")
 
 
 </div><div class="h3-box" markdown="1">
-  
-####  Introducing the Advanced `Medical Document Splitter` Annotator with More Flexibility and Customization for RAG Pipelines
 
-Discover our cutting-edge Internal Document Splitter—an innovative annotator designed to effortlessly break down extensive documents into manageable segments. Empowering users with the ability to define custom separators, this tool seamlessly divides texts, ensuring each chunk adheres to specified length criteria.
 
-InternalDocumentSplitter has a setSplitMode method to decide how to split documents. Default: 'regex'. It should be one of the following values:
-- `char`: Split text based on individual characters.
-- `token`: Split text based on tokens. You should supply tokens from inputCols.
-- `sentence`: Split text based on sentences. You should supply sentences from inputCols.
-- `recursive`: Split text recursively using a specific algorithm.
-- `regex`: Split text based on a regular expression pattern.
+#### Clinical Document Section Classifier Models to Classify the Sections within Clinical Documents
+
+The latest development introduces two new Section Header Classifier models aimed at leveraging Bert models for the classification of text within clinical documents, focusing on specific sections within the document structure. These models facilitate the categorization of text into distinct sections according to a defined taxonomy, including sections such as **Complications and Risk Factors, Consultation and Referral, Diagnostic and Laboratory Data, Discharge Information, Habits, History, Patient Information, Procedures, Impression, and Others**.
+
+The classifier model has a `headless` version too to cover the use cases in which there is no specific section header included in the section text. This allows classifying the sections even without a major clue (e.g. header, title, etc.) to indicate the section by itself.
+
+These models offer enhanced capabilities to precisely classify clinical document text content into specific sections, aiding in structured information extraction and facilitating streamlined analysis within healthcare and clinical research domains.
+
+| Model Name           |            Predicted Classes              |
+|----------------------|-------------------------------------------|
+| [`bert_sequence_classifier_clinical_sections`](https://nlp.johnsnowlabs.com/2023/12/21/bert_sequence_classifier_clinical_sections_en.html) | `Complications and Risk Factors`, `Consultation and Referral`, `Diagnostic and Laboratory Data`, `Discharge Information`, `Habits`, `History`, `Patient Information`, `Procedures`, `Impression`, `Other` |
+| [`bert_sequence_classifier_clinical_sections_headless`](https://nlp.johnsnowlabs.com/2023/12/21/bert_sequence_classifier_clinical_sections_headless_en.html)   | `Consultation and Referral`, `Habits`, `Complications and Risk Factors`, `Diagnostic and Laboratory Data`, `Discharge Information`, `History`, `Impression`, `Patient Information`, `Procedures`, `Other` |
 
 *Example*:
 
 ```python
-document_splitter = InternalDocumentSplitter()\
-    .setInputCols("document")\
-    .setOutputCol("splits")\
-    .setSplitMode("recursive")\
-    .setChunkSize(100)\
-    .setChunkOverlap(3)\
-    .setExplodeSplits(True)\
-    .setPatternsAreRegex(False)\
-    .setSplitPatterns(["\n\n", "\n", " "])\
-    .setKeepSeparators(False)\
-    .setTrimWhitespace(True)
-
-text = [( 
-    "The patient is a 28-year-old, who is status post gastric bypass surgery"
-    " nearly one year ago. \nHe has lost about 200 pounds and was otherwise doing well"
-    " until yesterday evening around 7:00-8:00 when he developed nausea and right upper quadrant pain," 
-    " which apparently wrapped around toward his right side and back. He feels like he was on it"
-    " but has not done so. He has overall malaise and a low-grade temperature of 100.3." 
-    " \n\nHe denies any prior similar or lesser symptoms. His last normal bowel movement was yesterday." 
-    " He denies any outright chills or blood per rectum." 
-)]
+sequenceClassifier = medical.BertForSequenceClassification\
+    .pretrained("bert_sequence_classifier_clinical_sections", "en", "clinical/models")\
+    .setInputCols(["document", "token"])\
+    .setOutputCol("prediction")\
+    .setCaseSensitive(False)
+
+text = """
+Medical Specialty: Cardiovascular / Pulmonary Sample Name: Aortic Valve Replacement
+
+Description: Aortic valve replacement using a mechanical valve and two-vessel coronary artery bypass grafting procedure using saphenous vein graft to the first obtuse marginal artery and left radial artery graft to the left anterior descending artery. (Medical Transcription Sample Report)
+
+DIAGNOSIS: Aortic valve stenosis with coronary artery disease associated with congestive heart failure. The patient has diabetes and is morbidly obese.
+
+PROCEDURES: Aortic valve replacement using a mechanical valve and two-vessel coronary artery bypass grafting procedure using saphenous vein graft to the first obtuse marginal artery and left radial artery graft to the left anterior descending artery.
+""""
 ```
 
 *Result*:
 
-|sentence                                          |doc_id|
-|--------------------------------------------------|------|
-|The patient is a 28-year-old, who is status post gastric bypass surgery nearly one year ago.       |0     |
-|He has lost about 200 pounds and was otherwise doing well until yesterday evening around 7:00-8:00 |1     |
-|when he developed nausea and right upper quadrant pain, which apparently wrapped around toward his |2     |
-|his right side and back. He feels like he was on it but has not done so. He has overall malaise and|3     |
-|and a low-grade temperature of 100.3.                                                              |4     |
-|He denies any prior similar or lesser symptoms. His last normal bowel movement was yesterday. He   |5     |
-|He denies any outright chills or blood per rectum.                                                 |6     |
+|                                                                       text|                       Classes|
+|:--------------------------------------------------------------------------|:-----------------------------|
+|         Medical Specialty:\nCardiovascular / Pulmonary\n\nSample Name: Aortic Valve Replacement\n\n|                       History|
+|Description: Aortic valve replacement using a mechanical valve and two-vessel coronary artery bypass grafting procedure using saphenous vein graft to the first obtuse marginal artery and left radial artery graft to the left anterior descending artery. (Medical Transcription Sample Report)|Complications and Risk Factors|
+|DIAGNOSIS: Aortic valve stenosis with coronary artery disease associated with congestive heart failure. The patient has diabetes and is morbidly obese. |Diagnostic and Laboratory Data|
+|PROCEDURES: Aortic valve replacement using a mechanical valve and two-vessel coronary artery bypass grafting procedure using saphenous vein graft to the first obtuse marginal artery and left radial artery graft to the left anterior descending artery.|                    Procedures|
 
-Please check [Medical Document Splitter Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/38.InternalDocumentSplitter.ipynb#scrollTo=RXIkmcBVbAZQ) for more information
+Please check [Section Header Splitter Demo](https://demo.johnsnowlabs.com/healthcare/SECTION_HEADER/)  and [Section Header Splitting and Classification Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/18.1.Section_Header_Splitting_and_Classification.ipynb)
 
 
+</div><div class="h3-box" markdown="1">
 
 
+#### Introducing `DocumentFiltererByClassifier` to Filter the Documents or Document Sections Using Classifier Outcomes
 
-</div><div class="h3-box" markdown="1">
-  
-#### Enhancing `ChunkFiltererApproach` by Introducing JSON-Based Entity Confidence Configuration
+The `DocumentFiltererByClassifier` function is designed to filter documents based on the outcomes generated by classifier annotators. It operates using a white list and a black list. The white list comprises classifier results that meet the criteria to pass through the filter, while the black list includes results that are prohibited from passing through. This filtering process is sensitive to cases by default. However, by setting `caseSensitive` to `False`, the filter becomes case-insensitive, allowing for a broader range of matches based on the specified criteria. This function serves as an effective tool for systematically sorting and managing documents based on specific classifier outcomes, facilitating streamlined document handling and organization. Here is an example on how to use this module with `DocumentSplitter` to filter out a specific section/ split based on a classifier.
 
-The new `setEntitiesConfidenceResourceAsJsonString` method allows users to finely tune entity confidence levels using a JSON configuration.
 
 *Example*:
 
 ```python
-chunk_filterer = ChunkFiltererApproach()\
-    .setInputCols("sentence","ner_chunk")\
-    .setOutputCol("chunk_filtered")\
-    .setFilterEntity("entity")\
-    .setEntitiesConfidenceResourceAsJsonString("""{'DURATION':'0.9',
-                                                  'DOSAGE':'0.9',
-                                                  'FREQUENCY':'0.9',
-                                                  'STRENGTH':'0.9',
-                                                  'DRUG':'0.9'}""")
-
-text ='The patient was prescribed 1 capsule of Advil for 5 days . He was seen by the endocrinology service and she was discharged on 40 units of insulin glargine at night , 12 units of insulin lispro with meals , metformin 1000 mg two times a day . It was determined that all SGLT2 inhibitors should be discontinued indefinitely fro 3 months .'
+document_splitter = InternalDocumentSplitter() \
+    .setInputCols("document")\
+    .setOutputCol("splits")\
+    .setSplitMode("recursive")\
+    .setChunkSize(100)\
+    .setChunkOverlap(3)\
+    .setExplodeSplits(True)\
+    .setSplitPatterns(["\n\n", "\n"])\
+
+sequenceClassifier = MedicalBertForSequenceClassification\
+    .pretrained('bert_sequence_classifier_clinical_sections', 'en', 'clinical/models')\
+    .setInputCols(["splits", "token"])\
+    .setOutputCol("prediction")\
+    .setCaseSensitive(False)
+
+document_filterer = DocumentFiltererByClassifier()\
+    .setInputCols(["splits", "prediction"])\
+    .setOutputCol("filteredDocuments")\
+    .setWhiteList(["Diagnostic and Laboratory Data"])\
+    .setCaseSensitive(False)
 ```
 
-*Without Filtering Results*:
 
-| chunks                       | begin | end | sentence_id | entities  |confidence |
-|:-----------------------------|------:|----:|------------:|:----------|--------:|
-| 1 capsule of Advil           |    27 |  44 |           0 | DRUG      |    0.64 |
-| for 5 days                   |    46 |  55 |           0 | DURATION  |    0.55 |
-| 40 units of insulin glargine |   126 | 153 |           1 | DRUG      |    0.62 |
-| at night                     |   155 | 162 |           1 | FREQUENCY |    0.74 |
-| 12 units of insulin lispro   |   166 | 191 |           1 | DRUG      |    0.67 |
-| with meals                   |   193 | 202 |           1 | FREQUENCY |    0.72 |
-| metformin 1000 mg            |   206 | 222 |           1 | DRUG      |    0.70 |
-| two times a day              |   224 | 238 |           1 | FREQUENCY |    0.67 |
-| SGLT2 inhibitors             |   269 | 284 |           2 | DRUG      |    0.89 |
+*Results before filtering*:
 
-*Filtered Results*:
+|                                                                          splits|                       classes|
+|--------------------------------------------------------------------------------|------------------------------|
+|Medical Specialty:\nCardiovascular / Pulmonary\n\nSample Name: Aortic Valve R...|                       History|
+|Description: Aortic valve replacement using a mechanical valve and two-vessel...|Complications and Risk Factors|
+|                                           (Medical Transcription Sample Report)|Complications and Risk Factors|
+|DIAGNOSIS: Aortic valve stenosis with coronary artery disease associated with...|Diagnostic and Laboratory Data|
+|PROCEDURES: Aortic valve replacement using a mechanical valve and two-vessel ...|                    Procedures|
+|                 ANESTHESIA: General endotracheal\n\nINCISION: Median sternotomy|                    Procedures|
+|INDICATIONS: The patient presented with severe congestive heart failure assoc...|     Consultation and Referral|
+|FINDINGS: The left ventricle is certainly hypertrophied· The aortic valve lea...|Diagnostic and Laboratory Data|
+|The radial artery was used for the left anterior descending artery. Flow was ...|Diagnostic and Laboratory Data|
+|PROCEDURE: The patient was brought to the operating room and placed in supine...|                    Procedures|
+|The patient went on cardiopulmonary bypass and the aortic cross-clamp was app...|                    Procedures|
+|The first obtuse marginal artery was a very large target and the vein graft t...|Diagnostic and Laboratory Data|
+|The patient came off cardiopulmonary bypass after aortic cross-clamp was rele...|                    Procedures|
 
-| chunks           | begin | end | sentence_id | entities  | confidence |
-|:-----------------|------:|----:|------------:|:----------|-----------:|
-| at night         |   155 | 162 |           1 | FREQUENCY |    0.74 |
-| with meals       |   193 | 202 |           1 | FREQUENCY |    0.72 |
-| SGLT2 inhibitors |   269 | 284 |           2 | DRUG      |    0.89 |
+*Results after filtering*:
 
+|                                                                          splits|                       classes|
+|--------------------------------------------------------------------------------|------------------------------|
+|DIAGNOSIS: Aortic valve stenosis with coronary artery disease associated with...|Diagnostic and Laboratory Data|
+|FINDINGS: The left ventricle is certainly hypertrophied· The aortic valve lea...|Diagnostic and Laboratory Data|
+|The radial artery was used for the left anterior descending artery. Flow was ...|Diagnostic and Laboratory Data|
+|The first obtuse marginal artery was a very large target and the vein graft t...|Diagnostic and Laboratory Data|
 
 
+Please check: [Section_Header_Splitting_and_Classification Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/18.1.Section_Header_Splitting_and_Classification.ipynb) for more information
 
 
 </div><div class="h3-box" markdown="1">
 
-#### Advanced Data Privacy with DeIdentification Unleashing Regex Patterns Unleashed
 
-The latest update in the DeIdentification library brings advanced data privacy with the introduction of the setRegexPatternsDictionaryAsJsonString method. This powerful feature empowers users to create custom regular expression patterns for masking specific protected entities. 
-
-*Example*:
+#### New Social Determinants Of Health (SDOH) Models That Are Categorizing Text, Identifying Health Risks And Recognizing Mental Health Concerns In Clinical Documents
 
-```python
-deid = DeIdentification()\
-    .setInputCols(["sentence", "token", "ner_chunk"]) \
-    .setOutputCol("deidentified") \
-    .setMode("mask") \
-    .setRegexPatternsDictionaryAsJsonString("{'NUMBER':'\d+'},"+
-                                            "{'NUMBER':'(\d+.?\d+.?\d+)'}")\
-    .setRegexOverride(True) # Prioritizing regex rules
-
-text ='''
-Record date : 2093-01-13 , David Hale , M.D . , Name : Hendrickson Ora , MR # 7194334 Date : 01/13/93 . PCP : Oliveira , 25 years-old , Record date : 2079-11-09 . Cocke County Baptist Hospital , Keats Street , ZIP 45662, Phone 55-555-5555 .
-'''
-```
+These new SDoH models bring in advanced categorization capabilities to assess critical factors affecting individual health.
 
-*Without RegexOverride Results (default regex)*:
+The models adeptly categorize text, assisting in identifying susceptibility to health risks related to frailty and recognizing mental health concerns within analyzed text data. Consequently, they contribute significantly to a more intricate understanding of how violence and abuse factors influence healthcare outcomes.
 
-```bash
-[Record date : <DATE> , David Hale , M.D ., , Name : Hendrickson Ora ,  Date : <DATE> ., PCP : Oliveira , 25 years-old , Record date : <DATE> ., Cocke County Baptist Hospital , Keats Street , ZIP 45662, Phone <PHONE> .]
-```
- 
-*With RegexOverride Results (custom regex)*:
+These innovative models signify a substantial leap forward in identifying and addressing key social determinants impacting overall health and well-being.
 
-```bash
-[Record date : <NUMBER> , David Hale , M.D ., , Name : Hendrickson Ora ,  Date : <NUMBER> ., PCP : Oliveira , <NUMBER> years-old , Record date : <NUMBER> ., Cocke County Baptist Hospital , Keats Street , ZIP <NUMBER>, Phone <NUMBER> .]
-```
 
-- Merging default regex rules and custom user-defined regex with `setCombineRegexPatterns`
+| Model Name           |            Predicted Classes              |
+|----------------------|-------------------------------------------|
+| [`bert_sequence_classifier_sdoh_violence_abuse`](https://nlp.johnsnowlabs.com/2023/12/20/bert_sequence_classifier_sdoh_violence_abuse_en.html) | `Domestic_Violence_Abuse`, `Personal_Violence_Abuse`, `No_Violence_Abuse`, `Unknown` |
+| [`bert_sequence_classifier_sdoh_mental_health`](https://nlp.johnsnowlabs.com/2023/12/20/bert_sequence_classifier_sdoh_mental_health_en.html)   | `Mental_Disorder`, `No_Or_Not_Mentioned` |
+| [`bert_sequence_classifier_sdoh_frailty_vulnerability`](https://nlp.johnsnowlabs.com/2023/12/20/bert_sequence_classifier_sdoh_frailty_vulnerability_en.html) |  `Frailty_Vulnerability`, `No_Or_Unknown` |
 
 *Example*:
 
 ```python
-deid = DeIdentification()\
-    .setInputCols(["sentence", "token", "ner_chunk"]) \
-    .setOutputCol("deidentified") \
-    .setMode("mask") \
-    .setCombineRegexPatterns(True)\
-    .setRegexPatternsDictionary("./custom_regex.txt")
+sequenceClassifier = MedicalBertForSequenceClassification\
+    .pretrained("bert_sequence_classifier_sdoh_violence_abuse", "en", "clinical/models")\
+    .setInputCols(["document", "token"])\
+    .setOutputCol("prediction")
+
+sample_texts = [
+    ["Repeated visits for fractures, with vague explanations suggesting potential family-related trauma."],
+    ["Patient presents with multiple bruises in various stages of healing, suggestive of repeated physical abuse."],
+    ["There are no reported instances or documented episodes indicating the patient poses a risk of violence."] ,
+    ["Patient B is a 40-year-old female who was diagnosed with breast cancer. She has received a treatment plan that includes surgery, chemotherapy, and radiation therapy."]
+]
 ```
 
-Please check: [Clinical DeIdentification Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/4.Clinical_DeIdentification.ipynb) for more information
 
+*Result*:
+
+|   text                |      result|
+|-----------------------|------------|
+|Repeated visits for fractures, with vague explanations suggesting potential family-related trauma.  | Domestic_Violence_Abuse |
+|Patient presents with multiple bruises in various stages of healing, suggestive of repeated physi...| Personal_Violence_Abuse |
+|There are no reported instances or documented episodes indicating the patient poses a risk of vio...|       No_Violence_Abuse |
+|Patient B is a 40-year-old female who was diagnosed with breast cancer. She has received a treatm...|                 Unknown |
 
 
+Please check [SOCIAL DETERMINANT CLASSIFICATION Demo](https://demo.johnsnowlabs.com/healthcare/SOCIAL_DETERMINANT_SEQUENCE_CLASSIFICATION/)
+
 
 </div><div class="h3-box" markdown="1">
 
-####  Robust Exception Handling in `MedicalNerModel` for Corrupted Inputs
+#### Robust Exception Handling To Allow Skipping The Corrupted Records Processed Via `NERConverter`, `ChunkMapperModel`, `DocMapperModel`, `SentenceEntityResolverModel`, `RelationExtractonModel`, `RelationExtractonDLModel` and `AssertionDLModel` Annotators
 
-Enhance the resilience of your Medical Named Entity Recognition (NER) model with the ExceptionHandling feature. When the `setDoExceptionHandling` is set to true, the model attempts to compute batch-wise as usual. In the event of an exception within a batch, the system switches to row-wise processing. Any exception during row processing results in the emission of an Error Annotation, ensuring that only the problematic rows are lost rather than the entire batch. 
+We added `doExceptionHandling` parameter into `NERConverterInternal`, `ChunkMapperModel`, `DocMapperModel`, `SentenceEntityResolverModel`, `RelationExtractonModel`, `RelationExtractonDLModel` and  `AssertionDLModel` annotators for a robust exception handling if the process is broken down due to corrupted inputs. If it is set as `True`, the annotator tries to process as usual and ff exception-causing data (e.g. corrupted record/ document) is passed to the annotator, an exception warning is emitted which has the exception message. **Processing continues with the next one** while the rest of the drecords within the same batch is parsed without interruption. The default behavious is `False` and will throw exception and break the process to inform users.
 
 *Example*:
 
@@ -213,9 +231,56 @@ clinical_ner = MedicalNerModel.pretrained("ner_oncology", "en", "clinical/models
     .setDoExceptionHandling(True)
 ```
 
-Please check: [Clinical Named Entity Recognition Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb) for more information
+</div><div class="h3-box" markdown="1">
+
+#### Exploring Generative AI Applications: RAG-Based Medical Chatbot Notebooks in Spark NLP Workshop Repository
+
+Within the workshop repository of Spark NLP, a new dedicated folder for generative AI notebook examples has been introduced at [generative-ai](https://github.com/JohnSnowLabs/spark-nlp-workshop/tree/master/generative-ai). This repository section hosts two notable notebooks:
+
+- [Medical Chatbot RAG JohnSnowLabs LangChain Notebook](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/generative-ai/Medical_Chatbot_RAG_JohnSnowLabs_LangChain.ipynb): This notebook delves into the application of Retrieval Augmented Generation (RAG) in developing a medical chatbot. Leveraging John Snow Labs' LangChain, this notebook explores the implementation and functionalities of RAG technology in healthcare contexts.
+
+- [Medical Chatbot RAG JohnSnowLabs Haystack Notebook](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/generative-ai/Medical_Chatbot_RAG_JohnSnowLabs_Haystack.ipynb): Another notebook dedicated to RAG-based medical chatbot development within the John Snow Labs ecosystem, this time utilizing Haystack. The notebook likely offers unique insights and approaches to creating medical chatbots using RAG within the John Snow Labs' framework.
+
+These notebooks provide valuable resources for individuals interested in understanding and implementing RAG-based generative AI specifically tailored for medical chatbot applications.
+
+
+</div><div class="h3-box" markdown="1">
+
+#### Introducing The `enableSentenceIncrement` Parameter In `InternalDocumentSplitter`, Offering Precise Sentence Indexing Control For Streamlined Text Processing With Annotators Like `SentenceDetector`
+
+The `enableSentenceIncrement` parameter holds significance as it directly influences the management of sentence indexing within the document splitting process. By regulating this feature, users gain more precise control over sentence index increments in metadata. This functionality is particularly valuable when employing split documents in subsequent annotators, such as the `SentenceDetector`, ensuring accurate and streamlined processing of text data.
+
+*Example*:
+
+```python
+document_splitter = InternalDocumentSplitter() \
+    .setInputCols(["document", "sentence"]) \
+    .setOutputCol("splits") \
+    .setSplitMode("sentence") \
+    .setMaxLength(2)\
+    .setExplodeSplits(True)\
+    .setEnableSentenceIncrement(True)
+
+sample_text = """Sample Name: Aortic Valve Replacement
+Description: Aortic valve replacement using a mechanical valve and two-vessel coronary artery bypass grafting procedure using saphenous vein graft to the first obtuse marginal artery and left radial artery graft to the left anterior descending artery.
+(Medical Transcription Sample Report)
+DIAGNOSIS: Aortic valve stenosis with coronary artery disease associated with congestive heart failure. The patient has diabetes and is morbidly obese.
+PROCEDURES: Aortic valve replacement using a mechanical valve and two-vessel coronary artery bypass grafting procedure using saphenous vein graft to the first obtuse marginal artery and left radial artery graft to the left anterior descending artery.
+"""
+```
+
+*Result*:
 
+|metadata                                 | result       |
+|:----------------------------------------|:-------------|
+| {id -> 0, sentence -> 0, document -> 0} |Description: Aortic valve replacement using a mechanical valve and two-vessel coronary artery byp...|
+| {id -> 0, sentence -> 1, document -> 1} |The patient has diabetes and is morbidly obese.\nANESTHESIA: General endotracheal\nINCISION: Medi...|
+| {id -> 0, sentence -> 2, document -> 2} |INDICATIONS: The patient presented with severe congestive heart failure associated with the patie...|
+| {id -> 0, sentence -> 3, document -> 3} |In addition, The patient had significant coronary artery disease consisting of a chronically occl...|
+| {id -> 0, sentence -> 4, document -> 4} |It was decided to perform a valve replacement as well as coronary artery bypass grafting procedur...|
+| {id -> 0, sentence -> 5, document -> 5} |It is a tricuspid type of valve. The coronary artery consists of a large left anterior descending...|
 
+Please check: [Internal Document Splitter Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/38.InternalDocumentSplitter.ipynb) for more information
 
 
 
@@ -223,21 +288,33 @@ Please check: [Clinical Named Entity Recognition Notebook](https://colab.researc
 
 #### Various Core Improvements: Bug Fixes, Enhanced Overall Robustness, And Reliability Of Spark NLP For Healthcare
 
-- Enhanced the `ChunkSentenceSplitter` annotator and revised documentation
-- Updated the `training_log_parser` and its utility script to align with the latest NumPy version
-- Transitioned the SecureRandom algorithm from Spark configuration to the system environment. Please check the previous [Random Seed Algorithm](https://nlp.johnsnowlabs.com/docs/en/spark_nlp_healthcare_versions/release_notes_5_1_3) implementation for details.
-- Revised some imports for improved functionality in Deidentification
++ Enabling `targetEntities` and `entityWeights` parameters in `ChunkEntityEmbeddings`
++ Refactored `ContextualParserApproach` rules instances for proper integration into LightPipelines
++ Fixed assertion pre-annotations issue in the ALAB module
++ Deprecated Messages Deletion: Removed deprecation warning for confidenceValue metadata in `ContextualParserApproach` and "deprecation_notice" message in `ChunkMapperModel`  
+
 
 </div><div class="h3-box" markdown="1">
 
 #### Updated Notebooks And Demonstrations For making Spark NLP For Healthcare Easier To Navigate And Understand
 
-- New [Drug Resolver Demo](https://demo.johnsnowlabs.com/healthcare/ER_DRUG/)
-- Updated [Multi Language Clinical NER Demo](https://demo.johnsnowlabs.com/healthcare/NER_CLINICAL_MULTI/) 
-- New [Medical Document Splitter Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/38.InternalDocumentSplitter.ipynb)
-- Updated [Clinical Named Entity Recognition Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/1.Clinical_Named_Entity_Recognition_Model.ipynb) with `setDoExceptionHandling` information
-- Updated [Clinical DeIdentification Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/4.Clinical_DeIdentification.ipynb) with `setRegexPatternsDictionaryAsJsonString` and `setCombineRegexPatterns` examples
-- Updated [Calculate Medicare Risk Adjustment Score Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.1.Calculate_Medicare_Risk_Adjustment_Score.ipynb) with the latest improvement
+- New [Medical Chatbot RAG JohnSnowLabs LangChain Notebook](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/generative-ai/Medical_Chatbot_RAG_JohnSnowLabs_LangChain.ipynb)
+- New [Medical Chatbot RAG JohnSnowLabs Haystack Notebook](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/generative-ai/Medical_Chatbot_RAG_JohnSnowLabs_Haystack.ipynb)
+- New [Section Header Splitting and Classification Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/18.1.Section_Header_Splitting_and_Classification.ipynb)
+- Updated [Clinical Entity Resolvers Notebook](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb) with new examples
+- New [Section Header Splitter Demo](https://demo.johnsnowlabs.com/healthcare/SECTION_HEADER/)
+- Updated [Social Determinant Classification Demo](https://demo.johnsnowlabs.com/healthcare/SOCIAL_DETERMINANT_SEQUENCE_CLASSIFICATION/)
+
+
+</div><div class="h3-box" markdown="1">
+
+#### We Have Added And Updated A Substantial Number Of New Clinical Models And Pipelines, Further Solidifying Our Offering In The Healthcare Domain.
+
++ `bert_sequence_classifier_sdoh_violence_abuse`
++ `bert_sequence_classifier_sdoh_mental_health`
++ `bert_sequence_classifier_sdoh_frailty_vulnerability`
++ `bert_sequence_classifier_clinical_sections`
++ `bert_sequence_classifier_clinical_sections_headless`
 
 
 
diff --git a/docs/en/spark_nlp_healthcare_versions/release_notes_5_2_0.md b/docs/en/spark_nlp_healthcare_versions/release_notes_5_2_0.md
new file mode 100644
index 0000000000..bf40aba9d2
--- /dev/null
+++ b/docs/en/spark_nlp_healthcare_versions/release_notes_5_2_0.md
@@ -0,0 +1,332 @@
+---
+layout: docs
+header: true
+seotitle: Spark NLP for Healthcare | John Snow Labs
+title: Spark NLP for Healthcare Release Notes 5.2.0
+permalink: /docs/en/spark_nlp_healthcare_versions/release_notes_5_2_0
+key: docs-licensed-release-notes
+modify_date: 2023-12-25
+show_nav: true
+sidebar:
+    nav: sparknlp-healthcare
+---
+
+<div class="h3-box" markdown="1">
+
+## 5.2.0
+
+#### Highlights
+
+We are delighted to announce remarkable enhancements and updates in our latest release of Spark NLP for Healthcare. **This release comes with the first DocumentFiltererByClassifier annotator, RAG with SparkNLP tutorials and well as 5 new clinical pretrained models and pipelines**. Here are the newies in this release:
+
++ A tutorial to build a Retrieval Augmented Generation (RAG) based clinical chatbot using LLMs with John Snow Labs in Databricks.
++ Clinical document section classifier models to classify the sections within clinical documents.
++ Introducing `DocumentFiltererByClassifier` to filter the documents or document sections using classifier outcomes.
++ New Social Determinants of Health (SDOH) models that are categorizing text, identifying health risks, and recognizing mental health concerns in clinical documents.
++ Robust exception handling to allow skipping the corrupted records processed via `NERConverter`, `ChunkMapperModel`, `DocMapperModel`, `SentenceEntityResolverModel`, `RelationExtractonModel`, `RelationExtractonDLModel` and `AssertionDLModel` annotators.
++ Exploring generative AI applications: RAG-based medical chatbot notebooks in Spark NLP workshop repository.
++ Introducing the `enableSentenceIncrement` parameter in `InternalDocumentSplitter`, offering precise sentence indexing control for streamlined text processing with annotators like sentenceDetector.
++ Various core improvements; bug fixes, enhanced overall robustness and reliability of Spark NLP for Healthcare:
+    - `enableSentenceIncrement` parameter in `InternalDocumentSplitter`, offering precise sentence indexing control for streamlined text processing with annotators like SentenceDetector.
+    - Enabling `targetEntities` and `entityWeights` parameters in `ChunkEntityEmbeddings`.
+    - Refactored `ContextualParserApproach` rules instances for proper integration into LightPipelines.
+    - Fixed assertion pre-annotations issue in the NLP Lab module.
+    - Deprecated messages deleted: Removed deprecation warning for `confidenceValue` metadata in `ContextualParserApproach` and "deprecation_notice" message in `ChunkMapperModel`.  
++ Updated notebooks and demonstrations for making Spark NLP for Healthcare easier to navigate and understand
++ The additions and updates of numerous new clinical models and pipelines continue to reinforce our offering in the healthcare domain
+
+These enhancements will elevate your experience with Spark NLP for Healthcare, enabling more efficient, accurate, and streamlined analysis of healthcare-related natural language data.
+
+
+</div><div class="h3-box" markdown="1">
+
+#### A tutorial to Build a Retrieval Augmented Generation (RAG) Based Clinical Catbot Using LLMs with John Snow Labs in Databricks
+
+In an era witnessing rapid advancements in Large Language Models (LLMs) and chatbot technologies, this informative session emphasizes the benefits of employing LLM systems based on RAG (Retrieval Augmented Generation). Particularly valuable in scenarios where precision in responses holds significance—such as addressing queries concerning medical patients or clinical protocols—RAG LLMs excel by providing accurate answers while minimizing imaginative outputs. They achieve this by explaining the source of each fact, mitigating errors, and utilizing private documents for information retrieval. Additionally, these systems facilitate near-real-time data updates without necessitating LLM reconfiguration.
+
+This tutorial guides participants through the creation of a RAG Large Language Model (LLM) clinical chatbot system. Leveraging John Snow Labs’ specialized healthcare-oriented LLM and NLP models integrated into the Databricks platform, the system employs LLMs to query a knowledge base via a vector database populated by Healthcare NLP at scale within a Databricks notebook. Utilizing a user-friendly graphical interface, users can engage in productive conversations with the system, thereby enhancing the efficiency and efficacy of healthcare workflows.
+
+Of paramount importance, the system is developed with a focus on data privacy, security, and compliance. It operates entirely within customers’ cloud infrastructure, ensuring zero data sharing and eliminating external API calls, thereby safeguarding sensitive healthcare information.
+
+[Webinar Slides](https://d5ln38p3754yc.cloudfront.net/content_object_shared_files/79064165adfc3626e2872ebb19ab5ed32f1bce8e/Webinar-JSL-Databricks-LangChain_122023.pdf?1702403478)
+
+[Medical Chatbot RAG JohnSnowLabs Databricks notebook link](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/databricks/python/tutorials/healthcare_case_studies/Medical_Chatbot_RAG_JohnSnowLabs_Databricks.ipynb)
+
+[![Building a RAG LLM Clinical Chatbot with John Snow Labs in Databricks](https://i.ytimg.com/vi/Q35kk-9opcw/hqdefault.jpg?sqp=-oaymwE2CPYBEIoBSFXyq4qpAygIARUAAIhCGAFwAcABBvABAfgB_gmAAtAFigIMCAAQARhFIFQoZTAP&rs=AOn4CLAXb9RdAEEa-NJqEw2KeQiFHKngvQ)](https://www.youtube.com/watch?v=Q35kk-9opcw "Click and go to the video")
+
+
+</div><div class="h3-box" markdown="1">
+
+
+#### Clinical Document Section Classifier Models to Classify the Sections within Clinical Documents
+
+The latest development introduces two new Section Header Classifier models aimed at leveraging Bert models for the classification of text within clinical documents, focusing on specific sections within the document structure. These models facilitate the categorization of text into distinct sections according to a defined taxonomy, including sections such as **Complications and Risk Factors, Consultation and Referral, Diagnostic and Laboratory Data, Discharge Information, Habits, History, Patient Information, Procedures, Impression, and Others**.
+
+The classifier model has a `headless` version too to cover the use cases in which there is no specific section header included in the section text. This allows classifying the sections even without a major clue (e.g. header, title, etc.) to indicate the section by itself.
+
+These models offer enhanced capabilities to precisely classify clinical document text content into specific sections, aiding in structured information extraction and facilitating streamlined analysis within healthcare and clinical research domains.
+
+| Model Name           |            Predicted Classes              |
+|----------------------|-------------------------------------------|
+| [`bert_sequence_classifier_clinical_sections`](https://nlp.johnsnowlabs.com/2023/12/21/bert_sequence_classifier_clinical_sections_en.html) | `Complications and Risk Factors`, `Consultation and Referral`, `Diagnostic and Laboratory Data`, `Discharge Information`, `Habits`, `History`, `Patient Information`, `Procedures`, `Impression`, `Other` |
+| [`bert_sequence_classifier_clinical_sections_headless`](https://nlp.johnsnowlabs.com/2023/12/21/bert_sequence_classifier_clinical_sections_headless_en.html)   | `Consultation and Referral`, `Habits`, `Complications and Risk Factors`, `Diagnostic and Laboratory Data`, `Discharge Information`, `History`, `Impression`, `Patient Information`, `Procedures`, `Other` |
+
+*Example*:
+
+```python
+sequenceClassifier = medical.BertForSequenceClassification\
+    .pretrained("bert_sequence_classifier_clinical_sections", "en", "clinical/models")\
+    .setInputCols(["document", "token"])\
+    .setOutputCol("prediction")\
+    .setCaseSensitive(False)
+
+text = """
+Medical Specialty: Cardiovascular / Pulmonary Sample Name: Aortic Valve Replacement
+
+Description: Aortic valve replacement using a mechanical valve and two-vessel coronary artery bypass grafting procedure using saphenous vein graft to the first obtuse marginal artery and left radial artery graft to the left anterior descending artery. (Medical Transcription Sample Report)
+
+DIAGNOSIS: Aortic valve stenosis with coronary artery disease associated with congestive heart failure. The patient has diabetes and is morbidly obese.
+
+PROCEDURES: Aortic valve replacement using a mechanical valve and two-vessel coronary artery bypass grafting procedure using saphenous vein graft to the first obtuse marginal artery and left radial artery graft to the left anterior descending artery.
+""""
+```
+
+*Result*:
+
+|                                                                       text|                       Classes|
+|:--------------------------------------------------------------------------|:-----------------------------|
+|         Medical Specialty:\nCardiovascular / Pulmonary\n\nSample Name: Aortic Valve Replacement\n\n|                       History|
+|Description: Aortic valve replacement using a mechanical valve and two-vessel coronary artery bypass grafting procedure using saphenous vein graft to the first obtuse marginal artery and left radial artery graft to the left anterior descending artery. (Medical Transcription Sample Report)|Complications and Risk Factors|
+|DIAGNOSIS: Aortic valve stenosis with coronary artery disease associated with congestive heart failure. The patient has diabetes and is morbidly obese. |Diagnostic and Laboratory Data|
+|PROCEDURES: Aortic valve replacement using a mechanical valve and two-vessel coronary artery bypass grafting procedure using saphenous vein graft to the first obtuse marginal artery and left radial artery graft to the left anterior descending artery.|                    Procedures|
+
+Please check [Section Header Splitter Demo](https://demo.johnsnowlabs.com/healthcare/SECTION_HEADER/)  and [Section Header Splitting and Classification Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/18.1.Section_Header_Splitting_and_Classification.ipynb)
+
+
+</div><div class="h3-box" markdown="1">
+
+
+#### Introducing `DocumentFiltererByClassifier` to Filter the Documents or Document Sections Using Classifier Outcomes
+
+The `DocumentFiltererByClassifier` function is designed to filter documents based on the outcomes generated by classifier annotators. It operates using a white list and a black list. The white list comprises classifier results that meet the criteria to pass through the filter, while the black list includes results that are prohibited from passing through. This filtering process is sensitive to cases by default. However, by setting `caseSensitive` to `False`, the filter becomes case-insensitive, allowing for a broader range of matches based on the specified criteria. This function serves as an effective tool for systematically sorting and managing documents based on specific classifier outcomes, facilitating streamlined document handling and organization. Here is an example on how to use this module with `DocumentSplitter` to filter out a specific section/ split based on a classifier.
+
+
+*Example*:
+
+```python
+document_splitter = InternalDocumentSplitter() \
+    .setInputCols("document")\
+    .setOutputCol("splits")\
+    .setSplitMode("recursive")\
+    .setChunkSize(100)\
+    .setChunkOverlap(3)\
+    .setExplodeSplits(True)\
+    .setSplitPatterns(["\n\n", "\n"])\
+
+sequenceClassifier = MedicalBertForSequenceClassification\
+    .pretrained('bert_sequence_classifier_clinical_sections', 'en', 'clinical/models')\
+    .setInputCols(["splits", "token"])\
+    .setOutputCol("prediction")\
+    .setCaseSensitive(False)
+
+document_filterer = DocumentFiltererByClassifier()\
+    .setInputCols(["splits", "prediction"])\
+    .setOutputCol("filteredDocuments")\
+    .setWhiteList(["Diagnostic and Laboratory Data"])\
+    .setCaseSensitive(False)
+```
+
+
+*Results before filtering*:
+
+|                                                                          splits|                       classes|
+|--------------------------------------------------------------------------------|------------------------------|
+|Medical Specialty:\nCardiovascular / Pulmonary\n\nSample Name: Aortic Valve R...|                       History|
+|Description: Aortic valve replacement using a mechanical valve and two-vessel...|Complications and Risk Factors|
+|                                           (Medical Transcription Sample Report)|Complications and Risk Factors|
+|DIAGNOSIS: Aortic valve stenosis with coronary artery disease associated with...|Diagnostic and Laboratory Data|
+|PROCEDURES: Aortic valve replacement using a mechanical valve and two-vessel ...|                    Procedures|
+|                 ANESTHESIA: General endotracheal\n\nINCISION: Median sternotomy|                    Procedures|
+|INDICATIONS: The patient presented with severe congestive heart failure assoc...|     Consultation and Referral|
+|FINDINGS: The left ventricle is certainly hypertrophied· The aortic valve lea...|Diagnostic and Laboratory Data|
+|The radial artery was used for the left anterior descending artery. Flow was ...|Diagnostic and Laboratory Data|
+|PROCEDURE: The patient was brought to the operating room and placed in supine...|                    Procedures|
+|The patient went on cardiopulmonary bypass and the aortic cross-clamp was app...|                    Procedures|
+|The first obtuse marginal artery was a very large target and the vein graft t...|Diagnostic and Laboratory Data|
+|The patient came off cardiopulmonary bypass after aortic cross-clamp was rele...|                    Procedures|
+
+*Results after filtering*:
+
+|                                                                          splits|                       classes|
+|--------------------------------------------------------------------------------|------------------------------|
+|DIAGNOSIS: Aortic valve stenosis with coronary artery disease associated with...|Diagnostic and Laboratory Data|
+|FINDINGS: The left ventricle is certainly hypertrophied· The aortic valve lea...|Diagnostic and Laboratory Data|
+|The radial artery was used for the left anterior descending artery. Flow was ...|Diagnostic and Laboratory Data|
+|The first obtuse marginal artery was a very large target and the vein graft t...|Diagnostic and Laboratory Data|
+
+
+Please check: [Section_Header_Splitting_and_Classification Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/18.1.Section_Header_Splitting_and_Classification.ipynb) for more information
+
+
+</div><div class="h3-box" markdown="1">
+
+
+#### New Social Determinants Of Health (SDOH) Models That Are Categorizing Text, Identifying Health Risks And Recognizing Mental Health Concerns In Clinical Documents
+
+These new SDoH models bring in advanced categorization capabilities to assess critical factors affecting individual health.
+
+The models adeptly categorize text, assisting in identifying susceptibility to health risks related to frailty and recognizing mental health concerns within analyzed text data. Consequently, they contribute significantly to a more intricate understanding of how violence and abuse factors influence healthcare outcomes.
+
+These innovative models signify a substantial leap forward in identifying and addressing key social determinants impacting overall health and well-being.
+
+
+| Model Name           |            Predicted Classes              |
+|----------------------|-------------------------------------------|
+| [`bert_sequence_classifier_sdoh_violence_abuse`](https://nlp.johnsnowlabs.com/2023/12/20/bert_sequence_classifier_sdoh_violence_abuse_en.html) | `Domestic_Violence_Abuse`, `Personal_Violence_Abuse`, `No_Violence_Abuse`, `Unknown` |
+| [`bert_sequence_classifier_sdoh_mental_health`](https://nlp.johnsnowlabs.com/2023/12/20/bert_sequence_classifier_sdoh_mental_health_en.html)   | `Mental_Disorder`, `No_Or_Not_Mentioned` |
+| [`bert_sequence_classifier_sdoh_frailty_vulnerability`](https://nlp.johnsnowlabs.com/2023/12/20/bert_sequence_classifier_sdoh_frailty_vulnerability_en.html) |  `Frailty_Vulnerability`, `No_Or_Unknown` |
+
+*Example*:
+
+```python
+sequenceClassifier = MedicalBertForSequenceClassification\
+    .pretrained("bert_sequence_classifier_sdoh_violence_abuse", "en", "clinical/models")\
+    .setInputCols(["document", "token"])\
+    .setOutputCol("prediction")
+
+sample_texts = [
+    ["Repeated visits for fractures, with vague explanations suggesting potential family-related trauma."],
+    ["Patient presents with multiple bruises in various stages of healing, suggestive of repeated physical abuse."],
+    ["There are no reported instances or documented episodes indicating the patient poses a risk of violence."] ,
+    ["Patient B is a 40-year-old female who was diagnosed with breast cancer. She has received a treatment plan that includes surgery, chemotherapy, and radiation therapy."]
+]
+```
+
+
+*Result*:
+
+|   text                |      result|
+|-----------------------|------------|
+|Repeated visits for fractures, with vague explanations suggesting potential family-related trauma.  | Domestic_Violence_Abuse |
+|Patient presents with multiple bruises in various stages of healing, suggestive of repeated physi...| Personal_Violence_Abuse |
+|There are no reported instances or documented episodes indicating the patient poses a risk of vio...|       No_Violence_Abuse |
+|Patient B is a 40-year-old female who was diagnosed with breast cancer. She has received a treatm...|                 Unknown |
+
+
+Please check [SOCIAL DETERMINANT CLASSIFICATION Demo](https://demo.johnsnowlabs.com/healthcare/SOCIAL_DETERMINANT_SEQUENCE_CLASSIFICATION/)
+
+
+</div><div class="h3-box" markdown="1">
+
+#### Robust Exception Handling To Allow Skipping The Corrupted Records Processed Via `NERConverter`, `ChunkMapperModel`, `DocMapperModel`, `SentenceEntityResolverModel`, `RelationExtractonModel`, `RelationExtractonDLModel` and `AssertionDLModel` Annotators
+
+We added `doExceptionHandling` parameter into `NERConverterInternal`, `ChunkMapperModel`, `DocMapperModel`, `SentenceEntityResolverModel`, `RelationExtractonModel`, `RelationExtractonDLModel` and  `AssertionDLModel` annotators for a robust exception handling if the process is broken down due to corrupted inputs. If it is set as `True`, the annotator tries to process as usual and ff exception-causing data (e.g. corrupted record/ document) is passed to the annotator, an exception warning is emitted which has the exception message. **Processing continues with the next one** while the rest of the drecords within the same batch is parsed without interruption. The default behavious is `False` and will throw exception and break the process to inform users.
+
+*Example*:
+
+```python
+clinical_ner = MedicalNerModel.pretrained("ner_oncology", "en", "clinical/models") \
+    .setInputCols(["sentence", "token", "embeddings"]) \
+    .setOutputCol("ner")\
+    .setDoExceptionHandling(True)
+```
+
+</div><div class="h3-box" markdown="1">
+
+#### Exploring Generative AI Applications: RAG-Based Medical Chatbot Notebooks in Spark NLP Workshop Repository
+
+Within the workshop repository of Spark NLP, a new dedicated folder for generative AI notebook examples has been introduced at [generative-ai](https://github.com/JohnSnowLabs/spark-nlp-workshop/tree/master/generative-ai). This repository section hosts two notable notebooks:
+
+- [Medical Chatbot RAG JohnSnowLabs LangChain Notebook](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/generative-ai/Medical_Chatbot_RAG_JohnSnowLabs_LangChain.ipynb): This notebook delves into the application of Retrieval Augmented Generation (RAG) in developing a medical chatbot. Leveraging John Snow Labs' LangChain, this notebook explores the implementation and functionalities of RAG technology in healthcare contexts.
+
+- [Medical Chatbot RAG JohnSnowLabs Haystack Notebook](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/generative-ai/Medical_Chatbot_RAG_JohnSnowLabs_Haystack.ipynb): Another notebook dedicated to RAG-based medical chatbot development within the John Snow Labs ecosystem, this time utilizing Haystack. The notebook likely offers unique insights and approaches to creating medical chatbots using RAG within the John Snow Labs' framework.
+
+These notebooks provide valuable resources for individuals interested in understanding and implementing RAG-based generative AI specifically tailored for medical chatbot applications.
+
+
+</div><div class="h3-box" markdown="1">
+
+#### Introducing The `enableSentenceIncrement` Parameter In `InternalDocumentSplitter`, Offering Precise Sentence Indexing Control For Streamlined Text Processing With Annotators Like `SentenceDetector`
+
+The `enableSentenceIncrement` parameter holds significance as it directly influences the management of sentence indexing within the document splitting process. By regulating this feature, users gain more precise control over sentence index increments in metadata. This functionality is particularly valuable when employing split documents in subsequent annotators, such as the `SentenceDetector`, ensuring accurate and streamlined processing of text data.
+
+*Example*:
+
+```python
+document_splitter = InternalDocumentSplitter() \
+    .setInputCols(["document", "sentence"]) \
+    .setOutputCol("splits") \
+    .setSplitMode("sentence") \
+    .setMaxLength(2)\
+    .setExplodeSplits(True)\
+    .setEnableSentenceIncrement(True)
+
+sample_text = """Sample Name: Aortic Valve Replacement
+Description: Aortic valve replacement using a mechanical valve and two-vessel coronary artery bypass grafting procedure using saphenous vein graft to the first obtuse marginal artery and left radial artery graft to the left anterior descending artery.
+(Medical Transcription Sample Report)
+DIAGNOSIS: Aortic valve stenosis with coronary artery disease associated with congestive heart failure. The patient has diabetes and is morbidly obese.
+PROCEDURES: Aortic valve replacement using a mechanical valve and two-vessel coronary artery bypass grafting procedure using saphenous vein graft to the first obtuse marginal artery and left radial artery graft to the left anterior descending artery.
+"""
+```
+
+*Result*:
+
+|metadata                                 | result       |
+|:----------------------------------------|:-------------|
+| {id -> 0, sentence -> 0, document -> 0} |Description: Aortic valve replacement using a mechanical valve and two-vessel coronary artery byp...|
+| {id -> 0, sentence -> 1, document -> 1} |The patient has diabetes and is morbidly obese.\nANESTHESIA: General endotracheal\nINCISION: Medi...|
+| {id -> 0, sentence -> 2, document -> 2} |INDICATIONS: The patient presented with severe congestive heart failure associated with the patie...|
+| {id -> 0, sentence -> 3, document -> 3} |In addition, The patient had significant coronary artery disease consisting of a chronically occl...|
+| {id -> 0, sentence -> 4, document -> 4} |It was decided to perform a valve replacement as well as coronary artery bypass grafting procedur...|
+| {id -> 0, sentence -> 5, document -> 5} |It is a tricuspid type of valve. The coronary artery consists of a large left anterior descending...|
+
+Please check: [Internal Document Splitter Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/38.InternalDocumentSplitter.ipynb) for more information
+
+
+
+</div><div class="h3-box" markdown="1">
+
+#### Various Core Improvements: Bug Fixes, Enhanced Overall Robustness, And Reliability Of Spark NLP For Healthcare
+
++ Enabling `targetEntities` and `entityWeights` parameters in `ChunkEntityEmbeddings`
++ Refactored `ContextualParserApproach` rules instances for proper integration into LightPipelines
++ Fixed assertion pre-annotations issue in the ALAB module
++ Deprecated Messages Deletion: Removed deprecation warning for confidenceValue metadata in `ContextualParserApproach` and "deprecation_notice" message in `ChunkMapperModel`  
+
+
+</div><div class="h3-box" markdown="1">
+
+#### Updated Notebooks And Demonstrations For making Spark NLP For Healthcare Easier To Navigate And Understand
+
+- New [Medical Chatbot RAG JohnSnowLabs LangChain Notebook](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/generative-ai/Medical_Chatbot_RAG_JohnSnowLabs_LangChain.ipynb)
+- New [Medical Chatbot RAG JohnSnowLabs Haystack Notebook](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/generative-ai/Medical_Chatbot_RAG_JohnSnowLabs_Haystack.ipynb)
+- New [Section Header Splitting and Classification Notebook](https://colab.research.google.com/github/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/18.1.Section_Header_Splitting_and_Classification.ipynb)
+- Updated [Clinical Entity Resolvers Notebook](https://github.com/JohnSnowLabs/spark-nlp-workshop/blob/master/tutorials/Certification_Trainings/Healthcare/3.Clinical_Entity_Resolvers.ipynb) with new examples
+- New [Section Header Splitter Demo](https://demo.johnsnowlabs.com/healthcare/SECTION_HEADER/)
+- Updated [Social Determinant Classification Demo](https://demo.johnsnowlabs.com/healthcare/SOCIAL_DETERMINANT_SEQUENCE_CLASSIFICATION/)
+
+
+</div><div class="h3-box" markdown="1">
+
+#### We Have Added And Updated A Substantial Number Of New Clinical Models And Pipelines, Further Solidifying Our Offering In The Healthcare Domain.
+
++ `bert_sequence_classifier_sdoh_violence_abuse`
++ `bert_sequence_classifier_sdoh_mental_health`
++ `bert_sequence_classifier_sdoh_frailty_vulnerability`
++ `bert_sequence_classifier_clinical_sections`
++ `bert_sequence_classifier_clinical_sections_headless`
+
+
+</div><div class="h3-box" markdown="1">
+
+For all Spark NLP for Healthcare models, please check: [Models Hub Page](https://nlp.johnsnowlabs.com/models?edition=Healthcare+NLP)
+
+
+</div><div class="h3-box" markdown="1">
+
+
+
+## Versions
+
+</div>
+{%- include docs-healthcare-pagination.html -%}