From f975cccf970046601300dafa18c0615ed7ac1777 Mon Sep 17 00:00:00 2001 From: Shengsheng Huang Date: Mon, 25 Mar 2024 23:53:38 +0800 Subject: [PATCH 01/13] migrate bigdl-llm to ipex-llm --- .../llms/{bigdl.ipynb => ipex_llm.ipynb} | 15 +++++------ .../llms/{bigdl.py => ipex_llm.py} | 26 +++++++++---------- .../llms/{test_bigdl.py => test_ipex_llm.py} | 12 ++++----- 3 files changed, 26 insertions(+), 27 deletions(-) rename docs/docs/integrations/llms/{bigdl.ipynb => ipex_llm.ipynb} (89%) rename libs/community/langchain_community/llms/{bigdl.py => ipex_llm.py} (91%) rename libs/community/tests/integration_tests/llms/{test_bigdl.py => test_ipex_llm.py} (71%) diff --git a/docs/docs/integrations/llms/bigdl.ipynb b/docs/docs/integrations/llms/ipex_llm.ipynb similarity index 89% rename from docs/docs/integrations/llms/bigdl.ipynb rename to docs/docs/integrations/llms/ipex_llm.ipynb index 60684898d7ee8..4c70ed1d824e4 100644 --- a/docs/docs/integrations/llms/bigdl.ipynb +++ b/docs/docs/integrations/llms/ipex_llm.ipynb @@ -4,11 +4,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# BigDL-LLM\n", + "# IPEX-LLM\n", "\n", - "> [BigDL-LLM](https://github.com/intel-analytics/BigDL/) is a low-bit LLM optimization library on Intel XPU (Xeon/Core/Flex/Arc/Max). It can make LLMs run extremely fast and consume much less memory on Intel platforms. It is open sourced under Apache 2.0 License.\n", + "> [IPEX-LLM](https://github.com/intel-analytics/ipex-llm/) is a low-bit LLM optimization library on Intel XPU (Xeon/Core/Flex/Arc/Max). It can make LLMs run extremely fast and consume much less memory on Intel platforms. It is open sourced under Apache 2.0 License.\n", "\n", - "This example goes over how to use LangChain to interact with BigDL-LLM for text generation. \n" + "This example goes over how to use LangChain to interact with IPEX-LLM for text generation. \n" ] }, { @@ -33,7 +33,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Install BigDL-LLM for running LLMs locally on Intel CPU." + "Install IEPX-LLM for running LLMs locally on Intel CPU." ] }, { @@ -42,8 +42,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Install BigDL\n", - "%pip install --pre --upgrade bigdl-llm[all]" + "%pip install --pre --upgrade ipex-llm[all]" ] }, { @@ -60,7 +59,7 @@ "outputs": [], "source": [ "from langchain.chains import LLMChain\n", - "from langchain_community.llms.bigdl import BigdlLLM\n", + "from langchain_community.llms.ipex_llm import IPEXLLM\n", "from langchain_core.prompts import PromptTemplate" ] }, @@ -110,7 +109,7 @@ } ], "source": [ - "llm = BigdlLLM.from_model_id(\n", + "llm = IPEXLLM.from_model_id(\n", " model_id=\"lmsys/vicuna-7b-v1.5\",\n", " model_kwargs={\"temperature\": 0, \"max_length\": 64, \"trust_remote_code\": True},\n", ")" diff --git a/libs/community/langchain_community/llms/bigdl.py b/libs/community/langchain_community/llms/ipex_llm.py similarity index 91% rename from libs/community/langchain_community/llms/bigdl.py rename to libs/community/langchain_community/llms/ipex_llm.py index b786bf546ed0d..c477d58776649 100644 --- a/libs/community/langchain_community/llms/bigdl.py +++ b/libs/community/langchain_community/llms/ipex_llm.py @@ -10,14 +10,14 @@ logger = logging.getLogger(__name__) -class BigdlLLM(LLM): - """Wrapper around the BigDL-LLM Transformer-INT4 model +class IPEXLLM(LLM): + """Wrapper around the IPEX-LLM model Example: .. code-block:: python - from langchain.llms import TransformersLLM - llm = TransformersLLM.from_model_id(model_id="THUDM/chatglm-6b") + from langchain.llms import IPEXLLM + llm = IPEXLLM.from_model_id(model_id="THUDM/chatglm-6b") """ model_id: str = DEFAULT_MODEL_ID @@ -25,7 +25,7 @@ class BigdlLLM(LLM): model_kwargs: Optional[dict] = None """Keyword arguments passed to the model.""" model: Any #: :meta private: - """BigDL-LLM Transformers-INT4 model.""" + """IPEXLLM model.""" tokenizer: Any #: :meta private: """Huggingface tokenizer model.""" streaming: bool = True @@ -56,7 +56,7 @@ def from_model_id( An object of TransformersLLM. """ try: - from bigdl.llm.transformers import ( + from ipex_llm.transformers import ( AutoModel, AutoModelForCausalLM, ) @@ -64,8 +64,8 @@ def from_model_id( except ImportError: raise ValueError( - "Could not import bigdl-llm or transformers. " - "Please install it with `pip install --pre --upgrade bigdl-llm[all]`." + "Could not import ipex_llm or transformers. " + "Please install it with `pip install --pre --upgrade ipex_llm[all]`." ) _model_kwargs = model_kwargs or {} @@ -109,7 +109,7 @@ def from_model_id_low_bit( Args: - model_id: Path for the bigdl transformers low-bit model checkpoint folder. + model_id: Path for the ipex-llm transformers low-bit model checkpoint folder. model_kwargs: Keyword arguments to pass to the model and tokenizer. kwargs: Extra arguments to pass to the model and tokenizer. @@ -117,7 +117,7 @@ def from_model_id_low_bit( An object of TransformersLLM. """ try: - from bigdl.llm.transformers import ( + from ipex_llm.transformers import ( AutoModel, AutoModelForCausalLM, ) @@ -125,8 +125,8 @@ def from_model_id_low_bit( except ImportError: raise ValueError( - "Could not import bigdl-llm or transformers. " - "Please install it with `pip install --pre --upgrade bigdl-llm[all]`" + "Could not import ipex_llm or transformers. " + "Please install it with `pip install --pre --upgrade ipex_llm[all]`" ) _model_kwargs = model_kwargs or {} @@ -163,7 +163,7 @@ def _identifying_params(self) -> Mapping[str, Any]: @property def _llm_type(self) -> str: - return "BigDL-llm" + return "ipex-llm" def _call( self, diff --git a/libs/community/tests/integration_tests/llms/test_bigdl.py b/libs/community/tests/integration_tests/llms/test_ipex_llm.py similarity index 71% rename from libs/community/tests/integration_tests/llms/test_bigdl.py rename to libs/community/tests/integration_tests/llms/test_ipex_llm.py index 905a373c48302..9426b8c05f7dd 100644 --- a/libs/community/tests/integration_tests/llms/test_bigdl.py +++ b/libs/community/tests/integration_tests/llms/test_ipex_llm.py @@ -1,12 +1,12 @@ -"""Test BigDL LLM""" +"""Test IPEX LLM""" from langchain_core.outputs import LLMResult -from langchain_community.llms.bigdl import BigdlLLM +from langchain_community.llms.ipex_llm import IPEXLLM def test_call() -> None: - """Test valid call to baichuan.""" - llm = BigdlLLM.from_model_id( + """Test valid call to ipex-llm.""" + llm = IPEXLLM.from_model_id( model_id="lmsys/vicuna-7b-v1.5", model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, ) @@ -15,8 +15,8 @@ def test_call() -> None: def test_generate() -> None: - """Test valid call to baichuan.""" - llm = BigdlLLM.from_model_id( + """Test valid call to ipex-llm.""" + llm = IPEXLLM.from_model_id( model_id="lmsys/vicuna-7b-v1.5", model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, ) From f9773f39234cb4e7329a9158890a20ae729f027a Mon Sep 17 00:00:00 2001 From: Shengsheng Huang Date: Tue, 26 Mar 2024 06:23:07 +0800 Subject: [PATCH 02/13] fix format --- libs/community/langchain_community/llms/ipex_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/community/langchain_community/llms/ipex_llm.py b/libs/community/langchain_community/llms/ipex_llm.py index c477d58776649..0a138efda53d2 100644 --- a/libs/community/langchain_community/llms/ipex_llm.py +++ b/libs/community/langchain_community/llms/ipex_llm.py @@ -109,7 +109,7 @@ def from_model_id_low_bit( Args: - model_id: Path for the ipex-llm transformers low-bit model checkpoint folder. + model_id: Path for the ipex-llm transformers low-bit model folder. model_kwargs: Keyword arguments to pass to the model and tokenizer. kwargs: Extra arguments to pass to the model and tokenizer. From 98fe5a97c63400e59a17057c7a003e73f0dda50b Mon Sep 17 00:00:00 2001 From: Shengsheng Huang Date: Tue, 26 Mar 2024 06:52:47 +0800 Subject: [PATCH 03/13] update example results --- docs/docs/integrations/llms/ipex_llm.ipynb | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/docs/docs/integrations/llms/ipex_llm.ipynb b/docs/docs/integrations/llms/ipex_llm.ipynb index 4c70ed1d824e4..dfea4f5af1791 100644 --- a/docs/docs/integrations/llms/ipex_llm.ipynb +++ b/docs/docs/integrations/llms/ipex_llm.ipynb @@ -88,7 +88,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "69e018750ffb4de1af22ce49cd6957f4", + "model_id": "fc4d9eddb1d04f1897e2d09919ceacc1", "version_major": 2, "version_minor": 0 }, @@ -103,8 +103,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-02-23 18:10:22,896 - INFO - Converting the current model to sym_int4 format......\n", - "2024-02-23 18:10:25,415 - INFO - BIGDL_OPT_IPEX: False\n" + "2024-03-26 06:50:44,210 - INFO - Converting the current model to sym_int4 format......\n" ] } ], @@ -124,16 +123,16 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 5, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The function `run` was deprecated in LangChain 0.1.0 and will be removed in 0.2.0. Use invoke instead.\n", - " warn_deprecated(\n", "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/transformers/generation/utils.py:1369: UserWarning: Using `max_length`'s default (4096) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n", + " warnings.warn(\n", + "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/ipex_llm/transformers/models/llama.py:218: UserWarning: Passing `padding_mask` is deprecated and will be removed in v4.37.Please make sure use `attention_mask` instead.`\n", " warnings.warn(\n" ] }, @@ -141,10 +140,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", - "To disable this warning, you can either:\n", - "\t- Avoid using `tokenizers` before the fork if possible\n", - "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "AI stands for \"Artificial Intelligence.\" It refers to the development of computer systems that can perform tasks that typically require human intelligence, such as visual perception, speech recognition, decision-making, and language translation. AI can be achieved through a combination of techniques such as machine learning, natural language processing, computer vision, and robotics. The ultimate goal of AI research is to create machines that can think and learn like humans, and can even exceed human capabilities in certain areas.\n" ] } @@ -155,6 +150,13 @@ "question = \"What is AI?\"\n", "output = llm_chain.run(question)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From 2d1083b90d6c41bd455d69d6da731b63c3082717 Mon Sep 17 00:00:00 2001 From: Shengsheng Huang Date: Tue, 26 Mar 2024 07:02:00 +0800 Subject: [PATCH 04/13] update error message --- libs/community/langchain_community/llms/ipex_llm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libs/community/langchain_community/llms/ipex_llm.py b/libs/community/langchain_community/llms/ipex_llm.py index 0a138efda53d2..ab1205c6859c5 100644 --- a/libs/community/langchain_community/llms/ipex_llm.py +++ b/libs/community/langchain_community/llms/ipex_llm.py @@ -64,8 +64,8 @@ def from_model_id( except ImportError: raise ValueError( - "Could not import ipex_llm or transformers. " - "Please install it with `pip install --pre --upgrade ipex_llm[all]`." + "Could not import ipex-llm or transformers. " + "Please install it with `pip install --pre --upgrade ipex-llm[all]`." ) _model_kwargs = model_kwargs or {} @@ -125,8 +125,8 @@ def from_model_id_low_bit( except ImportError: raise ValueError( - "Could not import ipex_llm or transformers. " - "Please install it with `pip install --pre --upgrade ipex_llm[all]`" + "Could not import ipex-llm or transformers. " + "Please install it with `pip install --pre --upgrade ipex-llm[all]`." ) _model_kwargs = model_kwargs or {} From 24eb36ddf7f1e6946d0b1fc0fc59f36751fb5b9e Mon Sep 17 00:00:00 2001 From: Shengsheng Huang Date: Tue, 26 Mar 2024 23:25:08 +0800 Subject: [PATCH 05/13] address review comments --- docs/docs/integrations/llms/ipex_llm.ipynb | 4 +- docs/vercel.json | 4 + .../langchain_community/llms/__init__.py | 7 + .../langchain_community/llms/bigdl_llm.py | 132 ++++++++++++++++++ .../langchain_community/llms/ipex_llm.py | 14 +- .../integration_tests/llms/test_ipex_llm.py | 6 +- 6 files changed, 155 insertions(+), 12 deletions(-) create mode 100644 libs/community/langchain_community/llms/bigdl_llm.py diff --git a/docs/docs/integrations/llms/ipex_llm.ipynb b/docs/docs/integrations/llms/ipex_llm.ipynb index dfea4f5af1791..4688f1b24a2ff 100644 --- a/docs/docs/integrations/llms/ipex_llm.ipynb +++ b/docs/docs/integrations/llms/ipex_llm.ipynb @@ -59,7 +59,7 @@ "outputs": [], "source": [ "from langchain.chains import LLMChain\n", - "from langchain_community.llms.ipex_llm import IPEXLLM\n", + "from langchain_community.llms.ipex_llm import IpexLLM\n", "from langchain_core.prompts import PromptTemplate" ] }, @@ -108,7 +108,7 @@ } ], "source": [ - "llm = IPEXLLM.from_model_id(\n", + "llm = IpexLLM.from_model_id(\n", " model_id=\"lmsys/vicuna-7b-v1.5\",\n", " model_kwargs={\"temperature\": 0, \"max_length\": 64, \"trust_remote_code\": True},\n", ")" diff --git a/docs/vercel.json b/docs/vercel.json index 38de5f571bc4a..27d5dbbb192c4 100644 --- a/docs/vercel.json +++ b/docs/vercel.json @@ -88,6 +88,10 @@ "source": "/docs/integrations/llms/huggingface_hub", "destination": "/docs/integrations/llms/huggingface_endpoint" }, + { + "source": "/docs/integrations/llms/ipex_llm", + "destination": "/docs/integrations/llms/ipex_llm" + }, { "source": "/docs/integrations/llms/watsonxllm", "destination": "/docs/integrations/llms/ibm_watsonx" diff --git a/libs/community/langchain_community/llms/__init__.py b/libs/community/langchain_community/llms/__init__.py index 818231580900f..f070658a9b01d 100644 --- a/libs/community/langchain_community/llms/__init__.py +++ b/libs/community/langchain_community/llms/__init__.py @@ -277,6 +277,10 @@ def _import_human() -> Type[BaseLLM]: return HumanInputLLM +def _import_ipex_llm() -> Type[BaseLLM]: + from langchain_community.llms.ipex_llm import IpexLLM + + return IpexLLM def _import_javelin_ai_gateway() -> Type[BaseLLM]: from langchain_community.llms.javelin_ai_gateway import JavelinAIGateway @@ -695,6 +699,8 @@ def __getattr__(name: str) -> Any: return _import_huggingface_text_gen_inference() elif name == "HumanInputLLM": return _import_human() + elif name == "IpexLLM": + return _import_ipex_llm() elif name == "JavelinAIGateway": return _import_javelin_ai_gateway() elif name == "KoboldApiLLM": @@ -851,6 +857,7 @@ def __getattr__(name: str) -> Any: "HuggingFacePipeline", "HuggingFaceTextGenInference", "HumanInputLLM", + "IpexLLM", "JavelinAIGateway", "KoboldApiLLM", "Konko", diff --git a/libs/community/langchain_community/llms/bigdl_llm.py b/libs/community/langchain_community/llms/bigdl_llm.py new file mode 100644 index 0000000000000..8feeefe109fde --- /dev/null +++ b/libs/community/langchain_community/llms/bigdl_llm.py @@ -0,0 +1,132 @@ +import logging +from typing import Any, List, Mapping, Optional + +from langchain_community.llms.ipex_llm import IpexLLM +from langchain_core.language_models.llms import LLM +logger = logging.getLogger(__name__) + +class BigdlLLM(IpexLLM): + """Wrapper around the BigdlLLM model + + Example: + .. code-block:: python + + from langchain_community.llms import BigdlLLM + llm = BigdlLLM.from_model_id(model_id="THUDM/chatglm-6b") + """ + @classmethod + def from_model_id( + cls, + model_id: str, + model_kwargs: Optional[dict] = None, + **kwargs: Any, + ) -> LLM: + """ + Construct object from model_id + + Args: + model_id: Path for the huggingface repo id to be downloaded or + the huggingface checkpoint folder. + model_kwargs: Keyword arguments to pass to the model and tokenizer. + kwargs: Extra arguments to pass to the model and tokenizer. + + Returns: + An object of BigdlLLM. + """ + try: + from bigdl.llm.transformers import ( + AutoModel, + AutoModelForCausalLM, + ) + from transformers import AutoTokenizer, LlamaTokenizer + + except ImportError: + raise ValueError( + "Could not import bigdl-llm or transformers. " + "Please install it with `pip install --pre --upgrade bigdl-llm[all]`." + ) + + _model_kwargs = model_kwargs or {} + + try: + tokenizer = AutoTokenizer.from_pretrained(model_id, **_model_kwargs) + except Exception: + tokenizer = LlamaTokenizer.from_pretrained(model_id, **_model_kwargs) + + try: + model = AutoModelForCausalLM.from_pretrained( + model_id, load_in_4bit=True, **_model_kwargs + ) + except Exception: + model = AutoModel.from_pretrained( + model_id, load_in_4bit=True, **_model_kwargs + ) + + if "trust_remote_code" in _model_kwargs: + _model_kwargs = { + k: v for k, v in _model_kwargs.items() if k != "trust_remote_code" + } + + return cls( + model_id=model_id, + model=model, + tokenizer=tokenizer, + model_kwargs=_model_kwargs, + **kwargs, + ) + + @classmethod + def from_model_id_low_bit( + cls, + model_id: str, + model_kwargs: Optional[dict] = None, + **kwargs: Any, + ) -> LLM: + """ + Construct low_bit object from model_id + + Args: + + model_id: Path for the bigdl-llm transformers low-bit model folder. + model_kwargs: Keyword arguments to pass to the model and tokenizer. + kwargs: Extra arguments to pass to the model and tokenizer. + + Returns: + An object of BigdlLLM. + """ + try: + from bigdl.llm.transformers import ( + AutoModel, + AutoModelForCausalLM, + ) + from transformers import AutoTokenizer, LlamaTokenizer + + except ImportError: + raise ValueError( + "Could not import bigdl-llm or transformers. " + "Please install it with `pip install --pre --upgrade bigdl-llm[all]`." + ) + + _model_kwargs = model_kwargs or {} + try: + tokenizer = AutoTokenizer.from_pretrained(model_id, **_model_kwargs) + except Exception: + tokenizer = LlamaTokenizer.from_pretrained(model_id, **_model_kwargs) + + try: + model = AutoModelForCausalLM.load_low_bit(model_id, **_model_kwargs) + except Exception: + model = AutoModel.load_low_bit(model_id, **_model_kwargs) + + if "trust_remote_code" in _model_kwargs: + _model_kwargs = { + k: v for k, v in _model_kwargs.items() if k != "trust_remote_code" + } + + return cls( + model_id=model_id, + model=model, + tokenizer=tokenizer, + model_kwargs=_model_kwargs, + **kwargs, + ) \ No newline at end of file diff --git a/libs/community/langchain_community/llms/ipex_llm.py b/libs/community/langchain_community/llms/ipex_llm.py index ab1205c6859c5..460300cd02b65 100644 --- a/libs/community/langchain_community/llms/ipex_llm.py +++ b/libs/community/langchain_community/llms/ipex_llm.py @@ -10,14 +10,14 @@ logger = logging.getLogger(__name__) -class IPEXLLM(LLM): - """Wrapper around the IPEX-LLM model +class IpexLLM(LLM): + """Wrapper around the IpexLLM model Example: .. code-block:: python - from langchain.llms import IPEXLLM - llm = IPEXLLM.from_model_id(model_id="THUDM/chatglm-6b") + from langchain_community.llms import IpexLLM + llm = IpexLLM.from_model_id(model_id="THUDM/chatglm-6b") """ model_id: str = DEFAULT_MODEL_ID @@ -25,7 +25,7 @@ class IPEXLLM(LLM): model_kwargs: Optional[dict] = None """Keyword arguments passed to the model.""" model: Any #: :meta private: - """IPEXLLM model.""" + """IpexLLM model.""" tokenizer: Any #: :meta private: """Huggingface tokenizer model.""" streaming: bool = True @@ -53,7 +53,7 @@ def from_model_id( kwargs: Extra arguments to pass to the model and tokenizer. Returns: - An object of TransformersLLM. + An object of IpexLLM. """ try: from ipex_llm.transformers import ( @@ -114,7 +114,7 @@ def from_model_id_low_bit( kwargs: Extra arguments to pass to the model and tokenizer. Returns: - An object of TransformersLLM. + An object of IpexLLM. """ try: from ipex_llm.transformers import ( diff --git a/libs/community/tests/integration_tests/llms/test_ipex_llm.py b/libs/community/tests/integration_tests/llms/test_ipex_llm.py index 9426b8c05f7dd..a56a5e83653ad 100644 --- a/libs/community/tests/integration_tests/llms/test_ipex_llm.py +++ b/libs/community/tests/integration_tests/llms/test_ipex_llm.py @@ -1,12 +1,12 @@ """Test IPEX LLM""" from langchain_core.outputs import LLMResult -from langchain_community.llms.ipex_llm import IPEXLLM +from langchain_community.llms.ipex_llm import IpexLLM def test_call() -> None: """Test valid call to ipex-llm.""" - llm = IPEXLLM.from_model_id( + llm = IpexLLM.from_model_id( model_id="lmsys/vicuna-7b-v1.5", model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, ) @@ -16,7 +16,7 @@ def test_call() -> None: def test_generate() -> None: """Test valid call to ipex-llm.""" - llm = IPEXLLM.from_model_id( + llm = IpexLLM.from_model_id( model_id="lmsys/vicuna-7b-v1.5", model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, ) From bf088bad333cd1081eb307af16fd3d53d2566e80 Mon Sep 17 00:00:00 2001 From: Shengsheng Huang Date: Tue, 26 Mar 2024 23:25:37 +0800 Subject: [PATCH 06/13] add bigdl ut as well --- .../integration_tests/llms/test_bigdl_llm.py | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 libs/community/tests/integration_tests/llms/test_bigdl_llm.py diff --git a/libs/community/tests/integration_tests/llms/test_bigdl_llm.py b/libs/community/tests/integration_tests/llms/test_bigdl_llm.py new file mode 100644 index 0000000000000..d2c109f0c9b74 --- /dev/null +++ b/libs/community/tests/integration_tests/llms/test_bigdl_llm.py @@ -0,0 +1,25 @@ +"""Test IPEX LLM""" +from langchain_core.outputs import LLMResult + +from langchain_community.llms.bigdl_llm import BigdlLLM + + +def test_call() -> None: + """Test valid call to ipex-llm.""" + llm = BigdlLLM.from_model_id( + model_id="lmsys/vicuna-7b-v1.5", + model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, + ) + output = llm("Hello!") + assert isinstance(output, str) + + +def test_generate() -> None: + """Test valid call to ipex-llm.""" + llm = BigdlLLM.from_model_id( + model_id="lmsys/vicuna-7b-v1.5", + model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, + ) + output = llm.generate(["Hello!"]) + assert isinstance(output, LLMResult) + assert isinstance(output.generations, list) From 87070caea37771c0c7a0c863f66f4a3689943e55 Mon Sep 17 00:00:00 2001 From: Shengsheng Huang Date: Wed, 27 Mar 2024 00:59:35 +0800 Subject: [PATCH 07/13] update import to use langchain_community.llms --- docs/docs/integrations/llms/ipex_llm.ipynb | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/docs/docs/integrations/llms/ipex_llm.ipynb b/docs/docs/integrations/llms/ipex_llm.ipynb index 4688f1b24a2ff..25519b2d92721 100644 --- a/docs/docs/integrations/llms/ipex_llm.ipynb +++ b/docs/docs/integrations/llms/ipex_llm.ipynb @@ -59,7 +59,7 @@ "outputs": [], "source": [ "from langchain.chains import LLMChain\n", - "from langchain_community.llms.ipex_llm import IpexLLM\n", + "from langchain_community.llms import IpexLLM\n", "from langchain_core.prompts import PromptTemplate" ] }, @@ -88,7 +88,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "fc4d9eddb1d04f1897e2d09919ceacc1", + "model_id": "27c08180714a44c7ab766624d5054163", "version_major": 2, "version_minor": 0 }, @@ -103,7 +103,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-03-26 06:50:44,210 - INFO - Converting the current model to sym_int4 format......\n" + "2024-03-27 00:58:43,670 - INFO - Converting the current model to sym_int4 format......\n" ] } ], @@ -123,16 +123,20 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ + "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The function `run` was deprecated in LangChain 0.1.0 and will be removed in 0.2.0. Use invoke instead.\n", + " warn_deprecated(\n", "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/transformers/generation/utils.py:1369: UserWarning: Using `max_length`'s default (4096) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n", " warnings.warn(\n", "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/ipex_llm/transformers/models/llama.py:218: UserWarning: Passing `padding_mask` is deprecated and will be removed in v4.37.Please make sure use `attention_mask` instead.`\n", + " warnings.warn(\n", + "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/ipex_llm/transformers/models/llama.py:218: UserWarning: Passing `padding_mask` is deprecated and will be removed in v4.37.Please make sure use `attention_mask` instead.`\n", " warnings.warn(\n" ] }, @@ -140,6 +144,10 @@ "name": "stdout", "output_type": "stream", "text": [ + "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n", + "To disable this warning, you can either:\n", + "\t- Avoid using `tokenizers` before the fork if possible\n", + "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n", "AI stands for \"Artificial Intelligence.\" It refers to the development of computer systems that can perform tasks that typically require human intelligence, such as visual perception, speech recognition, decision-making, and language translation. AI can be achieved through a combination of techniques such as machine learning, natural language processing, computer vision, and robotics. The ultimate goal of AI research is to create machines that can think and learn like humans, and can even exceed human capabilities in certain areas.\n" ] } From a82b0cb3adb4b937111935c403358a3684aa9e2f Mon Sep 17 00:00:00 2001 From: Shengsheng Huang Date: Wed, 27 Mar 2024 01:00:23 +0800 Subject: [PATCH 08/13] add deprecated warning, add BigdlLLM import --- libs/community/langchain_community/llms/__init__.py | 6 ++++++ libs/community/langchain_community/llms/bigdl_llm.py | 5 +++++ 2 files changed, 11 insertions(+) diff --git a/libs/community/langchain_community/llms/__init__.py b/libs/community/langchain_community/llms/__init__.py index f070658a9b01d..b87e9239b64e8 100644 --- a/libs/community/langchain_community/llms/__init__.py +++ b/libs/community/langchain_community/llms/__init__.py @@ -113,6 +113,10 @@ def _import_bedrock() -> Type[BaseLLM]: return Bedrock +def _import_bigdlllm() -> Type[BaseLLM]: + from langchain_community.llms.bigdl_llm import BigdlLLM + + return BigdlLLM def _import_bittensor() -> Type[BaseLLM]: from langchain_community.llms.bittensor import NIBittensorLLM @@ -649,6 +653,8 @@ def __getattr__(name: str) -> Any: return _import_beam() elif name == "Bedrock": return _import_bedrock() + elif name == "BigdlLLM": + return _import_bigdlllm() elif name == "NIBittensorLLM": return _import_bittensor() elif name == "CerebriumAI": diff --git a/libs/community/langchain_community/llms/bigdl_llm.py b/libs/community/langchain_community/llms/bigdl_llm.py index 8feeefe109fde..28fd8fcb2b72a 100644 --- a/libs/community/langchain_community/llms/bigdl_llm.py +++ b/libs/community/langchain_community/llms/bigdl_llm.py @@ -33,6 +33,8 @@ def from_model_id( Returns: An object of BigdlLLM. """ + logger.warning("BigdlLLM was deprecated. Please use IpexLLM instead.") + try: from bigdl.llm.transformers import ( AutoModel, @@ -94,6 +96,9 @@ def from_model_id_low_bit( Returns: An object of BigdlLLM. """ + + logger.warning("BigdlLLM was deprecated. Please use IpexLLM instead.") + try: from bigdl.llm.transformers import ( AutoModel, From 35913c826e76791addbdca018a84fbcfb8eee5b6 Mon Sep 17 00:00:00 2001 From: Shengsheng Huang Date: Wed, 27 Mar 2024 01:14:19 +0800 Subject: [PATCH 09/13] update comment --- libs/community/tests/integration_tests/llms/test_bigdl_llm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/community/tests/integration_tests/llms/test_bigdl_llm.py b/libs/community/tests/integration_tests/llms/test_bigdl_llm.py index d2c109f0c9b74..8fcbc5c81c935 100644 --- a/libs/community/tests/integration_tests/llms/test_bigdl_llm.py +++ b/libs/community/tests/integration_tests/llms/test_bigdl_llm.py @@ -5,7 +5,7 @@ def test_call() -> None: - """Test valid call to ipex-llm.""" + """Test valid call to bigdl-llm.""" llm = BigdlLLM.from_model_id( model_id="lmsys/vicuna-7b-v1.5", model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, @@ -15,7 +15,7 @@ def test_call() -> None: def test_generate() -> None: - """Test valid call to ipex-llm.""" + """Test valid call to bigdl-llm.""" llm = BigdlLLM.from_model_id( model_id="lmsys/vicuna-7b-v1.5", model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, From 564015b355872ca54055e1e26866ea236e566dc8 Mon Sep 17 00:00:00 2001 From: Shengsheng Huang Date: Wed, 27 Mar 2024 01:15:44 +0800 Subject: [PATCH 10/13] update comment --- libs/community/tests/integration_tests/llms/test_bigdl_llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/community/tests/integration_tests/llms/test_bigdl_llm.py b/libs/community/tests/integration_tests/llms/test_bigdl_llm.py index 8fcbc5c81c935..1967ca07a8477 100644 --- a/libs/community/tests/integration_tests/llms/test_bigdl_llm.py +++ b/libs/community/tests/integration_tests/llms/test_bigdl_llm.py @@ -1,4 +1,4 @@ -"""Test IPEX LLM""" +"""Test BigdlLLM""" from langchain_core.outputs import LLMResult from langchain_community.llms.bigdl_llm import BigdlLLM From 75ed94082524c3dd6c222563e16b8aac8550d508 Mon Sep 17 00:00:00 2001 From: Shengsheng Huang Date: Wed, 27 Mar 2024 01:18:15 +0800 Subject: [PATCH 11/13] add bigdl property --- libs/community/langchain_community/llms/bigdl_llm.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libs/community/langchain_community/llms/bigdl_llm.py b/libs/community/langchain_community/llms/bigdl_llm.py index 28fd8fcb2b72a..a529b2e8c40a2 100644 --- a/libs/community/langchain_community/llms/bigdl_llm.py +++ b/libs/community/langchain_community/llms/bigdl_llm.py @@ -134,4 +134,8 @@ def from_model_id_low_bit( tokenizer=tokenizer, model_kwargs=_model_kwargs, **kwargs, - ) \ No newline at end of file + ) + + @property + def _llm_type(self) -> str: + return "bigdl-llm" \ No newline at end of file From aa9bed1fcd679764739fa7edbf9ec1573aea6045 Mon Sep 17 00:00:00 2001 From: Shengsheng Huang Date: Wed, 27 Mar 2024 01:31:47 +0800 Subject: [PATCH 12/13] fix linting and test --- libs/community/langchain_community/llms/bigdl_llm.py | 5 +++-- libs/community/tests/unit_tests/llms/test_imports.py | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/community/langchain_community/llms/bigdl_llm.py b/libs/community/langchain_community/llms/bigdl_llm.py index a529b2e8c40a2..18decd814fcbb 100644 --- a/libs/community/langchain_community/llms/bigdl_llm.py +++ b/libs/community/langchain_community/llms/bigdl_llm.py @@ -1,8 +1,9 @@ import logging -from typing import Any, List, Mapping, Optional +from typing import Any, Optional from langchain_community.llms.ipex_llm import IpexLLM from langchain_core.language_models.llms import LLM + logger = logging.getLogger(__name__) class BigdlLLM(IpexLLM): @@ -100,7 +101,7 @@ def from_model_id_low_bit( logger.warning("BigdlLLM was deprecated. Please use IpexLLM instead.") try: - from bigdl.llm.transformers import ( + from bigdl.llm.transformers import ( AutoModel, AutoModelForCausalLM, ) diff --git a/libs/community/tests/unit_tests/llms/test_imports.py b/libs/community/tests/unit_tests/llms/test_imports.py index e7359360517d0..6f719de2bfcff 100644 --- a/libs/community/tests/unit_tests/llms/test_imports.py +++ b/libs/community/tests/unit_tests/llms/test_imports.py @@ -42,6 +42,7 @@ "HuggingFacePipeline", "HuggingFaceTextGenInference", "HumanInputLLM", + "IpexLLM", "KoboldApiLLM", "Konko", "LlamaCpp", From 8aff56ee6a40f4f620e06ef0bb518b64fcbc7469 Mon Sep 17 00:00:00 2001 From: Shengsheng Huang Date: Wed, 27 Mar 2024 01:41:26 +0800 Subject: [PATCH 13/13] update import sorting --- libs/community/langchain_community/llms/bigdl_llm.py | 3 ++- libs/community/langchain_community/llms/ipex_llm.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/libs/community/langchain_community/llms/bigdl_llm.py b/libs/community/langchain_community/llms/bigdl_llm.py index 18decd814fcbb..af0e156977a6f 100644 --- a/libs/community/langchain_community/llms/bigdl_llm.py +++ b/libs/community/langchain_community/llms/bigdl_llm.py @@ -1,9 +1,10 @@ -import logging from typing import Any, Optional from langchain_community.llms.ipex_llm import IpexLLM from langchain_core.language_models.llms import LLM +import logging + logger = logging.getLogger(__name__) class BigdlLLM(IpexLLM): diff --git a/libs/community/langchain_community/llms/ipex_llm.py b/libs/community/langchain_community/llms/ipex_llm.py index 460300cd02b65..993a1764e57e0 100644 --- a/libs/community/langchain_community/llms/ipex_llm.py +++ b/libs/community/langchain_community/llms/ipex_llm.py @@ -1,4 +1,3 @@ -import logging from typing import Any, List, Mapping, Optional from langchain_core.callbacks import CallbackManagerForLLMRun @@ -7,6 +6,8 @@ DEFAULT_MODEL_ID = "gpt2" +import logging + logger = logging.getLogger(__name__)