diff --git a/docs/docs/integrations/llms/bigdl.ipynb b/docs/docs/integrations/llms/ipex_llm.ipynb similarity index 77% rename from docs/docs/integrations/llms/bigdl.ipynb rename to docs/docs/integrations/llms/ipex_llm.ipynb index 60684898d7ee8..25519b2d92721 100644 --- a/docs/docs/integrations/llms/bigdl.ipynb +++ b/docs/docs/integrations/llms/ipex_llm.ipynb @@ -4,11 +4,11 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# BigDL-LLM\n", + "# IPEX-LLM\n", "\n", - "> [BigDL-LLM](https://github.com/intel-analytics/BigDL/) is a low-bit LLM optimization library on Intel XPU (Xeon/Core/Flex/Arc/Max). It can make LLMs run extremely fast and consume much less memory on Intel platforms. It is open sourced under Apache 2.0 License.\n", + "> [IPEX-LLM](https://github.com/intel-analytics/ipex-llm/) is a low-bit LLM optimization library on Intel XPU (Xeon/Core/Flex/Arc/Max). It can make LLMs run extremely fast and consume much less memory on Intel platforms. It is open sourced under Apache 2.0 License.\n", "\n", - "This example goes over how to use LangChain to interact with BigDL-LLM for text generation. \n" + "This example goes over how to use LangChain to interact with IPEX-LLM for text generation. \n" ] }, { @@ -33,7 +33,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Install BigDL-LLM for running LLMs locally on Intel CPU." + "Install IEPX-LLM for running LLMs locally on Intel CPU." ] }, { @@ -42,8 +42,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Install BigDL\n", - "%pip install --pre --upgrade bigdl-llm[all]" + "%pip install --pre --upgrade ipex-llm[all]" ] }, { @@ -60,7 +59,7 @@ "outputs": [], "source": [ "from langchain.chains import LLMChain\n", - "from langchain_community.llms.bigdl import BigdlLLM\n", + "from langchain_community.llms import IpexLLM\n", "from langchain_core.prompts import PromptTemplate" ] }, @@ -89,7 +88,7 @@ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "69e018750ffb4de1af22ce49cd6957f4", + "model_id": "27c08180714a44c7ab766624d5054163", "version_major": 2, "version_minor": 0 }, @@ -104,13 +103,12 @@ "name": "stderr", "output_type": "stream", "text": [ - "2024-02-23 18:10:22,896 - INFO - Converting the current model to sym_int4 format......\n", - "2024-02-23 18:10:25,415 - INFO - BIGDL_OPT_IPEX: False\n" + "2024-03-27 00:58:43,670 - INFO - Converting the current model to sym_int4 format......\n" ] } ], "source": [ - "llm = BigdlLLM.from_model_id(\n", + "llm = IpexLLM.from_model_id(\n", " model_id=\"lmsys/vicuna-7b-v1.5\",\n", " model_kwargs={\"temperature\": 0, \"max_length\": 64, \"trust_remote_code\": True},\n", ")" @@ -135,6 +133,10 @@ "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/langchain_core/_api/deprecation.py:117: LangChainDeprecationWarning: The function `run` was deprecated in LangChain 0.1.0 and will be removed in 0.2.0. Use invoke instead.\n", " warn_deprecated(\n", "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/transformers/generation/utils.py:1369: UserWarning: Using `max_length`'s default (4096) to control the generation length. This behaviour is deprecated and will be removed from the config in v5 of Transformers -- we recommend using `max_new_tokens` to control the maximum length of the generation.\n", + " warnings.warn(\n", + "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/ipex_llm/transformers/models/llama.py:218: UserWarning: Passing `padding_mask` is deprecated and will be removed in v4.37.Please make sure use `attention_mask` instead.`\n", + " warnings.warn(\n", + "/opt/anaconda3/envs/shane-langchain2/lib/python3.9/site-packages/ipex_llm/transformers/models/llama.py:218: UserWarning: Passing `padding_mask` is deprecated and will be removed in v4.37.Please make sure use `attention_mask` instead.`\n", " warnings.warn(\n" ] }, @@ -156,6 +158,13 @@ "question = \"What is AI?\"\n", "output = llm_chain.run(question)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/docs/vercel.json b/docs/vercel.json index 38de5f571bc4a..27d5dbbb192c4 100644 --- a/docs/vercel.json +++ b/docs/vercel.json @@ -88,6 +88,10 @@ "source": "/docs/integrations/llms/huggingface_hub", "destination": "/docs/integrations/llms/huggingface_endpoint" }, + { + "source": "/docs/integrations/llms/ipex_llm", + "destination": "/docs/integrations/llms/ipex_llm" + }, { "source": "/docs/integrations/llms/watsonxllm", "destination": "/docs/integrations/llms/ibm_watsonx" diff --git a/libs/community/langchain_community/llms/__init__.py b/libs/community/langchain_community/llms/__init__.py index 818231580900f..b87e9239b64e8 100644 --- a/libs/community/langchain_community/llms/__init__.py +++ b/libs/community/langchain_community/llms/__init__.py @@ -113,6 +113,10 @@ def _import_bedrock() -> Type[BaseLLM]: return Bedrock +def _import_bigdlllm() -> Type[BaseLLM]: + from langchain_community.llms.bigdl_llm import BigdlLLM + + return BigdlLLM def _import_bittensor() -> Type[BaseLLM]: from langchain_community.llms.bittensor import NIBittensorLLM @@ -277,6 +281,10 @@ def _import_human() -> Type[BaseLLM]: return HumanInputLLM +def _import_ipex_llm() -> Type[BaseLLM]: + from langchain_community.llms.ipex_llm import IpexLLM + + return IpexLLM def _import_javelin_ai_gateway() -> Type[BaseLLM]: from langchain_community.llms.javelin_ai_gateway import JavelinAIGateway @@ -645,6 +653,8 @@ def __getattr__(name: str) -> Any: return _import_beam() elif name == "Bedrock": return _import_bedrock() + elif name == "BigdlLLM": + return _import_bigdlllm() elif name == "NIBittensorLLM": return _import_bittensor() elif name == "CerebriumAI": @@ -695,6 +705,8 @@ def __getattr__(name: str) -> Any: return _import_huggingface_text_gen_inference() elif name == "HumanInputLLM": return _import_human() + elif name == "IpexLLM": + return _import_ipex_llm() elif name == "JavelinAIGateway": return _import_javelin_ai_gateway() elif name == "KoboldApiLLM": @@ -851,6 +863,7 @@ def __getattr__(name: str) -> Any: "HuggingFacePipeline", "HuggingFaceTextGenInference", "HumanInputLLM", + "IpexLLM", "JavelinAIGateway", "KoboldApiLLM", "Konko", diff --git a/libs/community/langchain_community/llms/bigdl_llm.py b/libs/community/langchain_community/llms/bigdl_llm.py new file mode 100644 index 0000000000000..af0e156977a6f --- /dev/null +++ b/libs/community/langchain_community/llms/bigdl_llm.py @@ -0,0 +1,143 @@ +from typing import Any, Optional + +from langchain_community.llms.ipex_llm import IpexLLM +from langchain_core.language_models.llms import LLM + +import logging + +logger = logging.getLogger(__name__) + +class BigdlLLM(IpexLLM): + """Wrapper around the BigdlLLM model + + Example: + .. code-block:: python + + from langchain_community.llms import BigdlLLM + llm = BigdlLLM.from_model_id(model_id="THUDM/chatglm-6b") + """ + @classmethod + def from_model_id( + cls, + model_id: str, + model_kwargs: Optional[dict] = None, + **kwargs: Any, + ) -> LLM: + """ + Construct object from model_id + + Args: + model_id: Path for the huggingface repo id to be downloaded or + the huggingface checkpoint folder. + model_kwargs: Keyword arguments to pass to the model and tokenizer. + kwargs: Extra arguments to pass to the model and tokenizer. + + Returns: + An object of BigdlLLM. + """ + logger.warning("BigdlLLM was deprecated. Please use IpexLLM instead.") + + try: + from bigdl.llm.transformers import ( + AutoModel, + AutoModelForCausalLM, + ) + from transformers import AutoTokenizer, LlamaTokenizer + + except ImportError: + raise ValueError( + "Could not import bigdl-llm or transformers. " + "Please install it with `pip install --pre --upgrade bigdl-llm[all]`." + ) + + _model_kwargs = model_kwargs or {} + + try: + tokenizer = AutoTokenizer.from_pretrained(model_id, **_model_kwargs) + except Exception: + tokenizer = LlamaTokenizer.from_pretrained(model_id, **_model_kwargs) + + try: + model = AutoModelForCausalLM.from_pretrained( + model_id, load_in_4bit=True, **_model_kwargs + ) + except Exception: + model = AutoModel.from_pretrained( + model_id, load_in_4bit=True, **_model_kwargs + ) + + if "trust_remote_code" in _model_kwargs: + _model_kwargs = { + k: v for k, v in _model_kwargs.items() if k != "trust_remote_code" + } + + return cls( + model_id=model_id, + model=model, + tokenizer=tokenizer, + model_kwargs=_model_kwargs, + **kwargs, + ) + + @classmethod + def from_model_id_low_bit( + cls, + model_id: str, + model_kwargs: Optional[dict] = None, + **kwargs: Any, + ) -> LLM: + """ + Construct low_bit object from model_id + + Args: + + model_id: Path for the bigdl-llm transformers low-bit model folder. + model_kwargs: Keyword arguments to pass to the model and tokenizer. + kwargs: Extra arguments to pass to the model and tokenizer. + + Returns: + An object of BigdlLLM. + """ + + logger.warning("BigdlLLM was deprecated. Please use IpexLLM instead.") + + try: + from bigdl.llm.transformers import ( + AutoModel, + AutoModelForCausalLM, + ) + from transformers import AutoTokenizer, LlamaTokenizer + + except ImportError: + raise ValueError( + "Could not import bigdl-llm or transformers. " + "Please install it with `pip install --pre --upgrade bigdl-llm[all]`." + ) + + _model_kwargs = model_kwargs or {} + try: + tokenizer = AutoTokenizer.from_pretrained(model_id, **_model_kwargs) + except Exception: + tokenizer = LlamaTokenizer.from_pretrained(model_id, **_model_kwargs) + + try: + model = AutoModelForCausalLM.load_low_bit(model_id, **_model_kwargs) + except Exception: + model = AutoModel.load_low_bit(model_id, **_model_kwargs) + + if "trust_remote_code" in _model_kwargs: + _model_kwargs = { + k: v for k, v in _model_kwargs.items() if k != "trust_remote_code" + } + + return cls( + model_id=model_id, + model=model, + tokenizer=tokenizer, + model_kwargs=_model_kwargs, + **kwargs, + ) + + @property + def _llm_type(self) -> str: + return "bigdl-llm" \ No newline at end of file diff --git a/libs/community/langchain_community/llms/bigdl.py b/libs/community/langchain_community/llms/ipex_llm.py similarity index 89% rename from libs/community/langchain_community/llms/bigdl.py rename to libs/community/langchain_community/llms/ipex_llm.py index b786bf546ed0d..993a1764e57e0 100644 --- a/libs/community/langchain_community/llms/bigdl.py +++ b/libs/community/langchain_community/llms/ipex_llm.py @@ -1,4 +1,3 @@ -import logging from typing import Any, List, Mapping, Optional from langchain_core.callbacks import CallbackManagerForLLMRun @@ -7,17 +6,19 @@ DEFAULT_MODEL_ID = "gpt2" +import logging + logger = logging.getLogger(__name__) -class BigdlLLM(LLM): - """Wrapper around the BigDL-LLM Transformer-INT4 model +class IpexLLM(LLM): + """Wrapper around the IpexLLM model Example: .. code-block:: python - from langchain.llms import TransformersLLM - llm = TransformersLLM.from_model_id(model_id="THUDM/chatglm-6b") + from langchain_community.llms import IpexLLM + llm = IpexLLM.from_model_id(model_id="THUDM/chatglm-6b") """ model_id: str = DEFAULT_MODEL_ID @@ -25,7 +26,7 @@ class BigdlLLM(LLM): model_kwargs: Optional[dict] = None """Keyword arguments passed to the model.""" model: Any #: :meta private: - """BigDL-LLM Transformers-INT4 model.""" + """IpexLLM model.""" tokenizer: Any #: :meta private: """Huggingface tokenizer model.""" streaming: bool = True @@ -53,10 +54,10 @@ def from_model_id( kwargs: Extra arguments to pass to the model and tokenizer. Returns: - An object of TransformersLLM. + An object of IpexLLM. """ try: - from bigdl.llm.transformers import ( + from ipex_llm.transformers import ( AutoModel, AutoModelForCausalLM, ) @@ -64,8 +65,8 @@ def from_model_id( except ImportError: raise ValueError( - "Could not import bigdl-llm or transformers. " - "Please install it with `pip install --pre --upgrade bigdl-llm[all]`." + "Could not import ipex-llm or transformers. " + "Please install it with `pip install --pre --upgrade ipex-llm[all]`." ) _model_kwargs = model_kwargs or {} @@ -109,15 +110,15 @@ def from_model_id_low_bit( Args: - model_id: Path for the bigdl transformers low-bit model checkpoint folder. + model_id: Path for the ipex-llm transformers low-bit model folder. model_kwargs: Keyword arguments to pass to the model and tokenizer. kwargs: Extra arguments to pass to the model and tokenizer. Returns: - An object of TransformersLLM. + An object of IpexLLM. """ try: - from bigdl.llm.transformers import ( + from ipex_llm.transformers import ( AutoModel, AutoModelForCausalLM, ) @@ -125,8 +126,8 @@ def from_model_id_low_bit( except ImportError: raise ValueError( - "Could not import bigdl-llm or transformers. " - "Please install it with `pip install --pre --upgrade bigdl-llm[all]`" + "Could not import ipex-llm or transformers. " + "Please install it with `pip install --pre --upgrade ipex-llm[all]`." ) _model_kwargs = model_kwargs or {} @@ -163,7 +164,7 @@ def _identifying_params(self) -> Mapping[str, Any]: @property def _llm_type(self) -> str: - return "BigDL-llm" + return "ipex-llm" def _call( self, diff --git a/libs/community/tests/integration_tests/llms/test_bigdl.py b/libs/community/tests/integration_tests/llms/test_bigdl_llm.py similarity index 80% rename from libs/community/tests/integration_tests/llms/test_bigdl.py rename to libs/community/tests/integration_tests/llms/test_bigdl_llm.py index 905a373c48302..1967ca07a8477 100644 --- a/libs/community/tests/integration_tests/llms/test_bigdl.py +++ b/libs/community/tests/integration_tests/llms/test_bigdl_llm.py @@ -1,11 +1,11 @@ -"""Test BigDL LLM""" +"""Test BigdlLLM""" from langchain_core.outputs import LLMResult -from langchain_community.llms.bigdl import BigdlLLM +from langchain_community.llms.bigdl_llm import BigdlLLM def test_call() -> None: - """Test valid call to baichuan.""" + """Test valid call to bigdl-llm.""" llm = BigdlLLM.from_model_id( model_id="lmsys/vicuna-7b-v1.5", model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, @@ -15,7 +15,7 @@ def test_call() -> None: def test_generate() -> None: - """Test valid call to baichuan.""" + """Test valid call to bigdl-llm.""" llm = BigdlLLM.from_model_id( model_id="lmsys/vicuna-7b-v1.5", model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, diff --git a/libs/community/tests/integration_tests/llms/test_ipex_llm.py b/libs/community/tests/integration_tests/llms/test_ipex_llm.py new file mode 100644 index 0000000000000..a56a5e83653ad --- /dev/null +++ b/libs/community/tests/integration_tests/llms/test_ipex_llm.py @@ -0,0 +1,25 @@ +"""Test IPEX LLM""" +from langchain_core.outputs import LLMResult + +from langchain_community.llms.ipex_llm import IpexLLM + + +def test_call() -> None: + """Test valid call to ipex-llm.""" + llm = IpexLLM.from_model_id( + model_id="lmsys/vicuna-7b-v1.5", + model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, + ) + output = llm("Hello!") + assert isinstance(output, str) + + +def test_generate() -> None: + """Test valid call to ipex-llm.""" + llm = IpexLLM.from_model_id( + model_id="lmsys/vicuna-7b-v1.5", + model_kwargs={"temperature": 0, "max_length": 16, "trust_remote_code": True}, + ) + output = llm.generate(["Hello!"]) + assert isinstance(output, LLMResult) + assert isinstance(output.generations, list) diff --git a/libs/community/tests/unit_tests/llms/test_imports.py b/libs/community/tests/unit_tests/llms/test_imports.py index e7359360517d0..6f719de2bfcff 100644 --- a/libs/community/tests/unit_tests/llms/test_imports.py +++ b/libs/community/tests/unit_tests/llms/test_imports.py @@ -42,6 +42,7 @@ "HuggingFacePipeline", "HuggingFaceTextGenInference", "HumanInputLLM", + "IpexLLM", "KoboldApiLLM", "Konko", "LlamaCpp",