From 783e7f6939da251d2f76bd592eaaeefa98db299b Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 29 Sep 2023 00:32:19 +0100 Subject: [PATCH 1/4] working es --- application/parser/open_ai_func.py | 7 +- application/requirements.txt | 1 + application/vectorstore/base.py | 2 +- application/vectorstore/elasticsearch.py | 199 +++++++++++++++++++++++ application/vectorstore/faiss.py | 5 +- frontend/package-lock.json | 52 +++--- 6 files changed, 235 insertions(+), 31 deletions(-) create mode 100644 application/vectorstore/elasticsearch.py diff --git a/application/parser/open_ai_func.py b/application/parser/open_ai_func.py index 969165d2f..0489eb870 100644 --- a/application/parser/open_ai_func.py +++ b/application/parser/open_ai_func.py @@ -1,8 +1,7 @@ import os import tiktoken -from langchain.embeddings import OpenAIEmbeddings -from langchain.vectorstores import FAISS +from application.vectorstore.faiss import FaissStore from retry import retry @@ -33,11 +32,9 @@ def call_openai_api(docs, folder_name, task_status): os.makedirs(f"{folder_name}") from tqdm import tqdm - docs_test = [docs[0]] - docs.pop(0) c1 = 0 - store = FAISS.from_documents(docs_test, OpenAIEmbeddings(openai_api_key=os.getenv("EMBEDDINGS_KEY"))) + store = FaissStore(path=f"{folder_name}", embeddings_key=os.getenv("EMBEDDINGS_KEY")) # Uncomment for MPNet embeddings # model_name = "sentence-transformers/all-mpnet-base-v2" diff --git a/application/requirements.txt b/application/requirements.txt index d978cb416..68532aa18 100644 --- a/application/requirements.txt +++ b/application/requirements.txt @@ -22,6 +22,7 @@ decorator==5.1.1 dill==0.3.6 dnspython==2.3.0 ecdsa==0.18.0 +elasticsearch==8.9.0 entrypoints==0.4 faiss-cpu==1.7.3 filelock==3.9.0 diff --git a/application/vectorstore/base.py b/application/vectorstore/base.py index ad481744f..18a3881b0 100644 --- a/application/vectorstore/base.py +++ b/application/vectorstore/base.py @@ -19,7 +19,7 @@ def search(self, *args, **kwargs): def is_azure_configured(self): return settings.OPENAI_API_BASE and settings.OPENAI_API_VERSION and settings.AZURE_DEPLOYMENT_NAME - def _get_docsearch(self, embeddings_name, embeddings_key=None): + def _get_embeddings(self, embeddings_name, embeddings_key=None): embeddings_factory = { "openai_text-embedding-ada-002": OpenAIEmbeddings, "huggingface_sentence-transformers/all-mpnet-base-v2": HuggingFaceHubEmbeddings, diff --git a/application/vectorstore/elasticsearch.py b/application/vectorstore/elasticsearch.py new file mode 100644 index 000000000..b87f851a4 --- /dev/null +++ b/application/vectorstore/elasticsearch.py @@ -0,0 +1,199 @@ +from application.vectorstore.base import BaseVectorStore +from application.core.settings import settings +import elasticsearch +#from langchain.vectorstores.elasticsearch import ElasticsearchStore + + +class ElasticsearchStore(BaseVectorStore): + _es_connection = None # Class attribute to hold the Elasticsearch connection + + def __init__(self, path, embeddings_key, index_name="docsgpt"): + super().__init__() + self.path = path.replace("/app/application/indexes/", "") + self.embeddings_key = embeddings_key + self.index_name = index_name + + if ElasticsearchStore._es_connection is None: + connection_params = {} + connection_params["cloud_id"] = settings.ELASTIC_CLOUD_ID + connection_params["basic_auth"] = (settings.ELASTIC_USERNAME, settings.ELASTIC_PASSWORD) + ElasticsearchStore._es_connection = elasticsearch.Elasticsearch(**connection_params) + + self.docsearch = ElasticsearchStore._es_connection + + def connect_to_elasticsearch( + *, + es_url = None, + cloud_id = None, + api_key = None, + username = None, + password = None, + ): + try: + import elasticsearch + except ImportError: + raise ImportError( + "Could not import elasticsearch python package. " + "Please install it with `pip install elasticsearch`." + ) + + if es_url and cloud_id: + raise ValueError( + "Both es_url and cloud_id are defined. Please provide only one." + ) + + connection_params = {} + + if es_url: + connection_params["hosts"] = [es_url] + elif cloud_id: + connection_params["cloud_id"] = cloud_id + else: + raise ValueError("Please provide either elasticsearch_url or cloud_id.") + + if api_key: + connection_params["api_key"] = api_key + elif username and password: + connection_params["basic_auth"] = (username, password) + + es_client = elasticsearch.Elasticsearch( + **connection_params, + ) + try: + es_client.info() + except Exception as e: + raise e + + return es_client + + def search(self, question, k=2, index_name=settings.ELASTIC_INDEX, *args, **kwargs): + embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key) + vector = embeddings.embed_query(question) + knn = { + "filter": [{"match": {"metadata.filename.keyword": self.path}}], + "field": "vector", + "k": k, + "num_candidates": 100, + "query_vector": vector, + } + full_query = { + "knn": knn, + "query": { + "bool": { + "must": [ + { + "match": { + "text": { + "query": question, + } + } + } + ], + "filter": [{"match": {"metadata.filename.keyword": self.path}}], + } + }, + "rank": {"rrf": {}}, + } + resp = self.docsearch.search(index=index_name, query=full_query['query'], size=k, knn=full_query['knn']) + return resp + + def _create_index_if_not_exists( + self, index_name, dims_length + ): + + if self.client.indices.exists(index=index_name): + print(f"Index {index_name} already exists.") + + else: + self.strategy.before_index_setup( + client=self.client, + text_field=self.query_field, + vector_query_field=self.vector_query_field, + ) + + indexSettings = self.index( + dims_length=dims_length, + ) + self.client.indices.create(index=index_name, **indexSettings) + def index( + self, + dims_length, + ): + + + return { + "mappings": { + "properties": { + "vector": { + "type": "dense_vector", + "dims": dims_length, + "index": True, + "similarity": "cosine", + }, + } + } + } + + def add_texts( + self, + texts, + metadatas = None, + ids = None, + refresh_indices = True, + create_index_if_not_exists = True, + bulk_kwargs = None, + **kwargs, + ): + + from elasticsearch.helpers import BulkIndexError, bulk + + bulk_kwargs = bulk_kwargs or {} + import uuid + embeddings = [] + ids = ids or [str(uuid.uuid4()) for _ in texts] + requests = [] + embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key) + + vectors = embeddings.embed_documents(list(texts)) + + dims_length = len(vectors[0]) + + if create_index_if_not_exists: + self._create_index_if_not_exists( + index_name=self.index_name, dims_length=dims_length + ) + + for i, (text, vector) in enumerate(zip(texts, vectors)): + metadata = metadatas[i] if metadatas else {} + + requests.append( + { + "_op_type": "index", + "_index": self.index_name, + "text": text, + "vector": vector, + "metadata": metadata, + "_id": ids[i], + } + ) + + + if len(requests) > 0: + try: + success, failed = bulk( + self.client, + requests, + stats_only=True, + refresh=refresh_indices, + **bulk_kwargs, + ) + return ids + except BulkIndexError as e: + print(f"Error adding texts: {e}") + firstError = e.errors[0].get("index", {}).get("error", {}) + print(f"First error reason: {firstError.get('reason')}") + raise e + + else: + return [] + diff --git a/application/vectorstore/faiss.py b/application/vectorstore/faiss.py index 9a562dce6..d85b6084e 100644 --- a/application/vectorstore/faiss.py +++ b/application/vectorstore/faiss.py @@ -8,8 +8,11 @@ def __init__(self, path, embeddings_key): super().__init__() self.path = path self.docsearch = FAISS.load_local( - self.path, self._get_docsearch(settings.EMBEDDINGS_NAME, settings.EMBEDDINGS_KEY) + self.path, self._get_embeddings(settings.EMBEDDINGS_NAME, settings.EMBEDDINGS_KEY) ) def search(self, *args, **kwargs): return self.docsearch.similarity_search(*args, **kwargs) + + def add_texts(self, *args, **kwargs): + return self.docsearch.add_texts(*args, **kwargs) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index ff8a21f68..415c483b2 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1346,9 +1346,9 @@ } }, "node_modules/@typescript-eslint/eslint-plugin/node_modules/semver": { - "version": "7.3.8", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", - "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dev": true, "dependencies": { "lru-cache": "^6.0.0" @@ -1490,9 +1490,9 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/semver": { - "version": "7.3.8", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", - "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dev": true, "dependencies": { "lru-cache": "^6.0.0" @@ -1549,9 +1549,9 @@ } }, "node_modules/@typescript-eslint/utils/node_modules/semver": { - "version": "7.3.8", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", - "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dev": true, "dependencies": { "lru-cache": "^6.0.0" @@ -1991,9 +1991,9 @@ } }, "node_modules/builtins/node_modules/semver": { - "version": "7.3.8", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", - "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dev": true, "dependencies": { "lru-cache": "^6.0.0" @@ -2055,9 +2055,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001450", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001450.tgz", - "integrity": "sha512-qMBmvmQmFXaSxexkjjfMvD5rnDL0+m+dUMZKoDYsGG8iZN29RuYh9eRoMvKsT6uMAWlyUUGDEQGJJYjzCIO9ew==", + "version": "1.0.30001541", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001541.tgz", + "integrity": "sha512-bLOsqxDgTqUBkzxbNlSBt8annkDpQB9NdzdTbO2ooJ+eC/IQcvDspDc058g84ejCelF7vHUx57KIOjEecOHXaw==", "dev": true, "funding": [ { @@ -2067,6 +2067,10 @@ { "type": "tidelift", "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" } ] }, @@ -2889,9 +2893,9 @@ } }, "node_modules/eslint-plugin-n/node_modules/semver": { - "version": "7.3.8", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", - "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "version": "7.5.4", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.5.4.tgz", + "integrity": "sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==", "dev": true, "dependencies": { "lru-cache": "^6.0.0" @@ -4478,9 +4482,9 @@ } }, "node_modules/lint-staged/node_modules/yaml": { - "version": "2.2.1", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.2.1.tgz", - "integrity": "sha512-e0WHiYql7+9wr4cWMx3TVQrNwejKaEe7/rHNmQmqRjazfOP5W8PB6Jpebb5o6fIapbz9o9+2ipcaTM2ZwDI6lw==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.3.2.tgz", + "integrity": "sha512-N/lyzTPaJasoDmfV7YTrYCI0G/3ivm/9wdG0aHuheKowWQwGTsK0Eoiw6utmzAnI6pkJa0DUVygvp3spqqEKXg==", "dev": true, "engines": { "node": ">= 14" @@ -6532,9 +6536,9 @@ } }, "node_modules/semver": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.0.tgz", - "integrity": "sha512-b39TBaTSfV6yBrapU89p5fKekE2m/NwnDocOVruQFS1/veMgdzuPcnOM34M6CwxW8jH/lxEa5rBoDeUwu5HHTw==", + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", "dev": true, "bin": { "semver": "bin/semver.js" From 347cfe253f92f8384a1bf99a9d47095083674d4d Mon Sep 17 00:00:00 2001 From: Alex Date: Fri, 29 Sep 2023 17:17:48 +0100 Subject: [PATCH 2/4] elastic2 --- application/api/answer/routes.py | 6 +- application/api/internal/routes.py | 37 ++-- application/api/user/routes.py | 27 ++- application/core/settings.py | 8 + application/parser/open_ai_func.py | 25 ++- application/vectorstore/elasticsearch.py | 204 ++++++++++++---------- application/vectorstore/faiss.py | 16 +- application/vectorstore/vector_creator.py | 16 ++ application/worker.py | 14 +- 9 files changed, 218 insertions(+), 135 deletions(-) create mode 100644 application/vectorstore/vector_creator.py diff --git a/application/api/answer/routes.py b/application/api/answer/routes.py index ae9ef71f7..b787115ef 100644 --- a/application/api/answer/routes.py +++ b/application/api/answer/routes.py @@ -14,7 +14,7 @@ from application.core.settings import settings from application.llm.openai import OpenAILLM, AzureOpenAILLM -from application.vectorstore.faiss import FaissStore +from application.vectorstore.vector_creator import VectorCreator from application.error import bad_request @@ -234,7 +234,7 @@ def stream(): vectorstore = get_vectorstore({"active_docs": data["active_docs"]}) else: vectorstore = "" - docsearch = FaissStore(vectorstore, embeddings_key) + docsearch = VectorCreator.create_vectorstore(settings.VECTOR_STORE, vectorstore, embeddings_key) return Response( complete_stream(question, docsearch, @@ -268,7 +268,7 @@ def api_answer(): vectorstore = get_vectorstore(data) # loading the index and the store and the prompt template # Note if you have used other embeddings than OpenAI, you need to change the embeddings - docsearch = FaissStore(vectorstore, embeddings_key) + docsearch = VectorCreator.create_vectorstore(settings.VECTOR_STORE, vectorstore, embeddings_key) if is_azure_configured(): llm = AzureOpenAILLM( diff --git a/application/api/internal/routes.py b/application/api/internal/routes.py index ca6da1747..e8a1b80b1 100644 --- a/application/api/internal/routes.py +++ b/application/api/internal/routes.py @@ -34,25 +34,26 @@ def upload_index_files(): if "name" not in request.form: return {"status": "no name"} job_name = secure_filename(request.form["name"]) - if "file_faiss" not in request.files: - print("No file part") - return {"status": "no file"} - file_faiss = request.files["file_faiss"] - if file_faiss.filename == "": - return {"status": "no file name"} - if "file_pkl" not in request.files: - print("No file part") - return {"status": "no file"} - file_pkl = request.files["file_pkl"] - if file_pkl.filename == "": - return {"status": "no file name"} - - # saves index files save_dir = os.path.join(current_dir, "indexes", user, job_name) - if not os.path.exists(save_dir): - os.makedirs(save_dir) - file_faiss.save(os.path.join(save_dir, "index.faiss")) - file_pkl.save(os.path.join(save_dir, "index.pkl")) + if settings.VECTOR_STORE == "faiss": + if "file_faiss" not in request.files: + print("No file part") + return {"status": "no file"} + file_faiss = request.files["file_faiss"] + if file_faiss.filename == "": + return {"status": "no file name"} + if "file_pkl" not in request.files: + print("No file part") + return {"status": "no file"} + file_pkl = request.files["file_pkl"] + if file_pkl.filename == "": + return {"status": "no file name"} + # saves index files + + if not os.path.exists(save_dir): + os.makedirs(save_dir) + file_faiss.save(os.path.join(save_dir, "index.faiss")) + file_pkl.save(os.path.join(save_dir, "index.pkl")) # create entry in vectors_collection vectors_collection.insert_one( { diff --git a/application/api/user/routes.py b/application/api/user/routes.py index f04631d23..2b1d505a5 100644 --- a/application/api/user/routes.py +++ b/application/api/user/routes.py @@ -11,6 +11,8 @@ from application.api.user.tasks import ingest from application.core.settings import settings +from application.vectorstore.vector_creator import VectorCreator + mongo = MongoClient(settings.MONGO_URI) db = mongo["docsgpt"] conversations_collection = db["conversations"] @@ -90,10 +92,17 @@ def delete_old(): return {"status": "error"} path_clean = "/".join(dirs) vectors_collection.delete_one({"location": path}) - try: - shutil.rmtree(path_clean) - except FileNotFoundError: - pass + if settings.VECTOR_STORE == "faiss": + try: + shutil.rmtree(os.path.join(current_dir, path_clean)) + except FileNotFoundError: + pass + else: + vetorstore = VectorCreator.create_vectorstore( + settings.VECTOR_STORE, path=os.path.join(current_dir, path_clean) + ) + vetorstore.delete_index() + return {"status": "ok"} @user.route("/api/upload", methods=["POST"]) @@ -173,11 +182,11 @@ def combined_json(): "location": "local", } ) - - data_remote = requests.get("https://d3dg1063dc54p9.cloudfront.net/combined.json").json() - for index in data_remote: - index["location"] = "remote" - data.append(index) + if settings.VECTOR_STORE == "faiss": + data_remote = requests.get("https://d3dg1063dc54p9.cloudfront.net/combined.json").json() + for index in data_remote: + index["location"] = "remote" + data.append(index) return jsonify(data) diff --git a/application/core/settings.py b/application/core/settings.py index d127c293b..7ec4aae83 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -13,6 +13,7 @@ class Settings(BaseSettings): TOKENS_MAX_HISTORY: int = 150 SELF_HOSTED_MODEL: bool = False UPLOAD_FOLDER: str = "inputs" + VECTOR_STORE: str = "elasticsearch" # "faiss" or "elasticsearch" API_URL: str = "http://localhost:7091" # backend url for celery worker @@ -23,6 +24,13 @@ class Settings(BaseSettings): AZURE_DEPLOYMENT_NAME: str = None # azure deployment name for answering AZURE_EMBEDDINGS_DEPLOYMENT_NAME: str = None # azure deployment name for embeddings + # elasticsearch + ELASTIC_CLOUD_ID: str = 'Docsgpt:ZXUtY2VudHJhbC0xLmF3cy5jbG91ZC5lcy5pbzo0NDMkYmNiZDYxZDE0ODE0NDNhMTkxNDU2YmI2MWViNzUxNTkkN2IwODMxZWYwMDI0NDFiOGJiNzgxZmQzYjI0MjIxYjA=' # cloud id for elasticsearch + ELASTIC_USERNAME: str = 'elastic' # username for elasticsearch + ELASTIC_PASSWORD: str = 'eSwoSbAhIWkXBsRdvhZxGPwc' # password for elasticsearch + ELASTIC_URL: str = None # url for elasticsearch + ELASTIC_INDEX: str = "docsgptbeta" # index name for elasticsearch + path = Path(__file__).parent.parent.absolute() settings = Settings(_env_file=path.joinpath(".env"), _env_file_encoding="utf-8") diff --git a/application/parser/open_ai_func.py b/application/parser/open_ai_func.py index 0489eb870..ede635a85 100644 --- a/application/parser/open_ai_func.py +++ b/application/parser/open_ai_func.py @@ -1,7 +1,8 @@ import os import tiktoken -from application.vectorstore.faiss import FaissStore +from application.vectorstore.vector_creator import VectorCreator +from application.core.settings import settings from retry import retry @@ -33,9 +34,22 @@ def call_openai_api(docs, folder_name, task_status): from tqdm import tqdm c1 = 0 - - store = FaissStore(path=f"{folder_name}", embeddings_key=os.getenv("EMBEDDINGS_KEY")) - + if settings.VECTOR_STORE == "faiss": + docs_init = [docs[0]] + docs.pop(0) + + store = VectorCreator.create_vectorstore( + settings.VECTOR_STORE, + docs_init = docs_init, + path=f"{folder_name}", + embeddings_key=os.getenv("EMBEDDINGS_KEY") + ) + else: + store = VectorCreator.create_vectorstore( + settings.VECTOR_STORE, + path=f"{folder_name}", + embeddings_key=os.getenv("EMBEDDINGS_KEY") + ) # Uncomment for MPNet embeddings # model_name = "sentence-transformers/all-mpnet-base-v2" # hf = HuggingFaceEmbeddings(model_name=model_name) @@ -54,7 +68,8 @@ def call_openai_api(docs, folder_name, task_status): store.save_local(f"{folder_name}") break c1 += 1 - store.save_local(f"{folder_name}") + if settings.VECTOR_STORE == "faiss": + store.save_local(f"{folder_name}") def get_user_permission(docs, folder_name): diff --git a/application/vectorstore/elasticsearch.py b/application/vectorstore/elasticsearch.py index b87f851a4..ca98c5eac 100644 --- a/application/vectorstore/elasticsearch.py +++ b/application/vectorstore/elasticsearch.py @@ -1,22 +1,38 @@ from application.vectorstore.base import BaseVectorStore from application.core.settings import settings import elasticsearch -#from langchain.vectorstores.elasticsearch import ElasticsearchStore + +class Document(str): + """Class for storing a piece of text and associated metadata.""" + + page_content: str + """String text.""" + metadata: dict + """Arbitrary metadata""" class ElasticsearchStore(BaseVectorStore): _es_connection = None # Class attribute to hold the Elasticsearch connection - def __init__(self, path, embeddings_key, index_name="docsgpt"): + def __init__(self, path, embeddings_key, index_name=settings.ELASTIC_INDEX): super().__init__() - self.path = path.replace("/app/application/indexes/", "") + self.path = path.replace("application/indexes/", "") self.embeddings_key = embeddings_key self.index_name = index_name if ElasticsearchStore._es_connection is None: connection_params = {} - connection_params["cloud_id"] = settings.ELASTIC_CLOUD_ID - connection_params["basic_auth"] = (settings.ELASTIC_USERNAME, settings.ELASTIC_PASSWORD) + if settings.ELASTIC_URL: + connection_params["hosts"] = [settings.ELASTIC_URL] + connection_params["http_auth"] = (settings.ELASTIC_USERNAME, settings.ELASTIC_PASSWORD) + elif settings.ELASTIC_CLOUD_ID: + connection_params["cloud_id"] = settings.ELASTIC_CLOUD_ID + connection_params["basic_auth"] = (settings.ELASTIC_USERNAME, settings.ELASTIC_PASSWORD) + else: + raise ValueError("Please provide either elasticsearch_url or cloud_id.") + + + ElasticsearchStore._es_connection = elasticsearch.Elasticsearch(**connection_params) self.docsearch = ElasticsearchStore._es_connection @@ -94,106 +110,112 @@ def search(self, question, k=2, index_name=settings.ELASTIC_INDEX, *args, **kwar }, "rank": {"rrf": {}}, } - resp = self.docsearch.search(index=index_name, query=full_query['query'], size=k, knn=full_query['knn']) - return resp + resp = self.docsearch.search(index=self.index_name, query=full_query['query'], size=k, knn=full_query['knn']) + # create Documnets objects from the results page_content ['_source']['text'], metadata ['_source']['metadata'] + import sys + print(self.path, file=sys.stderr) + print(resp, file=sys.stderr) + doc_list = [] + for hit in resp['hits']['hits']: + + doc_list.append(Document(page_content = hit['_source']['text'], metadata = hit['_source']['metadata'])) + return doc_list - def _create_index_if_not_exists( - self, index_name, dims_length - ): + def _create_index_if_not_exists( + self, index_name, dims_length + ): - if self.client.indices.exists(index=index_name): - print(f"Index {index_name} already exists.") + if self._es_connection.indices.exists(index=index_name): + print(f"Index {index_name} already exists.") - else: - self.strategy.before_index_setup( - client=self.client, - text_field=self.query_field, - vector_query_field=self.vector_query_field, - ) + else: - indexSettings = self.index( - dims_length=dims_length, - ) - self.client.indices.create(index=index_name, **indexSettings) - def index( - self, - dims_length, - ): - - - return { - "mappings": { - "properties": { - "vector": { - "type": "dense_vector", - "dims": dims_length, - "index": True, - "similarity": "cosine", - }, - } + indexSettings = self.index( + dims_length=dims_length, + ) + self._es_connection.indices.create(index=index_name, **indexSettings) + + def index( + self, + dims_length, + ): + return { + "mappings": { + "properties": { + "vector": { + "type": "dense_vector", + "dims": dims_length, + "index": True, + "similarity": "cosine", + }, } } + } - def add_texts( - self, - texts, - metadatas = None, - ids = None, - refresh_indices = True, - create_index_if_not_exists = True, - bulk_kwargs = None, - **kwargs, + def add_texts( + self, + texts, + metadatas = None, + ids = None, + refresh_indices = True, + create_index_if_not_exists = True, + bulk_kwargs = None, + **kwargs, ): - - from elasticsearch.helpers import BulkIndexError, bulk + + from elasticsearch.helpers import BulkIndexError, bulk - bulk_kwargs = bulk_kwargs or {} - import uuid - embeddings = [] - ids = ids or [str(uuid.uuid4()) for _ in texts] - requests = [] - embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key) + bulk_kwargs = bulk_kwargs or {} + import uuid + embeddings = [] + ids = ids or [str(uuid.uuid4()) for _ in texts] + requests = [] + embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key) - vectors = embeddings.embed_documents(list(texts)) + vectors = embeddings.embed_documents(list(texts)) - dims_length = len(vectors[0]) + dims_length = len(vectors[0]) - if create_index_if_not_exists: - self._create_index_if_not_exists( - index_name=self.index_name, dims_length=dims_length - ) + if create_index_if_not_exists: + self._create_index_if_not_exists( + index_name=self.index_name, dims_length=dims_length + ) - for i, (text, vector) in enumerate(zip(texts, vectors)): - metadata = metadatas[i] if metadatas else {} - - requests.append( - { - "_op_type": "index", - "_index": self.index_name, - "text": text, - "vector": vector, - "metadata": metadata, - "_id": ids[i], - } - ) + for i, (text, vector) in enumerate(zip(texts, vectors)): + metadata = metadatas[i] if metadatas else {} + + requests.append( + { + "_op_type": "index", + "_index": self.index_name, + "text": text, + "vector": vector, + "metadata": metadata, + "_id": ids[i], + } + ) - if len(requests) > 0: - try: - success, failed = bulk( - self.client, - requests, - stats_only=True, - refresh=refresh_indices, - **bulk_kwargs, - ) - return ids - except BulkIndexError as e: - print(f"Error adding texts: {e}") - firstError = e.errors[0].get("index", {}).get("error", {}) - print(f"First error reason: {firstError.get('reason')}") - raise e + if len(requests) > 0: + try: + success, failed = bulk( + self._es_connection, + requests, + stats_only=True, + refresh=refresh_indices, + **bulk_kwargs, + ) + return ids + except BulkIndexError as e: + print(f"Error adding texts: {e}") + firstError = e.errors[0].get("index", {}).get("error", {}) + print(f"First error reason: {firstError.get('reason')}") + raise e - else: - return [] + else: + return [] + + def delete_index(self): + self._es_connection.delete_by_query(index=self.index_name, query={"match": { + "metadata.filename.keyword": self.path}},) diff --git a/application/vectorstore/faiss.py b/application/vectorstore/faiss.py index d85b6084e..5c5cee703 100644 --- a/application/vectorstore/faiss.py +++ b/application/vectorstore/faiss.py @@ -4,15 +4,23 @@ class FaissStore(BaseVectorStore): - def __init__(self, path, embeddings_key): + def __init__(self, path, embeddings_key, docs_init=None): super().__init__() self.path = path - self.docsearch = FAISS.load_local( - self.path, self._get_embeddings(settings.EMBEDDINGS_NAME, settings.EMBEDDINGS_KEY) - ) + if docs_init: + self.docsearch = FAISS.from_documents( + docs_init, self._get_embeddings(settings.EMBEDDINGS_NAME, embeddings_key) + ) + else: + self.docsearch = FAISS.load_local( + self.path, self._get_embeddings(settings.EMBEDDINGS_NAME, settings.EMBEDDINGS_KEY) + ) def search(self, *args, **kwargs): return self.docsearch.similarity_search(*args, **kwargs) def add_texts(self, *args, **kwargs): return self.docsearch.add_texts(*args, **kwargs) + + def save_local(self, *args, **kwargs): + return self.docsearch.save_local(*args, **kwargs) diff --git a/application/vectorstore/vector_creator.py b/application/vectorstore/vector_creator.py new file mode 100644 index 000000000..cbc491f51 --- /dev/null +++ b/application/vectorstore/vector_creator.py @@ -0,0 +1,16 @@ +from application.vectorstore.faiss import FaissStore +from application.vectorstore.elasticsearch import ElasticsearchStore + + +class VectorCreator: + vectorstores = { + 'faiss': FaissStore, + 'elasticsearch':ElasticsearchStore + } + + @classmethod + def create_vectorstore(cls, type, *args, **kwargs): + vectorstore_class = cls.vectorstores.get(type.lower()) + if not vectorstore_class: + raise ValueError(f"No vectorstore class found for type {type}") + return vectorstore_class(*args, **kwargs) \ No newline at end of file diff --git a/application/worker.py b/application/worker.py index 91c19c309..141aa881a 100644 --- a/application/worker.py +++ b/application/worker.py @@ -83,11 +83,15 @@ def ingest_worker(self, directory, formats, name_job, filename, user): # get files from outputs/inputs/index.faiss and outputs/inputs/index.pkl # and send them to the server (provide user and name in form) file_data = {'name': name_job, 'user': user} - files = {'file_faiss': open(full_path + '/index.faiss', 'rb'), - 'file_pkl': open(full_path + '/index.pkl', 'rb')} - response = requests.post(urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data) - - response = requests.get(urljoin(settings.API_URL, "/api/delete_old?path=")) + if settings.VECTOR_STORE == "faiss": + files = {'file_faiss': open(full_path + '/index.faiss', 'rb'), + 'file_pkl': open(full_path + '/index.pkl', 'rb')} + response = requests.post(urljoin(settings.API_URL, "/api/upload_index"), files=files, data=file_data) + response = requests.get(urljoin(settings.API_URL, "/api/delete_old?path=" + full_path)) + else: + response = requests.post(urljoin(settings.API_URL, "/api/upload_index"), data=file_data) + + # delete local shutil.rmtree(full_path) From d85eb83ea2364b27e394f74ce85cdfa5f2f5fb75 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 30 Sep 2023 15:25:31 +0100 Subject: [PATCH 3/4] elastic search fixes --- application/vectorstore/elasticsearch.py | 22 +++++++++++----------- application/worker.py | 4 +++- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/application/vectorstore/elasticsearch.py b/application/vectorstore/elasticsearch.py index ca98c5eac..9df8800b7 100644 --- a/application/vectorstore/elasticsearch.py +++ b/application/vectorstore/elasticsearch.py @@ -5,10 +5,13 @@ class Document(str): """Class for storing a piece of text and associated metadata.""" - page_content: str - """String text.""" - metadata: dict - """Arbitrary metadata""" + def __new__(cls, page_content: str, metadata: dict): + instance = super().__new__(cls, page_content) + instance.page_content = page_content + instance.metadata = metadata + return instance + + class ElasticsearchStore(BaseVectorStore): @@ -16,7 +19,7 @@ class ElasticsearchStore(BaseVectorStore): def __init__(self, path, embeddings_key, index_name=settings.ELASTIC_INDEX): super().__init__() - self.path = path.replace("application/indexes/", "") + self.path = path.replace("application/indexes/", "").rstrip("/") self.embeddings_key = embeddings_key self.index_name = index_name @@ -86,7 +89,7 @@ def search(self, question, k=2, index_name=settings.ELASTIC_INDEX, *args, **kwar embeddings = self._get_embeddings(settings.EMBEDDINGS_NAME, self.embeddings_key) vector = embeddings.embed_query(question) knn = { - "filter": [{"match": {"metadata.filename.keyword": self.path}}], + "filter": [{"match": {"metadata.store.keyword": self.path}}], "field": "vector", "k": k, "num_candidates": 100, @@ -105,16 +108,13 @@ def search(self, question, k=2, index_name=settings.ELASTIC_INDEX, *args, **kwar } } ], - "filter": [{"match": {"metadata.filename.keyword": self.path}}], + "filter": [{"match": {"metadata.store.keyword": self.path}}], } }, "rank": {"rrf": {}}, } resp = self.docsearch.search(index=self.index_name, query=full_query['query'], size=k, knn=full_query['knn']) # create Documnets objects from the results page_content ['_source']['text'], metadata ['_source']['metadata'] - import sys - print(self.path, file=sys.stderr) - print(resp, file=sys.stderr) doc_list = [] for hit in resp['hits']['hits']: @@ -217,5 +217,5 @@ def add_texts( def delete_index(self): self._es_connection.delete_by_query(index=self.index_name, query={"match": { - "metadata.filename.keyword": self.path}},) + "metadata.store.keyword": self.path}},) diff --git a/application/worker.py b/application/worker.py index 141aa881a..5c87c7072 100644 --- a/application/worker.py +++ b/application/worker.py @@ -21,7 +21,9 @@ def metadata_from_filename(title): - return {'title': title} + store = title.split('/') + store = store[1] + '/' + store[2] + return {'title': title, 'store': store} def generate_random_string(length): From 3eacfb91aa30dab64c9c5a061a5fbbae67c5d9e7 Mon Sep 17 00:00:00 2001 From: Alex Date: Sat, 30 Sep 2023 15:32:37 +0100 Subject: [PATCH 4/4] fix314 --- application/core/settings.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/application/core/settings.py b/application/core/settings.py index 947038e7e..c31091448 100644 --- a/application/core/settings.py +++ b/application/core/settings.py @@ -25,11 +25,11 @@ class Settings(BaseSettings): AZURE_EMBEDDINGS_DEPLOYMENT_NAME: str = None # azure deployment name for embeddings # elasticsearch - ELASTIC_CLOUD_ID: str = 'Docsgpt:ZXUtY2VudHJhbC0xLmF3cy5jbG91ZC5lcy5pbzo0NDMkYmNiZDYxZDE0ODE0NDNhMTkxNDU2YmI2MWViNzUxNTkkN2IwODMxZWYwMDI0NDFiOGJiNzgxZmQzYjI0MjIxYjA=' # cloud id for elasticsearch - ELASTIC_USERNAME: str = 'elastic' # username for elasticsearch - ELASTIC_PASSWORD: str = 'eSwoSbAhIWkXBsRdvhZxGPwc' # password for elasticsearch + ELASTIC_CLOUD_ID: str = None # cloud id for elasticsearch + ELASTIC_USERNAME: str = None # username for elasticsearch + ELASTIC_PASSWORD: str = None # password for elasticsearch ELASTIC_URL: str = None # url for elasticsearch - ELASTIC_INDEX: str = "docsgptbeta" # index name for elasticsearch + ELASTIC_INDEX: str = "docsgpt" # index name for elasticsearch path = Path(__file__).parent.parent.absolute()