LLM: Get rid of dependency to pueblo[fileio]

crate · Apr 19, 2024 · e13b437 · e13b437
1 parent 8328f99
commit e13b437
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 7 deletions.
diff --git a/topic/machine-learning/llm-langchain/document_loader.ipynb b/topic/machine-learning/llm-langchain/document_loader.ipynb
@@ -66,8 +66,8 @@
    },
    "outputs": [],
    "source": [
+    "import requests\n",
     "from cratedb_toolkit.util import DatabaseAdapter\n",
-    "from pueblo.io import to_io\n",
     "\n",
     "\n",
     "def import_mlb_teams_2012():\n",
@@ -78,8 +78,8 @@
     "    \"\"\"\n",
     "    cratedb = DatabaseAdapter(dburi=CONNECTION_STRING)\n",
     "    url = \"https://github.com/crate-workbench/langchain/raw/cratedb/docs/docs/integrations/document_loaders/example_data/mlb_teams_2012.sql\"\n",
-    "    with to_io(url) as fp:\n",
-    "        cratedb.run_sql(fp.read())\n",
+    "    sql = requests.get(url).text\n",
+    "    cratedb.run_sql(sql)\n",
     "    cratedb.refresh_table(\"mlb_teams_2012\")\n",
     "\n",
     "\n",

diff --git a/topic/machine-learning/llm-langchain/document_loader.py b/topic/machine-learning/llm-langchain/document_loader.py
@@ -26,9 +26,9 @@
 """
 import os
 
+import requests
 from cratedb_toolkit.util import DatabaseAdapter
 from langchain.document_loaders import CrateDBLoader
-from pueblo.io import to_io
 from pprint import pprint
 
 
@@ -46,8 +46,8 @@ def import_mlb_teams_2012():
     """
     cratedb = DatabaseAdapter(dburi=CONNECTION_STRING)
     url = "https://github.com/crate-workbench/langchain/raw/cratedb/docs/docs/integrations/document_loaders/example_data/mlb_teams_2012.sql"
-    with to_io(url) as fp:
-        cratedb.run_sql(fp.read())
+    sql = requests.get(url).text
+    cratedb.run_sql(sql)
     cratedb.refresh_table("mlb_teams_2012")
 
 

diff --git a/topic/machine-learning/llm-langchain/requirements.txt b/topic/machine-learning/llm-langchain/requirements.txt
@@ -6,10 +6,11 @@ cratedb-toolkit==0.0.10
 # langchain[cratedb,openai]==0.0.354
 # langchain-community==0.0.8
 langchain-openai==0.0.6
-pueblo[cli,fileio,nlp]>=0.0.7
+pueblo[cli,nlp]>=0.0.7
 pydantic>=1,<3
 pypdf<5
 python-dotenv<2
+requests<3
 requests-cache<2
 sqlalchemy==2.*
 unstructured<0.12