From 30d7834db43942e16e67ff26db25a8921307b384 Mon Sep 17 00:00:00 2001 From: Gaurav Tarlok Kakkar Date: Sat, 2 Sep 2023 10:59:20 -0700 Subject: [PATCH] release: merge staging into master (#1032) Co-authored-by: Joy Arulraj Co-authored-by: Andy Xu Co-authored-by: Jiashen Cao Co-authored-by: rohith mulumudy Co-authored-by: Chitti Ankith Co-authored-by: Ankith Reddy Chitti --- README.md | 125 ++++++++++++++- docs/_toc.yml | 41 +++++ docs/conf.py | 30 ++++ docs/source/benchmarks/text_summarization.rst | 56 +++++++ .../source/dev-guide/release/pypi-account.rst | 6 + .../dev-guide/release/release-steps.rst | 6 + docs/source/overview/concepts.rst | 56 +++++++ docs/source/overview/connect-to-database.rst | 65 ++++++++ docs/source/overview/getting-started.rst | 122 +++++++++++++++ .../getting-started/installation-options.rst | 4 + docs/source/reference/api.rst | 53 +++++++ docs/source/reference/evaql/create.rst | 14 ++ docs/source/reference/evaql/drop.rst | 5 + docs/source/reference/evaql/select.rst | 9 ++ docs/source/reference/evaql/show.rst | 5 + docs/source/reference/evaql/udf.rst | 14 ++ docs/source/reference/evaql/use.rst | 13 ++ docs/source/reference/udfs/model-train.rst | 48 ++++++ docs/source/reference/udfs/openai.rst | 27 ++++ docs/source/usecases/food-review.rst | 147 ++++++++++++++++++ docs/source/usecases/image-classification.rst | 37 +++++ docs/source/usecases/qa-video.rst | 89 +++++++++++ docs/source/usecases/similar-image-search.rst | 83 ++++++++++ evadb/binder/binder_utils.py | 27 ++++ evadb/binder/statement_binder_context.py | 9 ++ evadb/executor/create_database_executor.py | 15 ++ evadb/executor/create_executor.py | 29 ++++ evadb/executor/create_udf_executor.py | 17 ++ evadb/executor/use_executor.py | 3 + evadb/third_party/databases/interface.py | 21 +++ .../databases/mysql/mysql_handler.py | 12 ++ .../databases/postgres/postgres_handler.py | 21 +++ .../databases/sqlite/sqlite_handler.py | 34 ++++ evadb/version.py | 9 ++ script/formatting/spelling.txt | 31 ++++ script/test/test.sh | 7 + setup.py | 10 ++ .../short/test_create_database_executor.py | 6 + .../third_party_tests/test_native_executor.py | 37 +++++ .../test_sqlite_native_storage_engine.py | 6 + 40 files changed, 1347 insertions(+), 2 deletions(-) create mode 100644 docs/source/reference/udfs/model-train.rst create mode 100644 docs/source/reference/udfs/openai.rst create mode 100644 docs/source/usecases/food-review.rst create mode 100644 docs/source/usecases/qa-video.rst create mode 100644 docs/source/usecases/similar-image-search.rst diff --git a/README.md b/README.md index 9adfbe5bde..5afd973e45 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@
@@ -212,6 +227,9 @@ EvaDB enables software developers to build AI apps in a few lines of code. Its p <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) - [Quick Links](#quick-links) - [Documentation](#documentation) - [Why EvaDB](#why-evadb) @@ -225,6 +243,7 @@ EvaDB enables software developers to build AI apps in a few lines of code. Its p - [Star History](#star-history) - [License](#license) ======= +<<<<<<< HEAD ======= - [Quick Links](#quick-links) >>>>>>> 2dacff69 (feat: sync master staging (#1050)) @@ -239,6 +258,14 @@ EvaDB enables software developers to build AI apps in a few lines of code. Its p - [Contributing](#contributing) - [Star History](#star-history) - [License](#license) +======= +- [Documentation](#documentation) +- [Why EvaDB](#why-evadb) +- [How does EvaDB work](#how-does-evadb-work) +- [Community and Support](#community-and-support) +- [Illustrative Queries](#illustrative-queries) +- [Illustrative Apps](#illustrative-apps) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) ## Documentation @@ -379,6 +406,73 @@ SELECT name FROM reddit_dataset ORDER BY ## Illustrative Apps +Here are some illustrative AI apps built using EvaDB (each notebook can be opened on Google Colab): + + * 🔮 PrivateGPT + * 🔮 ChatGPT-based Video Question Answering + * 🔮 Querying PDF Documents + * 🔮 Analysing Traffic Flow with YOLO + * 🔮 Examining Emotions of Movie + * 🔮 Image Segmentation with Hugging Face +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) + + +<<<<<<< HEAD +You can find the complete documentation of EvaDB at [evadb.ai/docs](https://evadb.ai/docs/) 📚✨🚀 + +## Why EvaDB + +In the world of AI, we've reached a stage where many AI tasks that were traditionally handled by AI or ML engineers can now be automated. EvaDB enables software developers with the ability to perform advanced AI tasks without needing to delve into the intricate details. + +EvaDB covers many AI applications, including regression, classification, image recognition, question answering, and many other generative AI applications. EvaDB targets 99% of AI problems that are often repetitive and can be automated with a simple function call in an SQL query. Until now, there is no comprehensive open-source framework for bringing AI into an existing SQL database system with a principled AI optimization framework, and that's where EvaDB comes in. + +Our target audience is software developers who may not necessarily have a background in AI but require AI capabilities to solve specific problems. We target programmers who write simple SQL queries inside their CRUD apps. With EvaDB, it is possible to easily add AI features to these apps by calling built-in AI functions in the queries. + +## How does EvaDB work + +
+
    +
  • Connect EvaDB to your database system with the `CREATE DATABASE` statement.
    • +
  • Write SQL queries with AI functions to get inference results:
    • +
      +
    • Pick a pre-trained AI model from Hugging Face, Open AI, Ultralytics, PyTorch, and built-in AI frameworks for generative AI, NLP, and vision applications;
      • +
    • or pick from a variety of state-of-the-art ML engines for classic ML use-cases (classification, regression, etc.);
      • +
    • or bring your custom model built with any AI/ML framework using `CREATE FUNCTION`.
      • +
    +
+ +Follow the [getting started](https://evadb.readthedocs.io/en/stable/source/overview/getting-started.html) guide to get on-boarded as fast as possible. +
+ +## Illustrative Queries + +* Run the MNIST Image Classification model to obtain digit labels for each frame in the video. + +```sql +SELECT MnistImageClassifier(data).label FROM mnist_video; +``` + +* Build a vector index on the feature embeddings returned by the SIFT Feature Extractor on a collection of Reddit images. + +```sql +CREATE INDEX reddit_sift_image_index + ON reddit_dataset (SiftFeatureExtractor(data)) + USING FAISS +``` + +* Retrieve the top-5 most similar images for the given image using the index. + +```sql +SELECT name FROM reddit_dataset ORDER BY + Similarity( + SiftFeatureExtractor(Open('reddit-images/g1074_d4mxztt.jpg')), + SiftFeatureExtractor(data) + ) + LIMIT 5 +``` + +## Illustrative Apps + Here are some illustrative AI apps built using EvaDB (each notebook can be opened on Google Colab): * 🔮 Sentiment Analysis using LLM within PostgreSQL @@ -427,15 +521,28 @@ EvaDB's AI-centric query optimizer takes a query as input and generates a query
+<<<<<<< HEAD * Get a transcript from a video stored in a table using a Speech Recognition model. Then, ask questions on the extracted transcript using ChatGPT. +======= +* Store the text returned by a Speech Recognition model on the audio component of a video in a table. +>>>>>>> c63abee7 (release: merge staging into master (#1032)) ```sql CREATE TABLE text_summary AS SELECT SpeechRecognizer(audio) FROM ukraine_video; +<<<<<<< HEAD +======= +``` + +* Run ChatGPT on the `text` column in a table. + +```sql +>>>>>>> c63abee7 (release: merge staging into master (#1032)) SELECT ChatGPT('Is this video summary related to Ukraine russia war', text) FROM text_summary; ``` +<<<<<<< HEAD * Train a classic ML model for prediction using the Ludwig AI engine. ```sql @@ -444,6 +551,16 @@ CREATE FUNCTION IF NOT EXISTS PredictHouseRent FROM TYPE Ludwig PREDICT 'rental_price' TIME_LIMIT 120; +======= +* Train an ML model using the Ludwig AI engine to predict a column in a table. + +```sql +CREATE UDF IF NOT EXISTS PredictHouseRent FROM +( SELECT * FROM HomeRentals ) +TYPE Ludwig +'predict' 'rental_price' +'time_limit' 120; +>>>>>>> c63abee7 (release: merge staging into master (#1032)) ```
@@ -453,7 +570,11 @@ TIME_LIMIT 120;
EvaDB's AI-centric query optimizer takes a query as input and generates a query plan. The query engine takes the query plan and hits the relevant backends to efficiently process the query: 1. SQL Database Systems (Structured Data) +<<<<<<< HEAD 2. AI Frameworks (Transform Unstructured Data to Structured Data; Unstructured data includes PDFs, text, images, etc. stored locally or on the cloud) +======= +2. AI Frameworks (Transform Unstructured Data to Structured Data, Unstructured data includes PDFs, images, podcasts, etc. stored on cloud buckets or local filesystem) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) 3. Vector Database Systems (Feature Embeddings) >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) diff --git a/docs/_toc.yml b/docs/_toc.yml index 252e3282b6..974bad27b5 100644 --- a/docs/_toc.yml +++ b/docs/_toc.yml @@ -39,6 +39,7 @@ parts: ======= - file: source/overview/connect-to-database title: Connect to Database +<<<<<<< HEAD - file: source/overview/model-inference title: Model inference - file: source/overview/concepts @@ -46,10 +47,15 @@ parts: sections: - file: source/overview/concepts/data-sources title: Data Sources +======= + - file: source/overview/concepts + title: Concepts +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) #- file: source/overview/faq - caption: Use Cases chapters: +<<<<<<< HEAD - file: source/usecases/sentiment-analysis.rst title: Sentiment Analysis - file: source/usecases/question-answering.rst @@ -75,7 +81,32 @@ parts: title: Home Rental Prediction # - file: source/usecases/privategpt.rst # title: PrivateGPT +<<<<<<< HEAD >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= + - file: source/usecases/food-review.rst + title: Sentiment Analysis + - file: source/usecases/image-classification.rst + title: Image Classification + - file: source/usecases/similar-image-search.rst + title: Image Search + - file: source/usecases/qa-video.rst + title: Video Question Answering + - file: source/usecases/08-chatgpt.ipynb + title: ChatGPT-based Video Question Answering + - file: source/usecases/12-query-pdf.ipynb + title: PDF Question Answering + - file: source/usecases/02-object-detection.ipynb + title: Object Detection + - file: source/usecases/03-emotion-analysis.ipynb + title: Emotions Analysis + - file: source/usecases/07-object-segmentation-huggingface.ipynb + title: Image Segmentation + - file: source/usecases/13-privategpt.ipynb + title: PrivateGPT +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) - caption: User Reference chapters: @@ -121,6 +152,16 @@ parts: - file: source/reference/evaql/delete - file: source/reference/evaql/rename - file: source/reference/evaql/use +<<<<<<< HEAD +======= + + - file: source/reference/databases/index + title: Data Sources + sections: + - file: source/reference/databases/postgres + - file: source/reference/databases/sqlite + - file: source/reference/databases/mysql +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) <<<<<<< HEAD >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) diff --git a/docs/conf.py b/docs/conf.py index d3823a1c80..4a2fb30f62 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -96,10 +96,15 @@ # directories to ignore when looking for source files. exclude_patterns = ["_build", "Thumbs.db", ".DS_Store", "README.md", "images/reference/README.md"] <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) # The name of the Pygments (syntax highlighting) style to use. pygments_style = "github-dark" @@ -178,6 +183,25 @@ nitpick_ignore_regex = [('py:class', r'.*')] <<<<<<< HEAD ======= +<<<<<<< HEAD +======= + + +for i in os.listdir("../tutorials"): + if i in [ + "13-privategpt.ipynb", + "08-chatgpt.ipynb", + "12-query-pdf.ipynb", + "02-object-detection.ipynb", + "03-emotion-analysis.ipynb", + "07-object-segmentation-huggingface.ipynb", + "chatgpt.png", + ]: + shutil.copy(f"../tutorials/{i}", "./source/usecases/") + +nb_execution_mode = "off" +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) <<<<<<< HEAD @@ -209,11 +233,17 @@ def setup(app): app.add_js_file("js/top-navigation.js", defer="defer") <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) app.add_js_file("https://cdn.jsdelivr.net/npm/@docsearch/js@3.3.3/dist/umd/index.js",defer="defer") app.add_js_file("js/algolia.js",defer="defer") ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= app.add_js_file("https://cdn.jsdelivr.net/npm/@docsearch/js@3.3.3/dist/umd/index.js",defer="defer") app.add_js_file("js/algolia.js",defer="defer") >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) diff --git a/docs/source/benchmarks/text_summarization.rst b/docs/source/benchmarks/text_summarization.rst index 052afdacf0..1cf1bc223d 100644 --- a/docs/source/benchmarks/text_summarization.rst +++ b/docs/source/benchmarks/text_summarization.rst @@ -1,5 +1,8 @@ <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) Text Summarization Benchmark ============================ @@ -13,6 +16,7 @@ Prepare dataset --------------- ======= Text summarization benchmark +<<<<<<< HEAD ======= Text Summarization Benchmark >>>>>>> 2dacff69 (feat: sync master staging (#1050)) @@ -32,6 +36,14 @@ All the relevant files are located in the `text summarization benchmark folder o Prepare dataset --------------- >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +============================ +In this benchmark, we compare the performance of text summarization between EvaDB and MindsDB on `CNN-DailyMail News `_. + +1. Prepare dataset +------------------ +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) .. code-block:: bash @@ -40,16 +52,22 @@ Prepare dataset <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) Use EvaDB for Text Summarization -------------------------------- ======= 2. Using EvaDB to summarize the CNN DailyMail News -------------------------------------------------- >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= Use EvaDB for Text Summarization -------------------------------- >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) .. note:: @@ -62,6 +80,7 @@ Use EvaDB for Text Summarization python text_summarization_with_evadb.py +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD Loading Data Into EvaDB @@ -119,6 +138,8 @@ Prepare sqlite database for MindsDB ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) ======= +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) Loading Data Into EvaDB ~~~~~~~~~~~~~~~~~~~~~~~ @@ -166,7 +187,17 @@ Use MindsDB for Text Summarization Setup SQLite Database ~~~~~~~~~~~~~~~~~~~~~~ +<<<<<<< HEAD >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +======= +3. Using MindsDB to summarize the CNN DailyMail News +---------------------------------------------------- + +Prepare sqlite database for MindsDB +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) .. code-block:: bash @@ -179,6 +210,7 @@ Setup SQLite Database Install MindsDB ~~~~~~~~~~~~~~~ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD Follow the `MindsDB installation guide `_ to install it via ``pip``. @@ -194,12 +226,24 @@ Follow the `Setup for Source Code via pip >>>>>> 8c5b63dc (release: merge staging into master (#1032)) ======= +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) Follow the `MindsDB nstallation guide `_ to install it via ``pip``. .. note:: You will need to manually run ``pip install evaluate`` for the ``HuggingFace`` model to work in MindsDB. +<<<<<<< HEAD >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +======= +Follow the `Setup for Source Code via pip `_ to install mindsdb. + +.. note:: + + At the time of this documentation, we need to manually ``pip install evaluate`` for huggingface model to work in MindsDB. +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) After installation, use the ``MySQL`` client for connecting to ``MindsDB``. Update the port number if needed. @@ -209,16 +253,22 @@ After installation, use the ``MySQL`` client for connecting to ``MindsDB``. Upda <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) Benchmark MindsDB ~~~~~~~~~~~~~~~~~ ======= Run Experiment ~~~~~~~~~~~~~~ >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= Benchmark MindsDB ~~~~~~~~~~~~~~~~~ >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) Connect ``MindsDB`` to the ``sqlite`` database we created before: @@ -259,6 +309,9 @@ Use the ``text summarization`` model to summarize the CNN news dataset: <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) Benchmarking Results -------------------- ======= @@ -266,10 +319,13 @@ Benchmarking Results --------------------- Below are numbers from a server with 56 Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz and two Quadro P6000 GPU. >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= Benchmarking Results -------------------- >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) Here are the key runtime metrics for the ``Text Summarization`` benchmark. diff --git a/docs/source/dev-guide/release/pypi-account.rst b/docs/source/dev-guide/release/pypi-account.rst index 929a6d2b99..c0cc711716 100644 --- a/docs/source/dev-guide/release/pypi-account.rst +++ b/docs/source/dev-guide/release/pypi-account.rst @@ -19,10 +19,16 @@ Create a .pypirc in your home directory. It should look like this: <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) Then run ``chmod 600 ~/.pypirc`` so that only you can read/write the file. ======= Then run ``chmod 600 ./.pypirc`` so that only you can read/write the file. >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= Then run ``chmod 600 ~/.pypirc`` so that only you can read/write the file. >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) diff --git a/docs/source/dev-guide/release/release-steps.rst b/docs/source/dev-guide/release/release-steps.rst index d97d858c79..3305090376 100644 --- a/docs/source/dev-guide/release/release-steps.rst +++ b/docs/source/dev-guide/release/release-steps.rst @@ -44,6 +44,9 @@ Build Wheel and Release <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) python script/releasing/releaser.py -n minor -u ======= git tag -a v0.0.6 -m "v0.0.6 release" @@ -115,6 +118,9 @@ Build Wheel and Release Credits: `Snorkel `_ >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= python script/releasing/releaser.py -n minor -u >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) diff --git a/docs/source/overview/concepts.rst b/docs/source/overview/concepts.rst index 87f7004e1d..bf4356f20d 100644 --- a/docs/source/overview/concepts.rst +++ b/docs/source/overview/concepts.rst @@ -155,6 +155,7 @@ After registering ``MnistImageClassifier`` function, you can call the function i FROM MnistVideo WHERE id = 30; +<<<<<<< HEAD --- Use the 'MnistImageClassifier' function's output to filter frames --- This query returns the frame ids of the frames with digit 6 --- We limit to the first five frames containing digit 6 @@ -163,6 +164,7 @@ After registering ``MnistImageClassifier`` function, you can call the function i WHERE MnistImageClassifier(data).label = '6' LIMIT 5; <<<<<<< HEAD +<<<<<<< HEAD AI-Centric Query Optimization ----------------------------- @@ -172,6 +174,57 @@ EvaDB optimizes the AI queries to save money spent on running models and reduce <<<<<<< HEAD Query optimization has powered SQL database systems for several decades. It is the bridge that connects the declarative query language to efficient query execution on hardware. EvaDB accelerates AI queries using a collection of optimizations detailed in the :ref:`optimizations` page. ======= +======= +======= +Save time and money +---------------------- + +EvaDB automatically optimizes the queries to save inference cost and query execution time using its Cascades-style extensible query optimizer. EvaDB's optimizer is tailored for AI pipelines. The Cascades query optimization framework has worked well in SQL database systems for several decades. Query optimization in EvaDB is the bridge that connects the declarative query language to efficient execution. + +EvaDB accelerates AI pipelines using a collection of optimizations inspired by SQL database systems including function caching, sampling, and cost-based operator reordering. + +EvaDB supports an AI-oriented query language for analyzing both structured and unstructured data. Here are some illustrative apps: + + +The :ref:`Getting Started` page shows how you can use EvaDB for different AI tasks and how you can easily extend EvaDB to support your custom deep learning model through user-defined functions. + +The :ref:`User Guides` section contains Jupyter Notebooks that demonstrate how to use various features of EvaDB. Each notebook includes a link to Google Colab, where you can run the code yourself. + + + + +User-Defined Function (UDF) or Function +------------------------------------------ + +User-defined functions are thin wrappers around deep learning models. They +allow us to use deep learning models in AI queries. + +Here is an illustrative UDF for classifying MNIST images. + +.. code-block:: bash + + !wget -nc https://raw.githubusercontent.com/georgia-tech-db/evadb/master/evadb/udfs/mnist_image_classifier.py + +.. code-block:: python + + cursor.create_function("MnistImageClassifier", True, 'mnist_image_classifier.py') + response = cursor.df() + print(response) + +That's it! You can now use the newly registered UDF anywhere in the query -- in the ``select`` or ``filter`` calls. + +.. code-block:: python + + query = cursor.table("MNISTVideo") + query = query.filter("id = 30 OR id = 50 OR id = 70") + + # Here, we are selecting the output of the function + query = query.select("data, MnistImageClassifier(data).label") + response = query.df() + +.. code-block:: python + +>>>>>>> c63abee7 (release: merge staging into master (#1032)) query2 = cursor.table("MNISTVideo") # Here, we are also filtering based on the output of the function @@ -179,5 +232,8 @@ Query optimization has powered SQL database systems for several decades. It is t query2 = query2.select("data, MnistImageClassifier(data).label") response = query2.df() >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) diff --git a/docs/source/overview/connect-to-database.rst b/docs/source/overview/connect-to-database.rst index 45a27d96c0..fd00f694e5 100644 --- a/docs/source/overview/connect-to-database.rst +++ b/docs/source/overview/connect-to-database.rst @@ -6,6 +6,7 @@ EvaDB supports an extensive range of data sources for structured and unstructure Connect to a SQL Database System -------------------------------- +<<<<<<< HEAD 1. Use the ``CREATE DATABASE`` statement to connect to an existing SQL database. .. code-block:: @@ -19,11 +20,28 @@ Connect to a SQL Database System "port": "5432", "database": "restaurant_reviews" }; +======= +1. Use the `CREATE DATABASE` statement to connect to an existing SQL database. + +.. code-block:: python + + cursor.query(""" + CREATE DATABASE restaurant_reviews + WITH ENGINE = 'postgres', + PARAMETERS = { + "user": "eva", + "password": "password", + "host": "localhost", + "port": "5432", + "database": "restaurant_reviews" + };""").df() +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) .. note:: Go over the :ref:`CREATE DATABASE` statement for more details. The :ref:`Databases` page lists all the database systems that EvaDB currently supports. +<<<<<<< HEAD 2. Preview the Available Data Using ``SELECT`` You can now preview the available data in the ``restaurant_reviews`` database with a standard :ref:`SELECT` statement. @@ -45,6 +63,35 @@ You can also run native queries directly in the connected database system by the 'I ordered fried rice but it is too salty.' ) }; +======= +2. Preview the Available Data Using `SELECT` + +You can now preview the available data in the `restaurant_reviews` database with a standard :ref:`SELECT` statement. + +.. code-block:: python + + cursor.query(""" + SELECT * + FROM restaurant_reviews.food_review; + """).df() + +3. Run Native Queries in the Connected Database With `USE` + +You can also run native queries directly in the connected database system by the :ref:`USE` statement. + +.. code-block:: python + + cursor.query( + """ + USE restaurant_reviews { + INSERT INTO food_review (name, review) + VALUES ( + 'Customer 1', + 'I ordered fried rice but it is too salty.' + ) + }; + """).df() +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) Load Unstructured Data @@ -56,17 +103,35 @@ EvaDB supports diverse types of unstructured data. Here are some examples: You can load a collection of images obtained from Reddit from the local filesystem into EvaDB using the :ref:`LOAD` statement. +<<<<<<< HEAD .. code-block:: sql LOAD IMAGE 'reddit-images/*.jpg' INTO reddit_dataset; +======= +.. code-block:: python + + cursor.query(""" + LOAD IMAGE 'reddit-images/*.jpg' + INTO reddit_dataset; + """).df() +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) 2. Load Video from Cloud Bucket You can load a video from an S3 cloud bucket into EvaDB using the :ref:`LOAD` statement. +<<<<<<< HEAD .. code-block:: sql LOAD VIDEO 's3://bucket/eva_videos/mnist.mp4' INTO MNISTVid; +======= +.. code-block:: python + + cursor.query(""" + LOAD VIDEO 's3://bucket/eva_videos/mnist.mp4' + INTO MNISTVid; + """).df() +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) .. note:: diff --git a/docs/source/overview/getting-started.rst b/docs/source/overview/getting-started.rst index 3d7bcd0766..1699a7d8df 100644 --- a/docs/source/overview/getting-started.rst +++ b/docs/source/overview/getting-started.rst @@ -1,12 +1,18 @@ <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) .. _getting-started: ======= .. _Getting Started: >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= .. _getting-started: >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) Getting Started ================= @@ -26,6 +32,8 @@ To install EvaDB, we recommend using the `pip` package manager. .. code-block:: bash python -m venv evadb-venv +<<<<<<< HEAD +======= <<<<<<< HEAD .. warning:: @@ -123,17 +131,120 @@ The program runs a SQL query for listing all the built-in functions in EvaDB. It # Connect to EvaDB and get a database cursor for running queries cursor = evadb.connect().cursor() + # List all the built-in functions in EvaDB + print(cursor.query("SHOW UDFS;").df()) + +Now, run the Python program: + +.. code-block:: bash + + python -m run_evadb.py + +You should see a list of built-in functions including but not limited to the following: + +.. code-block:: bash + + name inputs ... impl metadata + 0 ArrayCount [Input_Array NDARRAY ANYTYPE (), Search_Key ANY] ... /home/jarulraj3/evadb/evadb/udfs/ndarray/array... [] + 1 Crop [Frame_Array NDARRAY UINT8 (3, None, None), bb... ... /home/jarulraj3/evadb/evadb/udfs/ndarray/crop.py [] + 2 ChatGPT [query NDARRAY STR (1,), content NDARRAY STR (... ... /home/jarulraj3/evadb/evadb/udfs/chatgpt.py [] + + [3 rows x 6 columns] + +.. note:: + Go over the :ref:`Python API` to learn more about `connect()` and `cursor`. +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) + +Now, activate the virtual environment: + +<<<<<<< HEAD +.. code-block:: bash + + source evadb-venv/bin/activate + +2. Once inside the virtual environment, run the command below to mitigate the dependency issues. + +.. code-block:: bash + + pip install --upgrade pip setuptools wheel + +3. Install EvaDB + +.. code-block:: bash + + pip install evadb + +4. Verify EvaDB installation + +.. code-block:: bash + + pip freeze + +You should see a list of installed packages including but not limited to the following: + +.. code-block:: bash + + Package Version + ----------------- ------- + aenum 3.1.15 + decorator 5.1.1 + diskcache 5.6.3 + evadb 0.3.3 + greenlet 2.0.2 + lark 1.1.7 + numpy 1.25.2 + pandas 2.1.0 + ... + +5. Run EvaDB + +Copy the following Python program to a file called `run_evadb.py`. + +The program runs a SQL query for listing all the built-in functions in EvaDB. It consists of importing and connecting to EvaDB, and then running the query. The query's result is returned as a Dataframe. +======= + EvaDB supports additional installation options for extending its functionality. Go over the :doc:`Installation Options ` for all the available options. + +Illustrative AI App +------------------- + +Here is a simple, illustrative `MNIST image classification `_ AI app in EvaDB. +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) + +.. code-block:: python + + # Import the EvaDB package + import evadb + +<<<<<<< HEAD + # Connect to EvaDB and get a database cursor for running queries + cursor = evadb.connect().cursor() + # List all the built-in functions in EvaDB print(cursor.query("SHOW FUNCTIONS;").df()) +======= + # Load the MNIST video into EvaDB + # Each frame in the loaded MNIST video contains a digit + cursor.load("mnist.mp4", "MNISTVid", format="video").df() + + # We now construct an AI pipeline to run the image classifier + # over all the digit images in the video +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) Now, run the Python program: +<<<<<<< HEAD .. code-block:: bash +======= + # Run the model on a subset of frames + # Here, id refers to the frame id + query = query.filter("id = 30 OR id = 50 OR id = 70 OR id = 0 OR id = 140") +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) python -m run_evadb.py You should see a list of built-in functions including but not limited to the following: +<<<<<<< HEAD .. code-block:: bash name inputs ... impl metadata @@ -254,4 +365,15 @@ Try out EvaDB by experimenting with the introductory `MNIST notebook on Colab `_. Try out EvaDB by experimenting with this introductory notebook. +<<<<<<< HEAD >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +======= +Try out EvaDB by experimenting with the introductory `MNIST notebook on Colab `_. + +.. image:: ../../images/reference/mnist.png + +.. note:: + Go over the :ref:`Python API` to learn more about the functions used in this app. +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) diff --git a/docs/source/overview/getting-started/installation-options.rst b/docs/source/overview/getting-started/installation-options.rst index 5d16956f0b..7129484552 100644 --- a/docs/source/overview/getting-started/installation-options.rst +++ b/docs/source/overview/getting-started/installation-options.rst @@ -68,9 +68,13 @@ EvaDB provides the following additional installation options for extending its f * ``pip install evadb[qdrant]`` for installing the Qdrant vector database system. It enables use-cases related to similarity search based on feature vectors. +<<<<<<< HEAD * ``pip install evadb[ludwig]`` for installing the Ludwig model training framework. It enables use-cases related to training and fine-tuning AI models. * ``pip install evadb[forecasting]`` for installing the statsforecast forecasting framework. It enables use-cases related to time series forecasting. +======= +* ``pip install evadb[ludwig]`` for installing the Ludwig model training framework. It enables use-cases related to training and fine-tunining AI models. +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) * ``pip install evadb[ray]`` for installing the Ray compute engine. It enables EvaDB to do more efficient query execution on CPUs and GPUs. diff --git a/docs/source/reference/api.rst b/docs/source/reference/api.rst index 4bf99029f7..28282f67fd 100644 --- a/docs/source/reference/api.rst +++ b/docs/source/reference/api.rst @@ -2,13 +2,19 @@ <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) Python API ======= Basic API >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= Python API >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) ========== To begin a querying session in EvaDB, obtain a connection with a cursor using ``connect`` and ``cursor`` functions. After getting the cursor, you can run queries with the ``query`` function in this manner: @@ -99,4 +105,51 @@ EvaDBQuery Interface ``cursor.query("...")`` only construct the query and not run the query. ``cursor.query("...").df()`` will both construct and run the query. +<<<<<<< HEAD >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +<<<<<<< HEAD +======= +EvaDBCursor Interface +--------------------- + +Using the cursor, you can refer to a table, load documents, create functions, create vector index, and several other tasks. + +After connecting to a table using ``table``, you can construct a complex query using the ``EvaDBQuery`` interface. + +.. autosummary:: + :toctree: ./doc + + ~evadb.EvaDBCursor.table + ~evadb.EvaDBCursor.load + ~evadb.EvaDBCursor.query + ~evadb.EvaDBCursor.create_function + ~evadb.EvaDBCursor.create_table + ~evadb.EvaDBCursor.create_vector_index + ~evadb.EvaDBCursor.drop_table + ~evadb.EvaDBCursor.drop_function + ~evadb.EvaDBCursor.drop_index + ~evadb.EvaDBCursor.df + ~evadb.EvaDBCursor.show + ~evadb.EvaDBCursor.insert + ~evadb.EvaDBCursor.explain + ~evadb.EvaDBCursor.rename + +EvaDBQuery Interface +--------------------- + +.. autosummary:: + :toctree: ./doc + + ~evadb.EvaDBQuery.select + ~evadb.EvaDBQuery.cross_apply + ~evadb.EvaDBQuery.filter + ~evadb.EvaDBQuery.df + ~evadb.EvaDBQuery.alias + ~evadb.EvaDBQuery.limit + ~evadb.EvaDBQuery.order + ~evadb.EvaDBQuery.show + ~evadb.EvaDBQuery.sql_query + ~evadb.EvaDBQuery.execute +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) diff --git a/docs/source/reference/evaql/create.rst b/docs/source/reference/evaql/create.rst index 1783977ef0..28d29c352d 100644 --- a/docs/source/reference/evaql/create.rst +++ b/docs/source/reference/evaql/create.rst @@ -49,8 +49,13 @@ To create a table, specify the schema of the table. object_id INTEGER ); +<<<<<<< HEAD CREATE INDEX ------------ +======= +CREATE UDF +---------- +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) The CREATE INDEX statement allows us to construct an EvaDB based index to accelerate semantic based searching. The index can be created on either a column of a table directly or outputs from a function running on a column of a table. @@ -99,8 +104,17 @@ To register an user-defined function, specify the implementation details of the TYPE Classification IMPL 'evadb/functions/fastrcnn_object_detector.py'; +<<<<<<< HEAD CREATE FUNCTION via Type ---------------------------- +======= +.. _create-udf-train: + +CREATE UDF via Training +----------------------- + +To register an user-defined function by training a predication model. +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) .. code-block:: sql diff --git a/docs/source/reference/evaql/drop.rst b/docs/source/reference/evaql/drop.rst index 8a84525df4..7bf67fca8b 100644 --- a/docs/source/reference/evaql/drop.rst +++ b/docs/source/reference/evaql/drop.rst @@ -9,8 +9,13 @@ DROP TABLE DROP TABLE DETRACVideo; +<<<<<<< HEAD DROP FUNCTION ------------- +======= +DROP UDF +-------- +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) .. code:: mysql diff --git a/docs/source/reference/evaql/select.rst b/docs/source/reference/evaql/select.rst index e63a2bbacd..3e3be6144d 100644 --- a/docs/source/reference/evaql/select.rst +++ b/docs/source/reference/evaql/select.rst @@ -37,6 +37,7 @@ Search for frames containing greater than 3 cars WHERE ArrayCount(FastRCNNObjectDetector(data).label, 'car') > 3 ORDER BY id; +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD SELECT TUPLES WITH MULTIPLE PREDICATES @@ -49,6 +50,14 @@ SELECT WITH MULTIPLE UDFS SELECT WITH MULTIPLE FUNCTIONS ------------------------------ >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +SELECT WITH MULTIPLE FUNCTIONS +------------------------------ +======= +SELECT WITH MULTIPLE UDFS +------------------------- +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) Compose multiple user-defined functions in a single query to construct semantically complex queries. diff --git a/docs/source/reference/evaql/show.rst b/docs/source/reference/evaql/show.rst index 41a46e17d0..4352f3abb3 100644 --- a/docs/source/reference/evaql/show.rst +++ b/docs/source/reference/evaql/show.rst @@ -1,8 +1,13 @@ SHOW ==== +<<<<<<< HEAD SHOW FUNCTIONS -------------- +======= +SHOW UDFS +--------- +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) List the registered user-defined functions diff --git a/docs/source/reference/evaql/udf.rst b/docs/source/reference/evaql/udf.rst index 883c0e7e8b..e27cef510b 100644 --- a/docs/source/reference/evaql/udf.rst +++ b/docs/source/reference/evaql/udf.rst @@ -1,10 +1,19 @@ :orphan: +<<<<<<< HEAD Functions ========= SHOW FUNCTIONS -------------- +======= + +UDF +=== + +SHOW UDFS +--------- +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) Here is a list of built-in user-defined functions in EvaDB. @@ -23,8 +32,13 @@ FastRCNNObjectDetector is a model for detecting objects. MVITActionRecognition i ArrayCount and Crop are utility functions for counting the number of objects in an array and cropping a bounding box from an image, respectively. +<<<<<<< HEAD SELECT WITH MULTIPLE FUNCTIONS ------------------------------ +======= +SELECT WITH MULTIPLE UDFS +------------------------- +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) Here is a query that illustrates how to use multiple functions in a single query. diff --git a/docs/source/reference/evaql/use.rst b/docs/source/reference/evaql/use.rst index ec98c6aef7..84e585c04b 100644 --- a/docs/source/reference/evaql/use.rst +++ b/docs/source/reference/evaql/use.rst @@ -11,12 +11,19 @@ The USE statement allows us to run arbitrary native queries in the connected dat USE === +<<<<<<< HEAD <<<<<<< HEAD The USE statement allows us to run arbitary native queries in the connected database. >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) ======= The USE statement allows us to run arbitrary native queries in the connected database. >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +The USE statement allows us to run arbitrary native queries in the connected database. +======= +The USE statement allows us to run arbitary native queries in the connected database. +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) .. code:: text @@ -25,13 +32,19 @@ The USE statement allows us to run arbitrary native queries in the connected dat * [database_connection] is an external database connection instanced by the `CREATE DATABASE statement`. <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) * [native_query] is an arbitrary SQL query supported by the [database_connection]. ======= * [native_query] is an arbitary SQL query supprted by the [database_connection]. >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= * [native_query] is an arbitrary SQL query supported by the [database_connection]. >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) .. warning:: diff --git a/docs/source/reference/udfs/model-train.rst b/docs/source/reference/udfs/model-train.rst new file mode 100644 index 0000000000..de6e84eeee --- /dev/null +++ b/docs/source/reference/udfs/model-train.rst @@ -0,0 +1,48 @@ +Training and Finetuning +======================== + +1. You can train a predication model easily in EvaDB + +.. note:: + + Install Ludwig in your EvaDB virtual environment: ``pip install evadb[ludwig]``. + +.. code-block:: sql + + CREATE UDF IF NOT EXISTS PredictHouseRent FROM + ( SELECT sqft, location, rental_price FROM HomeRentals ) + TYPE Ludwig + 'predict' 'rental_price' + 'time_limit' 120; + +In the above query, you are creating a new customized UDF by automatically training a model from the `HomeRentals` table. The `rental_price` column will be the target column for predication, while `sqft` and `location` are the inputs. + +You can also simply give all other columns in `HomeRentals` as inputs and let the underlying automl framework to figure it out. Below is an example query: + +.. code-block:: sql + + CREATE UDF IF NOT EXISTS PredictHouseRent FROM + ( SELECT * FROM HomeRentals ) + TYPE Ludwig + 'predict' 'rental_price' + 'time_limit' 120; + +.. note:: + + Check :ref:`create-udf-train` for available configurations for training models. + +2. After training completes, you can use the `PredictHouseRent` like all other UDFs in EvaDB + +.. code-block:: sql + + CREATE PredictHouseRent(sqft, location) FROM HomeRentals; + +You can also simply give all columns in `HomeRentals` as inputs for inference. The customized UDF with the underlying model can figure out the proper inference columns via the training columns. + +.. code-block:: sql + + CREATE PredictHouseRent(*) FROM HomeRentals; + +Check out our `Integration Tests `_ for working example. + + diff --git a/docs/source/reference/udfs/openai.rst b/docs/source/reference/udfs/openai.rst new file mode 100644 index 0000000000..d229eff44a --- /dev/null +++ b/docs/source/reference/udfs/openai.rst @@ -0,0 +1,27 @@ +OpenAI Models +===================== + +This section provides an overview of how you can use OpenAI models in EvaDB. + + +Chat Completion UDFs +-------------------- + +To create a chat completion UDF in EvaDB, use the following SQL command: + +.. code-block:: sql + + CREATE UDF IF NOT EXISTS OpenAIChatCompletion + IMPL 'evadb/udfs/openai_chat_completion_udf.py' + 'model' 'gpt-3.5-turbo' + +EvaDB supports the following models for chat completion task: + +- "gpt-4" +- "gpt-4-0314" +- "gpt-4-32k" +- "gpt-4-32k-0314" +- "gpt-3.5-turbo" +- "gpt-3.5-turbo-0301" + +The chat completion UDF can be composed in interesting ways with other UDFs. Please check the `Google Colab `_ for an example of combining chat completion task with caption extraction and video summarization models from Hugging Face and feeding it to chat completion to ask questions about the results. diff --git a/docs/source/usecases/food-review.rst b/docs/source/usecases/food-review.rst new file mode 100644 index 0000000000..8f688b0e83 --- /dev/null +++ b/docs/source/usecases/food-review.rst @@ -0,0 +1,147 @@ +ChatGPT + Postgres Tutorial +=========================== + +.. raw:: html + + + + + + +
+ Run on Google Colab + + View source on GitHub + + Download notebook +


+ + +In this tutorial, we demonstrate how to use EvaDB + ChatGPT to analyze the tone of food reviews stored in PostgreSQL. Then, based on the analysis, we further use +EvaDB + ChatGPT to address negative reviews by proposing a solution to the customer. + +For this use case, we assume user has a Postgres server running locally. You can also check our notebook above to skip Postgres setup. + +1. Connect to EvaDB +--------------------- + +.. code-block:: python + + import evadb + cursor = evadb.connect().cursor() + +2. Connect to an Existing Postgres Database +--------------------------------------------- + +.. tab-set:: + + .. tab-item:: Python + + .. code-block:: python + + params = { + "user": "eva", + "password": "password", + "host": "localhost", + "port": "5432", + "database": "evadb", + } + query = f"CREATE DATABASE postgres_data WITH ENGINE = 'postgres', PARAMETERS = {params};" + cursor.query(query).df() + + .. tab-item:: SQL + + .. code-block:: text + + CREATE DATABASE postgres_data WITH ENGINE = 'postgres', PARAMETERS = { + "user": "eva", + "password": "password", + "host": "localhost", + "port": "5432", + "database": "evadb" + } + +3. Sentiment Analysis of Food Review using ChatGPT +--------------------------------------------------- + +We then use EvaDB + ChatGPT to analyze whether the review is "positive" or "negative" with customized ChatGPT prompt. For this use case, +we assume reviews have been already loaded into the table inside PostgreSQL. +You can check our `Jupyter Notebook `__ for how to load data. + +.. tab-set:: + + .. tab-item:: Python + + .. code-block:: python + + cursor.query(""" + SELECT ChatGPT( + "Is the review positive or negative. Only reply 'positive' or 'negative'. Here are examples. The food is very bad: negative. The food is very good: postive.", + review) + FROM postgres_data.review_table; + """).df() + + .. tab-item:: SQL + + .. code-block:: sql + + SELECT ChatGPT( + "Is the review positive or negative. Only reply 'positive' or 'negative'. Here are examples. The food is very bad: negative. The food is very good: postive.", + review) + FROM postgres_data.review_table; + +This will return tone analysis results for existing reviews. + +.. code-block:: + + +------------------------------+ + | chatgpt.response | + |------------------------------| + | negative | + | positive | + | negative | + +------------------------------+ + +4. Response to Negative Reviews using ChatGPT +--------------------------------------------- + +.. tab-set:: + + .. tab-item:: Python + + .. code-block:: python + + cursor.query(""" + SELECT ChatGPT( + "Respond the the review with solution to address the review's concern", + review) + FROM postgres_data.review_table + WHERE ChatGPT( + "Is the review positive or negative. Only reply 'positive' or 'negative'. Here are examples. The food is very bad: negative. The food is very good: postive.", + review) = "negative"; + """).df() + + .. tab-item:: SQL + + .. code-block:: sql + + SELECT ChatGPT( + "Respond the the review with solution to address the review's concern", + review) + FROM postgres_data.review_table + WHERE ChatGPT( + "Is the review positive or negative. Only reply 'positive' or 'negative'. Here are examples. The food is very bad: negative. The food is very good: postive.", + review) = "negative"; + +This query will first filter out positive reviews and then apply ChatGPT again to create response to negative reviews. This will give results. + +.. code-block:: + + +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | chatgpt.response | + |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| + | Dear valued customer, Thank you for bringing this matter to our attention. We apologize for the inconvenience caused by the excessive saltiness of your fried rice. We understand how important it is to have a satisfying dining experience, and we would like to make it right for you ... | + | Dear [Customer's Name], Thank you for bringing this issue to our attention. We apologize for the inconvenience caused by the missing chicken sandwich in your takeout order. We understand how frustrating it can be when an item is missing from your meal. To address this concern, we ... | + +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +Check out our `Jupyter Notebook `__ for working example. \ No newline at end of file diff --git a/docs/source/usecases/image-classification.rst b/docs/source/usecases/image-classification.rst index 78f1e35882..2351b5d0c1 100644 --- a/docs/source/usecases/image-classification.rst +++ b/docs/source/usecases/image-classification.rst @@ -1,5 +1,6 @@ <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD .. _image-classification: ======= .. _image classification: @@ -79,12 +80,25 @@ In the following query, we call the classifier on every image in the video. The FROM mnist_video; ======= +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) .. _image-classification: +======= +.. _image classification: + +Image Classification Pipeline using EvaDB +========================================= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) Image Classification ==================== +<<<<<<< HEAD .. raw:: html +======= +1. Connect to EvaDB +------------------- +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) @@ -103,7 +117,12 @@ Image Classification Introduction ------------ +<<<<<<< HEAD In this tutorial, we present how to use ``PyTorch`` models in EvaDB to classify images. In particular, we focus on classifying images from the ``MNIST`` dataset that contains ``digits``. EvaDB makes it easy to do image classification using its built-in support for ``PyTorch`` models. +======= +2. Register Image Classification Model as a Function in SQL +----------------------------------------------------------- +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) In this tutorial, besides classifying images, we will also showcase a query where the model's output is used to retrieve images with the digit ``6``. @@ -111,8 +130,13 @@ In this tutorial, besides classifying images, we will also showcase a query wher We will assume that the input ``MNIST`` video is loaded into ``EvaDB``. To download the video and load it into ``EvaDB``, see the complete `image classification notebook on Colab `_. +<<<<<<< HEAD Create Image Classification Function ------------------------------------ +======= +3. Execute Image Classification through SQL +------------------------------------------- +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) To create a custom ``MnistImageClassifier`` function, use the ``CREATE FUNCTION`` statement. The code for the custom classification model is available `here `_. @@ -154,6 +178,9 @@ This query returns the label of all the images: | 4 | <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) +------------------------------+ ======= @@ -184,9 +211,12 @@ Like normal SQL, you can also specify conditions to filter out some frames of th SELECT MnistImageClassifier(data).label FROM mnist_video WHERE id < 2 >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= +------------------------------+ >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) Filtering Based on AI Function @@ -211,6 +241,7 @@ Now, the ``DataFrame`` only contains images of the digit ``6``. | 6 | +------------------------------+ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD .. include:: ../shared/footer.rst @@ -222,3 +253,9 @@ Check out our `Jupyter Notebook >>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +.. include:: ../shared/footer.rst +======= +Check out our `Jupyter Notebook `_ for working example. +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) diff --git a/docs/source/usecases/qa-video.rst b/docs/source/usecases/qa-video.rst new file mode 100644 index 0000000000..133eefca24 --- /dev/null +++ b/docs/source/usecases/qa-video.rst @@ -0,0 +1,89 @@ +Q&A Application on Videos +========================= + +1. Connect to EvaDB +------------------- + +.. code-block:: python + + import evadb + cursor = evadb.connect().cursor() + +2. Register Functions +--------------------- + +Register speech-to-text **whisper** model from `HuggingFace` + +.. code-block:: python + + cursor.query(""" + CREATE UDF SpeechRecognizer + TYPE HuggingFace + 'task' 'automatic-speech-recognition' + 'model' 'openai/whisper-base'; + """).execute() + +.. note:: + + EvaDB allows users to register any model in HuggingFace as a function. + +Register **OpenAI** LLM model + +.. code-block:: python + + cursor.query(""" + CREATE UDF ChatGPT + IMPL 'evadb/udfs/chatgpt.py' + """).execute() + + # Set OpenAI token + import os + os.environ["OPENAI_KEY"] = "sk-..." + +.. note:: + + ChatGPT function is a wrapper around OpenAI API call. You can also switch to other LLM models that can run locally. + +3. Summarize Video in Text +-------------------------- + +Create a table with text summary of the video. Text summarization is generated by running speech-to-text ``Whisper`` model from ``HuggingFace``. + +.. code-block:: python + + cursor.query(""" + CREATE TABLE text_summary AS + SELECT SpeechRecognizer(audio) FROM ukraine_video; + """).execute() + +This results a table shown below. + +.. code-block:: + + +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | text_summary.text | + |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| + | The war in Ukraine has been on for 415 days. Who is winning it? Not Russia. Certainly not Ukraine. It is the US oil companies. US oil companies have reached $200 billion in pure profits. The earnings are still on. They are still milking this war and sharing the spoils. Let us look at how Exxon mobile has been doing. In 2022, the company made $56 billion in profits. Oil companies capitalized on instability and they are profiting from pain. American oil companies are masters of this art. You may remember the war in Iraq. The US went to war in Iraq by selling a lie. The Americans did not find any weapons of mass destruction but they did find lots of oil. And in the year since, American officials have admitted this. And this story is not over. It's repeating itself in Ukraine. They are feeding another war and filling the coffers of US oil companies. | + +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + +4. Q&A using ChatGPT +--------------------- + +We can now embed the ChatGPT prompt inside SQL with text summary from the table as its knowledge base. + +.. code-block:: python + + cursor.query(""" + SELECT ChatGPT('Is this video summary related to Ukraine russia war', text) + FROM text_summary; + """).df() + +This query returns a projected ``DataFrame``. + +.. code-block:: + + +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + | chatgpt.response | + |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| + | Based on the provided context, it seems that the video summary is related to the Ukraine-Russia war. It discusses how US oil companies are allegedly profiting from the war in Ukraine, similar to how they allegedly benefited from the war in Iraq. | + +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/docs/source/usecases/similar-image-search.rst b/docs/source/usecases/similar-image-search.rst new file mode 100644 index 0000000000..91563cc2cd --- /dev/null +++ b/docs/source/usecases/similar-image-search.rst @@ -0,0 +1,83 @@ +Image Similarity Search Pipeline using EvaDB on Images +====================================================== + +In this use case, we want to search similar images based on an image provided by the user. To implement this use case, we leverage EvaDB's capability of easily expressing feature extraction pipeline. Additionally, we also leverage EvaDB's capability of building a similarity search index and searching the index to +locate similar images through ``FAISS`` library. + +For this use case, we use a reddit image dataset that can be downloaded from `Here `_. +We populate a table in the database that contains all images. + +1. Connect to EvaDB +------------------- + +.. code-block:: python + + import evadb + cursor = evadb.connect().cursor() + +2. Register SIFT as Function +---------------------------- + +.. code-block:: python + + cursor.query(""" + CREATE UDF IF NOT EXISTS SiftFeatureExtractor + IMPL 'evadb/udfs/sift_feature_extractor.py' + """).execute() + +3. Search Similar Images +------------------------ + +To locate images that have similar appearance, we will first build an index based on embeddings of images. +Then, for the given image, EvaDB can find similar images by searching in the index. + +Build Index using ``FAISS`` +*************************** + +The below query creates a new index on the projected column ``SiftFeatureExtractor(data)`` from the ``reddit_dataset`` table. + +.. code-block:: python + + cursor.query(""" + CREATE INDEX reddit_sift_image_index + ON reddit_dataset (SiftFeatureExtractor(data)) + USING FAISS + """).execute() + +Search Index for a Given Image +******************************* + +EvaDB leverages the ``ORDER BY ... LIMIT ...`` SQL syntax to retrieve the top 5 similar images. +In this example, ``Similarity(x, y)`` is a built-in function to calculate distance between ``x`` and ``y``. +In current version, ``x`` is a single tuple and ``y`` is a column that contains multiple tuples. +By default EvaDB does pairwise distance calculation between ``x`` and all tuples from ``y``. +In this case, EvaDB leverages the index that we have already built. + +.. code-block:: python + + query = cursor.query(""" + SELECT name FROM reddit_dataset ORDER BY + Similarity( + SiftFeatureExtractor(Open('reddit-images/g1074_d4mxztt.jpg')), + SiftFeatureExtractor(data) + ) + LIMIT 5 + """) + query.df() + +The ``DataFrame`` contains the top 5 similar images. + +.. code-block:: + + +---------------------------------+ + | reddit_dataset.name | + |---------------------------------| + | reddit-images/g1074_d4mxztt.jpg | + | reddit-images/g348_d7ju7dq.jpg | + | reddit-images/g1209_ct6bf1n.jpg | + | reddit-images/g1190_cln9xzr.jpg | + | reddit-images/g1190_clna2x2.jpg | + +---------------------------------+ + +Check out our `Jupyter Notebook `_ for working example. +We also demonstrate more complicated features of EvaDB for similarity search. diff --git a/evadb/binder/binder_utils.py b/evadb/binder/binder_utils.py index 4e46e9ed67..efc80d224e 100644 --- a/evadb/binder/binder_utils.py +++ b/evadb/binder/binder_utils.py @@ -59,6 +59,9 @@ def check_data_source_and_table_are_valid( """ <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) error = None if catalog.get_database_catalog_entry(database_name) is None: ======= @@ -90,10 +93,13 @@ def check_data_source_and_table_are_valid( raise BinderError(error) else: >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= error = None if catalog.get_database_catalog_entry(database_name) is None: >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) error = "{} data source does not exist. Create the new database source using CREATE DATABASE.".format( database_name, ) @@ -112,13 +118,19 @@ def check_data_source_and_table_are_valid( def create_table_catalog_entry_for_data_source( <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) table_name: str, database_name: str, column_info: pd.DataFrame ======= table_name: str, column_info: pd.DataFrame >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= table_name: str, database_name: str, column_info: pd.DataFrame >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) ): column_name_list = list(column_info["name"]) column_type_list = [ @@ -129,13 +141,19 @@ def create_table_catalog_entry_for_data_source( for name, dtype in zip(column_name_list, column_type_list): <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) column_list.append(ColumnCatalogEntry(name.lower(), dtype)) ======= column_list.append(ColumnCatalogEntry(name, dtype)) >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= column_list.append(ColumnCatalogEntry(name.lower(), dtype)) >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) # Assemble table. table_catalog_entry = TableCatalogEntry( @@ -173,8 +191,11 @@ def bind_native_table_info(catalog: CatalogManager, table_info: TableInfo): db_catalog_entry = catalog.get_database_catalog_entry(table_info.database_name) <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) with get_database_handler( db_catalog_entry.engine, **db_catalog_entry.params ) as handler: @@ -184,6 +205,9 @@ def bind_native_table_info(catalog: CatalogManager, table_info: TableInfo): table_info.table_name, table_info.database_name, column_df ) <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) ======= handler = get_database_handler(db_catalog_entry.engine, **db_catalog_entry.params) handler.connect() @@ -194,8 +218,11 @@ def bind_native_table_info(catalog: CatalogManager, table_info: TableInfo): table_info.table_name, column_df ) >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) def bind_evadb_table_info(catalog: CatalogManager, table_info: TableInfo): diff --git a/evadb/binder/statement_binder_context.py b/evadb/binder/statement_binder_context.py index cc49d2c4f6..839e74c91e 100644 --- a/evadb/binder/statement_binder_context.py +++ b/evadb/binder/statement_binder_context.py @@ -91,8 +91,11 @@ def add_table_alias(self, alias: str, database_name: str, table_name: str): db_catalog_entry.engine, **db_catalog_entry.params <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) ) as handler: # Assemble columns. response = handler.get_columns(table_name) @@ -103,6 +106,9 @@ def add_table_alias(self, alias: str, database_name: str, table_name: str): table_name, database_name, column_df ) <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) ======= ) handler.connect() @@ -113,8 +119,11 @@ def add_table_alias(self, alias: str, database_name: str, table_name: str): table_name, column_df ) >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) else: table_obj = self._catalog().get_table_catalog_entry(table_name) diff --git a/evadb/executor/create_database_executor.py b/evadb/executor/create_database_executor.py index 793a9ce078..fa19acdfc9 100644 --- a/evadb/executor/create_database_executor.py +++ b/evadb/executor/create_database_executor.py @@ -30,8 +30,11 @@ def __init__(self, db: EvaDBDatabase, node: CreateDatabaseStatement): def exec(self, *args, **kwargs): <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) # Check if database already exists. db_catalog_entry = self.catalog().get_database_catalog_entry( self.node.database_name @@ -45,11 +48,17 @@ def exec(self, *args, **kwargs): else: raise ExecutorError(f"{self.node.database_name} already exists.") <<<<<<< HEAD +<<<<<<< HEAD ======= # TODO: handle if_not_exists >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= + # TODO: handle if_not_exists +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) logger.debug( f"Trying to connect to the provided engine {self.node.engine} with params {self.node.param_dict}" @@ -57,6 +66,9 @@ def exec(self, *args, **kwargs): <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) # Check the validity of database entry. with get_database_handler(self.node.engine, **self.node.param_dict): pass @@ -74,11 +86,14 @@ def exec(self, *args, **kwargs): if not resp.status: raise ExecutorError(f"Cannot establish connection due to {resp.error}") >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= # Check the validity of database entry. with get_database_handler(self.node.engine, **self.node.param_dict): pass >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) logger.debug(f"Creating database {self.node}") self.catalog().insert_database_catalog_entry( diff --git a/evadb/executor/create_executor.py b/evadb/executor/create_executor.py index 2d46bf6b78..c416c5bf95 100644 --- a/evadb/executor/create_executor.py +++ b/evadb/executor/create_executor.py @@ -35,6 +35,7 @@ def __init__(self, db: EvaDBDatabase, node: CreatePlan): def exec(self, *args, **kwargs): <<<<<<< HEAD +<<<<<<< HEAD <<<<<<< HEAD # create a table in the active database if set is_native_table = self.node.table_info.database_name is not None @@ -49,6 +50,17 @@ def exec(self, *args, **kwargs): # create a table in the ative database if set is_native_table = self.node.table_info.database_name is not None >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= + # create a table in the ative database if set + is_native_table = self.node.table_info.database_name is not None +======= + if not handle_if_not_exists( + self.catalog(), self.node.table_info, self.node.if_not_exists + ): + create_table_done = False + logger.debug(f"Creating table {self.node.table_info}") +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) check_if_exists = handle_if_not_exists( self.catalog(), self.node.table_info, self.node.if_not_exists @@ -67,8 +79,11 @@ def exec(self, *args, **kwargs): ) <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) else: catalog_entry = create_table_catalog_entry_for_native_table( self.node.table_info, self.node.column_list @@ -122,6 +137,7 @@ def exec(self, *args, **kwargs): len(self.children) ) child = self.children[0] +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) @@ -164,3 +180,16 @@ def exec(self, *args, **kwargs): self.catalog().delete_table_catalog_entry(catalog_entry) raise e >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= + + # Populate the table + for batch in child.exec(): + batch.drop_column_alias() + storage_engine.write(catalog_entry, batch) + except Exception as e: + # rollback if the create call fails + if create_table_done: + storage_engine.drop(catalog_entry) + raise e +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) diff --git a/evadb/executor/create_udf_executor.py b/evadb/executor/create_udf_executor.py index 7d68aab5f3..97c937c5b7 100644 --- a/evadb/executor/create_udf_executor.py +++ b/evadb/executor/create_udf_executor.py @@ -12,9 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +<<<<<<< HEAD import hashlib import os import pickle +======= +import os +>>>>>>> c63abee7 (release: merge staging into master (#1032)) from pathlib import Path from typing import Dict, List @@ -37,7 +41,10 @@ from evadb.utils.errors import UDFIODefinitionError from evadb.utils.generic_utils import ( load_udf_class_from_file, +<<<<<<< HEAD try_to_import_forecast, +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) try_to_import_ludwig, try_to_import_torch, try_to_import_ultralytics, @@ -72,7 +79,11 @@ def handle_huggingface_udf(self): def handle_ludwig_udf(self): """Handle ludwig UDFs +<<<<<<< HEAD Use Ludwig's auto_train engine to train/tune models. +======= + Use ludwig's auto_train engine to train/tune models. +>>>>>>> c63abee7 (release: merge staging into master (#1032)) """ try_to_import_ludwig() from ludwig.automl import auto_train @@ -131,6 +142,7 @@ def handle_ultralytics_udf(self): self.node.metadata, ) +<<<<<<< HEAD def handle_forecasting_udf(self): """Handle forecasting UDFs""" aggregated_batch_list = [] @@ -244,6 +256,8 @@ def handle_forecasting_udf(self): metadata_here, ) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) def handle_generic_udf(self): """Handle generic UDFs @@ -284,8 +298,11 @@ def exec(self, *args, **kwargs): name, impl_path, udf_type, io_list, metadata = self.handle_ultralytics_udf() elif self.node.udf_type == "Ludwig": name, impl_path, udf_type, io_list, metadata = self.handle_ludwig_udf() +<<<<<<< HEAD elif self.node.udf_type == "Forecasting": name, impl_path, udf_type, io_list, metadata = self.handle_forecasting_udf() +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) else: name, impl_path, udf_type, io_list, metadata = self.handle_generic_udf() diff --git a/evadb/executor/use_executor.py b/evadb/executor/use_executor.py index 2ebe6dea45..371d9c0db1 100644 --- a/evadb/executor/use_executor.py +++ b/evadb/executor/use_executor.py @@ -41,6 +41,9 @@ def exec(self, *args, **kwargs) -> Iterator[Batch]: ======= <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) handler = get_database_handler( db_catalog_entry.engine, **db_catalog_entry.params, diff --git a/evadb/third_party/databases/interface.py b/evadb/third_party/databases/interface.py index 2398f23e4c..6f22c98402 100644 --- a/evadb/third_party/databases/interface.py +++ b/evadb/third_party/databases/interface.py @@ -18,11 +18,17 @@ <<<<<<< HEAD ======= +<<<<<<< HEAD from evadb.executor.executor_utils import ExecutorError >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +<<<<<<< HEAD <<<<<<< HEAD from evadb.executor.executor_utils import ExecutorError +======= +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) <<<<<<< HEAD ======= @@ -52,6 +58,7 @@ def _get_database_handler(engine: str, **kwargs): elif engine == "mysql": return mod.MysqlHandler(engine, **kwargs) <<<<<<< HEAD +<<<<<<< HEAD <<<<<<< HEAD elif engine == "mariadb": return mod.MariaDbHandler(engine, **kwargs) @@ -67,14 +74,23 @@ def _get_database_handler(engine: str, **kwargs): elif engine == "github": return mod.GithubHandler(engine, **kwargs) >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= + elif engine == "mariadb": + return mod.MariaDbHandler(engine, **kwargs) +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) else: raise NotImplementedError(f"Engine {engine} is not supported") +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) @contextmanager def get_database_handler(engine: str, **kwargs): handler = _get_database_handler(engine, **kwargs) @@ -87,11 +103,16 @@ def get_database_handler(engine: str, **kwargs): handler.disconnect() +<<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) def dynamic_import(handler_dir): import_path = f"evadb.third_party.databases.{handler_dir}.{handler_dir}_handler" return importlib.import_module(import_path) diff --git a/evadb/third_party/databases/mysql/mysql_handler.py b/evadb/third_party/databases/mysql/mysql_handler.py index 966b66f56f..3e121fe79e 100644 --- a/evadb/third_party/databases/mysql/mysql_handler.py +++ b/evadb/third_party/databases/mysql/mysql_handler.py @@ -51,16 +51,22 @@ def disconnect(self): <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) def get_sqlalchmey_uri(self) -> str: return f"mysql+mysqlconnector://{self.user}:{self.password}@{self.host}:{self.port}/{self.database}" ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= def get_sqlalchmey_uri(self) -> str: return f"mysql+mysqlconnector://{self.user}:{self.password}@{self.host}:{self.port}/{self.database}" >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) def check_connection(self) -> DBHandlerStatus: if self.connection: return DBHandlerStatus(status=True) @@ -107,17 +113,23 @@ def _fetch_results_as_df(self, cursor): return pd.DataFrame({"status": ["success"]}) <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) res_df = pd.DataFrame( res, columns=[desc[0].lower() for desc in cursor.description] ) ======= res_df = pd.DataFrame(res, columns=[desc[0] for desc in cursor.description]) >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= res_df = pd.DataFrame( res, columns=[desc[0].lower() for desc in cursor.description] ) >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) return res_df except mysql.connector.ProgrammingError as e: if str(e) == "no results to fetch": diff --git a/evadb/third_party/databases/postgres/postgres_handler.py b/evadb/third_party/databases/postgres/postgres_handler.py index ade56a8783..bfa1a89f56 100644 --- a/evadb/third_party/databases/postgres/postgres_handler.py +++ b/evadb/third_party/databases/postgres/postgres_handler.py @@ -109,6 +109,9 @@ def get_columns(self, table_name: str) -> DBHandlerResponse: try: <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) query = f"SELECT column_name as name, data_type as dtype, udt_name FROM information_schema.columns WHERE table_name='{table_name}'" columns_df = pd.read_sql_query(query, self.connection) columns_df["dtype"] = columns_df.apply( @@ -119,6 +122,7 @@ def get_columns(self, table_name: str) -> DBHandlerResponse: columns_df = pd.read_sql_query(query, self.connection) columns_df["dtype"] = columns_df["dtype"].apply(self._pg_to_python_types) >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= query = f"SELECT column_name as name, data_type as dtype, udt_name FROM information_schema.columns WHERE table_name='{table_name}'" columns_df = pd.read_sql_query(query, self.connection) @@ -126,6 +130,8 @@ def get_columns(self, table_name: str) -> DBHandlerResponse: lambda x: self._pg_to_python_types(x["dtype"], x["udt_name"]), axis=1 ) >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) return DBHandlerResponse(data=columns_df) except psycopg2.Error as e: return DBHandlerResponse(data=None, error=str(e)) @@ -173,16 +179,22 @@ def execute_native_query(self, query_string: str) -> DBHandlerResponse: <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) def _pg_to_python_types(self, pg_type: str, udt_name: str): primitive_type_mapping = { ======= def _pg_to_python_types(self, pg_type: str): mapping = { >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= def _pg_to_python_types(self, pg_type: str, udt_name: str): primitive_type_mapping = { >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) "integer": int, "bigint": int, "smallint": int, @@ -196,10 +208,13 @@ def _pg_to_python_types(self, pg_type: str, udt_name: str): # Add more mappings as needed } +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) user_defined_type_mapping = { "vector": np.ndarray # Handle user defined types constructed by Postgres extension. @@ -210,12 +225,18 @@ def _pg_to_python_types(self, pg_type: str, udt_name: str): elif pg_type == "USER-DEFINED" and udt_name in user_defined_type_mapping: return user_defined_type_mapping[udt_name] <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) ======= if pg_type in mapping: return mapping[pg_type] >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) else: raise Exception( f"Unsupported column {pg_type} encountered in the postgres table. Please raise a feature request!" diff --git a/evadb/third_party/databases/sqlite/sqlite_handler.py b/evadb/third_party/databases/sqlite/sqlite_handler.py index e8b776f3e2..85ef0975a0 100644 --- a/evadb/third_party/databases/sqlite/sqlite_handler.py +++ b/evadb/third_party/databases/sqlite/sqlite_handler.py @@ -14,12 +14,18 @@ # limitations under the License. <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD import datetime ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) ======= import datetime >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +import datetime +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) import sqlite3 import pandas as pd @@ -66,16 +72,22 @@ def disconnect(self): <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) def get_sqlalchmey_uri(self) -> str: return f"sqlite:///{self.database}" ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= def get_sqlalchmey_uri(self) -> str: return f"sqlite:///{self.database}" >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) def check_connection(self) -> DBHandlerStatus: """ Check connection to the handler. @@ -124,17 +136,25 @@ def get_columns(self, table_name: str) -> DBHandlerResponse: columns_df.rename(columns={"type": "dtype"}, inplace=True) <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) columns_df["dtype"] = columns_df["dtype"].apply( self._sqlite_to_python_types ) +<<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) return DBHandlerResponse(data=columns_df) except sqlite3.Error as e: return DBHandlerResponse(data=None, error=str(e)) @@ -143,8 +163,11 @@ def _fetch_results_as_df(self, cursor): try: <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) # Handling case-sensitive databases like SQLite can be tricky. Currently, # EvaDB converts all columns to lowercase, which may result in issues with # these databases. As we move forward, we are actively working on improving @@ -162,12 +185,15 @@ def _fetch_results_as_df(self, cursor): res, columns=[desc[0] for desc in cursor.description] >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= res = cursor.fetchall() res_df = pd.DataFrame( res, columns=[desc[0].lower() for desc in cursor.description] >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) if cursor.description else [], ) @@ -195,8 +221,11 @@ def execute_native_query(self, query_string: str) -> DBHandlerResponse: return DBHandlerResponse(data=None, error=str(e)) <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) def _sqlite_to_python_types(self, sqlite_type: str): mapping = { @@ -238,7 +267,12 @@ def _sqlite_to_python_types(self, sqlite_type: str): f"Unsupported column {sqlite_type} encountered in the sqlite table. Please raise a feature request!" ) <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) diff --git a/evadb/version.py b/evadb/version.py index dfaac7e631..6b4d12b43f 100644 --- a/evadb/version.py +++ b/evadb/version.py @@ -4,6 +4,7 @@ <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD _REVISION = "8" ======= _REVISION = "3" @@ -15,11 +16,19 @@ _REVISION = "8+dev" >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) ======= +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) _REVISION = "7" ======= _REVISION = "3" >>>>>>> 9c0af83d ([RELEASE]: v0.3.3 (#984)) +<<<<<<< HEAD >>>>>>> d562016d ([RELEASE]: v0.3.3 (#984)) +======= +======= +_REVISION = "4+dev" +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) VERSION_SHORT = f"{_MAJOR}.{_MINOR}" VERSION = f"{_MAJOR}.{_MINOR}.{_REVISION}" diff --git a/script/formatting/spelling.txt b/script/formatting/spelling.txt index 8235e9dc5d..f7ba8e9ba6 100644 --- a/script/formatting/spelling.txt +++ b/script/formatting/spelling.txt @@ -1,5 +1,6 @@ <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD personal_ws-1.1 en 1776 ======= personal_ws-1.1 en 1467 @@ -7,6 +8,12 @@ personal_ws-1.1 en 1467 ======= personal_ws-1.1 en 1563 >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +personal_ws-1.1 en 1563 +======= +personal_ws-1.1 en 1467 +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) ABCD ABCMeta ANYDIM @@ -613,6 +620,7 @@ MyVideos MydbHandler <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD MysqlHandler NBEATS NCHAR @@ -621,6 +629,11 @@ NCHAR ======= MysqlHandler >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +MysqlHandler +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) NEQ NHITS NLP @@ -803,14 +816,20 @@ SampleExecutorTest SamplePlan <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) SampleTable SampleVideoTable ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= SampleTable SampleVideoTable >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) Scalability ScanPlan SchemaUtils @@ -1296,12 +1315,18 @@ featureextractor feedstock <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD fetchall ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) ======= fetchall >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +fetchall +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) ffill ffmpeg fileFormat @@ -1695,6 +1720,7 @@ singledispatchmethod sk <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD sklearn smallint ======= @@ -1702,6 +1728,11 @@ smallint ======= smallint >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +smallint +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) smi softmax spacy diff --git a/script/test/test.sh b/script/test/test.sh index 49550b9640..96a76df90b 100644 --- a/script/test/test.sh +++ b/script/test/test.sh @@ -95,6 +95,7 @@ long_integration_test() { notebook_test() { <<<<<<< HEAD +<<<<<<< HEAD <<<<<<< HEAD PYTHONPATH=./ python -m pytest --durations=5 --nbmake --overwrite "./tutorials" --capture=sys --tb=short -v --log-level=WARNING --nbmake-timeout=3000 --ignore="tutorials/08-chatgpt.ipynb" --ignore="tutorials/14-food-review-tone-analysis-and-response.ipynb" --ignore="tutorials/15-AI-powered-join.ipynb" --ignore="tutorials/16-homesale-forecasting.ipynb" --ignore="tutorials/17-home-rental-prediction.ipynb" --ignore="tutorials/18-stable-diffusion.ipynb" ======= @@ -103,6 +104,12 @@ notebook_test() { ======= PYTHONPATH=./ python -m pytest --durations=5 --nbmake --overwrite "./tutorials" --capture=sys --tb=short -v --log-level=WARNING --nbmake-timeout=3000 --ignore="tutorials/08-chatgpt.ipynb" --ignore="tutorials/14-food-review-tone-analysis-and-response.ipynb" --ignore="tutorials/15-AI-powered-join.ipynb" --ignore="tutorials/16-homesale-forecasting.ipynb" --ignore="tutorials/17-home-rental-prediction.ipynb" >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= + PYTHONPATH=./ python -m pytest --durations=5 --nbmake --overwrite "./tutorials" --capture=sys --tb=short -v --log-level=WARNING --nbmake-timeout=3000 --ignore="tutorials/08-chatgpt.ipynb" --ignore="tutorials/14-food-review-tone-analysis-and-response.ipynb" --ignore="tutorials/15-AI-powered-join.ipynb" --ignore="tutorials/16-homesale-forecasting.ipynb" --ignore="tutorials/17-home-rental-prediction.ipynb" +======= + PYTHONPATH=./ python -m pytest --durations=5 --nbmake --overwrite "./tutorials" --capture=sys --tb=short -v --log-level=WARNING --nbmake-timeout=3000 --ignore="tutorials/08-chatgpt.ipynb" --ignore="tutorials/14-food-review-tone-analysis-and-response.ipynb" +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) code=$? print_error_code $code "NOTEBOOK TEST" } diff --git a/setup.py b/setup.py index c5f6211d7b..930159f90b 100644 --- a/setup.py +++ b/setup.py @@ -108,16 +108,21 @@ def read(path, encoding="utf-8"): qdrant_libs = ["qdrant_client"] # cannot install on 3.11 due to grcpio <<<<<<< HEAD +<<<<<<< HEAD pinecone_libs = ["pinecone-client"] chromadb_libs = ["chromadb"] ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) pinecone_libs = ["pinecone-client"] chromadb_libs = ["chromadb"] +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) postgres_libs = [ "psycopg2", @@ -125,6 +130,7 @@ def read(path, encoding="utf-8"): ludwig_libs = ["ludwig[hyperopt,distributed]"] # MODEL TRAIN AND FINE TUNING <<<<<<< HEAD +<<<<<<< HEAD sklearn_libs = ["scikit-learn"] @@ -140,6 +146,8 @@ def read(path, encoding="utf-8"): ] ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) sklearn_libs = ["scikit-learn"] @@ -147,6 +155,8 @@ def read(path, encoding="utf-8"): "statsforecast", # MODEL TRAIN AND FINE TUNING "neuralforecast" # MODEL TRAIN AND FINE TUNING ] +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) ### NEEDED FOR DEVELOPER TESTING ONLY diff --git a/test/integration_tests/short/test_create_database_executor.py b/test/integration_tests/short/test_create_database_executor.py index 14392d9f6f..abae7602d8 100644 --- a/test/integration_tests/short/test_create_database_executor.py +++ b/test/integration_tests/short/test_create_database_executor.py @@ -18,6 +18,7 @@ from mock import patch +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD from evadb.executor.executor_utils import ExecutorError @@ -26,6 +27,11 @@ ======= from evadb.executor.executor_utils import ExecutorError >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +from evadb.executor.executor_utils import ExecutorError +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) from evadb.server.command_handler import execute_query_fetch_all diff --git a/test/third_party_tests/test_native_executor.py b/test/third_party_tests/test_native_executor.py index 822cfbc6de..502af25548 100644 --- a/test/third_party_tests/test_native_executor.py +++ b/test/third_party_tests/test_native_executor.py @@ -138,6 +138,25 @@ def _create_native_table_using_select_query(self): self.assertEqual(res_batch.frames["derived_table.name"][1], "bb") self.assertEqual(res_batch.frames["derived_table.age"][1], 2) + def _create_evadb_table_using_select_query(self): + execute_query_fetch_all( + self.evadb, + """CREATE TABLE eva_table AS SELECT name, age FROM test_data_source.test_table;""", + ) + + # check if the create table is successful + res_batch = execute_query_fetch_all(self.evadb, "Select * from eva_table") + self.assertEqual(len(res_batch), 2) + self.assertEqual(res_batch.frames["eva_table.name"][0], "aa") + self.assertEqual(res_batch.frames["eva_table.age"][0], 1) + self.assertEqual(res_batch.frames["eva_table.name"][1], "bb") + self.assertEqual(res_batch.frames["eva_table.age"][1], 2) + + execute_query_fetch_all( + self.evadb, + "DROP TABLE IF EXISTS eva_table;", + ) + def _execute_evadb_query(self): self._create_table_in_native_database() self._insert_value_into_native_database("aa", 1, "aaaa") @@ -155,6 +174,7 @@ def _execute_evadb_query(self): self._create_evadb_table_using_select_query() <<<<<<< HEAD +<<<<<<< HEAD <<<<<<< HEAD self._create_native_table_using_select_query() ======= @@ -162,6 +182,11 @@ def _execute_evadb_query(self): ======= self._create_native_table_using_select_query() >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= + self._create_native_table_using_select_query() +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) self._drop_table_in_native_database() self._drop_table_in_evadb_database() @@ -233,6 +258,7 @@ def test_should_run_query_in_postgres(self): self._raise_error_on_multiple_creation() self._raise_error_on_invalid_connection() +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD def test_should_run_query_in_mariadb(self): @@ -284,6 +310,8 @@ def test_should_run_query_in_sqlite(self): "database": "evadb.db", >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) ======= +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) def test_should_run_query_in_mariadb(self): # Create database. params = { @@ -308,7 +336,16 @@ def test_should_run_query_in_sqlite(self): params = { "database": f"{current_file_dir}/evadb.db", +<<<<<<< HEAD >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= + def test_should_run_query_in_sqlite(self): + # Create database. + params = { + "database": "evadb.db", +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +>>>>>>> c63abee7 (release: merge staging into master (#1032)) } query = f"""CREATE DATABASE test_data_source WITH ENGINE = "sqlite", diff --git a/test/unit_tests/storage/test_sqlite_native_storage_engine.py b/test/unit_tests/storage/test_sqlite_native_storage_engine.py index 7d84982a68..52157bfafa 100644 --- a/test/unit_tests/storage/test_sqlite_native_storage_engine.py +++ b/test/unit_tests/storage/test_sqlite_native_storage_engine.py @@ -32,13 +32,19 @@ def __init__(self): @pytest.mark.notparallel <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) class SQLiteNativeStorageEngineTest(unittest.TestCase): ======= class SQLiiteNativeStorageEngineTest(unittest.TestCase): >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD ======= class SQLiteNativeStorageEngineTest(unittest.TestCase): >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +>>>>>>> c63abee7 (release: merge staging into master (#1032)) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs)