From 03a6c55581cfb26cc1a540423922837f294aec83 Mon Sep 17 00:00:00 2001 From: Jiashen Cao Date: Wed, 6 Sep 2023 00:40:48 -0400 Subject: [PATCH] feat: sync master staging (#1050) Co-authored-by: Joy Arulraj Co-authored-by: Jiashen Cao Co-authored-by: Andy Xu Co-authored-by: Sayan Sinha Co-authored-by: Hersh Dhillon --- README.md | 34 +++- docs/_toc.yml | 54 +++++-- docs/conf.py | 5 + docs/source/benchmarks/text_summarization.rst | 117 +++++++++++++- docs/source/overview/concepts.rst | 15 ++ docs/source/overview/getting-started.rst | 40 ++++- .../getting-started/installation-options.rst | 4 + docs/source/reference/ai/custom.rst | 4 + .../source/reference/ai/model-forecasting.rst | 21 +++ docs/source/reference/api.rst | 17 ++ docs/source/reference/evaql/create.rst | 19 ++- docs/source/reference/evaql/drop.rst | 10 +- docs/source/reference/evaql/select.rst | 10 ++ docs/source/reference/evaql/show.rst | 5 + docs/source/reference/evaql/udf.rst | 14 +- docs/source/reference/evaql/use.rst | 18 +++ docs/source/reference/udfs/model-train.rst | 48 ------ docs/source/reference/udfs/openai.rst | 27 ---- docs/source/shared/postgresql.rst | 39 +++++ docs/source/usecases/emotion-analysis.rst | 18 +++ docs/source/usecases/food-review.rst | 147 ------------------ docs/source/usecases/image-classification.rst | 82 ++++++++++ docs/source/usecases/object-detection.rst | 29 ++++ docs/source/usecases/qa-video.rst | 89 ----------- docs/source/usecases/question-answering.rst | 18 +++ docs/source/usecases/sentiment-analysis.rst | 9 ++ docs/source/usecases/similar-image-search.rst | 83 ---------- docs/source/usecases/text-summarization.rst | 9 ++ evadb/binder/statement_binder.py | 50 ++++++ evadb/executor/create_function_executor.py | 127 +++++++++++++++ evadb/executor/create_udf_executor.py | 25 +++ evadb/executor/drop_object_executor.py | 5 + evadb/functions/chatgpt.py | 6 + evadb/functions/forecast.py | 48 ++++++ evadb/functions/function_bootstrap_queries.py | 20 +++ evadb/optimizer/operators.py | 12 ++ evadb/optimizer/optimizer_utils.py | 9 ++ evadb/optimizer/rules/rules.py | 19 +++ evadb/optimizer/statement_to_opr_converter.py | 6 + evadb/parser/create_function_statement.py | 9 ++ evadb/parser/evadb.lark | 15 ++ .../parser/lark_visitor/_create_statements.py | 7 + evadb/parser/lark_visitor/_drop_statement.py | 8 + evadb/parser/lark_visitor/_functions.py | 24 +++ evadb/plan_nodes/create_function_plan.py | 12 ++ .../third_party/databases/mariadb/__init__.py | 4 + evadb/utils/generic_utils.py | 3 + script/formatting/spelling.txt | 39 +++++ setup.py | 15 ++ .../long/test_function_executor.py | 8 + .../long/test_model_forecasting.py | 38 +++++ .../long/test_model_train.py | 6 + test/integration_tests/long/test_reuse.py | 8 + .../short/test_drop_executor.py | 11 ++ .../short/test_select_executor.py | 6 + test/markers.py | 6 + .../binder/test_statement_binder.py | 20 +++ .../executor/test_create_udf_executor.py | 18 +++ .../unit_tests/executor/test_plan_executor.py | 6 + .../test_statement_to_opr_converter.py | 14 ++ test/unit_tests/parser/test_parser.py | 24 +++ test/unit_tests/plan_nodes/test_plan.py | 12 ++ .../test_sqlite_native_storage_engine.py | 9 ++ test/util.py | 9 ++ ...1-similarity-search-for-motif-mining.ipynb | 6 + 65 files changed, 1227 insertions(+), 422 deletions(-) delete mode 100644 docs/source/reference/udfs/model-train.rst delete mode 100644 docs/source/reference/udfs/openai.rst delete mode 100644 docs/source/usecases/food-review.rst delete mode 100644 docs/source/usecases/qa-video.rst delete mode 100644 docs/source/usecases/similar-image-search.rst diff --git a/README.md b/README.md index 5afd973e45..29a920720b 100644 --- a/README.md +++ b/README.md @@ -223,13 +223,24 @@ EvaDB enables software developers to build AI apps in a few lines of code. Its p
  • 📝 following us on Medium +👋 Hey! If you're excited about our vision of bringing AI inside database systems, show some ❤️ by: + + ## Quick Links <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) - [Quick Links](#quick-links) - [Documentation](#documentation) - [Why EvaDB](#why-evadb) @@ -244,6 +255,7 @@ EvaDB enables software developers to build AI apps in a few lines of code. Its p - [License](#license) ======= <<<<<<< HEAD +<<<<<<< HEAD ======= - [Quick Links](#quick-links) >>>>>>> 2dacff69 (feat: sync master staging (#1050)) @@ -259,13 +271,26 @@ EvaDB enables software developers to build AI apps in a few lines of code. Its p - [Star History](#star-history) - [License](#license) ======= +======= +======= +- [Quick Links](#quick-links) +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) - [Documentation](#documentation) - [Why EvaDB](#why-evadb) - [How does EvaDB work](#how-does-evadb-work) -- [Community and Support](#community-and-support) - [Illustrative Queries](#illustrative-queries) - [Illustrative Apps](#illustrative-apps) +<<<<<<< HEAD >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +- [More Illustrative Queries](#more-illustrative-queries) +- [Architecture of EvaDB](#architecture-of-evadb) +- [Community and Support](#community-and-support) +- [Contributing](#contributing) +- [Star History](#star-history) +- [License](#license) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ## Documentation @@ -555,12 +580,17 @@ TIME_LIMIT 120; * Train an ML model using the Ludwig AI engine to predict a column in a table. ```sql -CREATE UDF IF NOT EXISTS PredictHouseRent FROM +CREATE FUNCTION IF NOT EXISTS PredictHouseRent FROM ( SELECT * FROM HomeRentals ) TYPE Ludwig +<<<<<<< HEAD 'predict' 'rental_price' 'time_limit' 120; >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +PREDICT 'rental_price' +TIME_LIMIT 120; +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ``` diff --git a/docs/_toc.yml b/docs/_toc.yml index 974bad27b5..8b6776b265 100644 --- a/docs/_toc.yml +++ b/docs/_toc.yml @@ -47,14 +47,18 @@ parts: sections: - file: source/overview/concepts/data-sources title: Data Sources +<<<<<<< HEAD ======= - file: source/overview/concepts title: Concepts >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) #- file: source/overview/faq - caption: Use Cases chapters: +<<<<<<< HEAD <<<<<<< HEAD - file: source/usecases/sentiment-analysis.rst title: Sentiment Analysis @@ -86,24 +90,23 @@ parts: ======= ======= - file: source/usecases/food-review.rst +======= + - file: source/usecases/sentiment-analysis.rst +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) title: Sentiment Analysis + - file: source/usecases/question-answering.rst + title: Question Answering + - file: source/usecases/text-summarization.rst + title: Text Summarization - file: source/usecases/image-classification.rst title: Image Classification - - file: source/usecases/similar-image-search.rst + - file: source/usecases/image-search.rst title: Image Search - - file: source/usecases/qa-video.rst - title: Video Question Answering - - file: source/usecases/08-chatgpt.ipynb - title: ChatGPT-based Video Question Answering - - file: source/usecases/12-query-pdf.ipynb - title: PDF Question Answering - - file: source/usecases/02-object-detection.ipynb + - file: source/usecases/object-detection.rst title: Object Detection - - file: source/usecases/03-emotion-analysis.ipynb - title: Emotions Analysis - - file: source/usecases/07-object-segmentation-huggingface.ipynb - title: Image Segmentation - - file: source/usecases/13-privategpt.ipynb + - file: source/usecases/emotion-analysis.rst + title: Emotion Analysis + - file: source/usecases/privategpt.rst title: PrivateGPT >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) >>>>>>> c63abee7 (release: merge staging into master (#1032)) @@ -155,6 +158,9 @@ parts: <<<<<<< HEAD ======= + - file: source/reference/api + title: Python API + - file: source/reference/databases/index title: Data Sources sections: @@ -163,9 +169,12 @@ parts: - file: source/reference/databases/mysql >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD <<<<<<< HEAD >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) ======= +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) - file: source/reference/api title: Python API @@ -214,12 +223,16 @@ parts: title: Model Training with Ludwig - file: source/reference/ai/model-train-sklearn title: Model Training with Sklearn +<<<<<<< HEAD - file: source/reference/ai/model-train-xgboost title: Model Training with XGBoost +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= - file: source/reference/ai/index title: AI Engines sections: +<<<<<<< HEAD <<<<<<< HEAD - file: source/reference/ai/model-train title: Model Training @@ -230,6 +243,11 @@ parts: - file: source/reference/ai/model-train-sklearn title: Model Training with Sklearn >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= + - file: source/reference/ai/model-train + title: Model Training +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) - file: source/reference/ai/model-forecasting title: Time Series Forecasting - file: source/reference/ai/hf @@ -238,16 +256,23 @@ parts: title: OpenAI - file: source/reference/ai/yolo title: YOLO +<<<<<<< HEAD <<<<<<< HEAD - file: source/reference/ai/stablediffusion title: Stable Diffusion - file: source/reference/ai/custom-ai-function title: Bring Your Own AI Function +======= + - file: source/reference/ai/custom + title: Custom Model +<<<<<<< HEAD +>>>>>>> 53dafecf (feat: sync master staging (#1050)) - file: source/reference/optimizations title: Optimizations ======= +<<<<<<< HEAD - file: source/reference/ai/custom title: Custom Model <<<<<<< HEAD @@ -257,6 +282,9 @@ parts: - file: source/reference/optimizations title: Optimizations >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) # - file: source/reference/io # title: IO Descriptors diff --git a/docs/conf.py b/docs/conf.py index 4a2fb30f62..f6ca979e40 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -106,6 +106,7 @@ ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) + # The name of the Pygments (syntax highlighting) style to use. pygments_style = "github-dark" @@ -186,6 +187,7 @@ <<<<<<< HEAD ======= +<<<<<<< HEAD for i in os.listdir("../tutorials"): if i in [ @@ -203,6 +205,7 @@ >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) >>>>>>> c63abee7 (release: merge staging into master (#1032)) +<<<<<<< HEAD <<<<<<< HEAD for i in os.listdir("../tutorials"): @@ -220,6 +223,8 @@ nb_execution_mode = "off" >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) # -- Initialize Sphinx ---------------------------------------------- diff --git a/docs/source/benchmarks/text_summarization.rst b/docs/source/benchmarks/text_summarization.rst index 1cf1bc223d..7700f3144c 100644 --- a/docs/source/benchmarks/text_summarization.rst +++ b/docs/source/benchmarks/text_summarization.rst @@ -1,8 +1,11 @@ <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Text Summarization Benchmark ============================ @@ -17,6 +20,7 @@ Prepare dataset ======= Text summarization benchmark <<<<<<< HEAD +<<<<<<< HEAD ======= Text Summarization Benchmark >>>>>>> 2dacff69 (feat: sync master staging (#1050)) @@ -37,13 +41,31 @@ Prepare dataset --------------- >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= +======= +======= +Text Summarization Benchmark +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ============================ -In this benchmark, we compare the performance of text summarization between EvaDB and MindsDB on `CNN-DailyMail News `_. +<<<<<<< HEAD 1. Prepare dataset ------------------ >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +======= +In this benchmark, we compare the runtime performance of EvaDB and MindsDB on +a text summarization application operating on a news dataset. In particular, +we focus on the `CNN-DailyMail News `_ dataset. + +All the relevant files are located in the `text summarization benchmark folder on Github `_. + +Prepare dataset +--------------- +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. code-block:: bash @@ -53,8 +75,11 @@ In this benchmark, we compare the performance of text summarization between EvaD <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Use EvaDB for Text Summarization -------------------------------- ======= @@ -62,12 +87,18 @@ Use EvaDB for Text Summarization -------------------------------------------------- >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= Use EvaDB for Text Summarization -------------------------------- >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. note:: @@ -83,6 +114,7 @@ Use EvaDB for Text Summarization <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD Loading Data Into EvaDB ~~~~~~~~~~~~~~~~~~~~~~~ @@ -140,6 +172,8 @@ Prepare sqlite database for MindsDB ======= ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Loading Data Into EvaDB ~~~~~~~~~~~~~~~~~~~~~~~ @@ -197,7 +231,59 @@ Setup SQLite Database Prepare sqlite database for MindsDB ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +======= +Loading Data Into EvaDB +~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: sql + + CREATE TABLE IF NOT EXISTS cnn_news_test( + id TEXT(128), + article TEXT(4096), + highlights TEXT(1024) + ); + +Creating Text Summarization Function in EvaDB +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: sql + + CREATE UDF IF NOT EXISTS TextSummarizer + TYPE HuggingFace + TASK 'summarization' + MODEL 'sshleifer/distilbart-cnn-12-6' + MIN_LENGTH 5 + MAX_LENGTH 100; + + +Tuning EvaDB for Maximum GPU Utilization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: python + + cursor._evadb.config.update_value("executor", "batch_mem_size", 300000) + cursor._evadb.config.update_value("executor", "gpu_ids", [0,1]) + cursor._evadb.config.update_value("experimental", "ray", True) + + +Text Summarization Query in EvaDB +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. code-block:: sql + + CREATE TABLE IF NOT EXISTS cnn_news_summary AS + SELECT TextSummarizer(article) FROM cnn_news_test; + +Use MindsDB for Text Summarization +----------------------------------- + +Setup SQLite Database +~~~~~~~~~~~~~~~~~~~~~~ +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. code-block:: bash @@ -213,7 +299,11 @@ Install MindsDB <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD Follow the `MindsDB installation guide `_ to install it via ``pip``. +======= +Follow the `MindsDB nstallation guide `_ to install it via ``pip``. +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. note:: @@ -226,14 +316,18 @@ Follow the `Setup for Source Code via pip >>>>>> 8c5b63dc (release: merge staging into master (#1032)) ======= +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Follow the `MindsDB nstallation guide `_ to install it via ``pip``. .. note:: You will need to manually run ``pip install evaluate`` for the ``HuggingFace`` model to work in MindsDB. <<<<<<< HEAD +<<<<<<< HEAD >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= ======= @@ -244,6 +338,9 @@ Follow the `Setup for Source Code via pip >>>>>> 8c5b63dc (release: merge staging into master (#1032)) >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) After installation, use the ``MySQL`` client for connecting to ``MindsDB``. Update the port number if needed. @@ -254,8 +351,11 @@ After installation, use the ``MySQL`` client for connecting to ``MindsDB``. Upda <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Benchmark MindsDB ~~~~~~~~~~~~~~~~~ ======= @@ -263,12 +363,18 @@ Run Experiment ~~~~~~~~~~~~~~ >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= Benchmark MindsDB ~~~~~~~~~~~~~~~~~ >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Connect ``MindsDB`` to the ``sqlite`` database we created before: @@ -310,8 +416,11 @@ Use the ``text summarization`` model to summarize the CNN news dataset: <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Benchmarking Results -------------------- ======= @@ -320,12 +429,18 @@ Benchmarking Results Below are numbers from a server with 56 Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz and two Quadro P6000 GPU. >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= Benchmarking Results -------------------- >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Here are the key runtime metrics for the ``Text Summarization`` benchmark. diff --git a/docs/source/overview/concepts.rst b/docs/source/overview/concepts.rst index bf4356f20d..1f27a17c33 100644 --- a/docs/source/overview/concepts.rst +++ b/docs/source/overview/concepts.rst @@ -128,7 +128,11 @@ EvaDB accelerates AI queries using a collection of optimizations inspired by SQL AI Functions ------------ +<<<<<<< HEAD ``Functions`` are typically thin wrappers around AI models and are extensively used in queries. Here is an `illustrative AI function for classifying MNIST images `_. +======= +``Functions`` are typically thin wrappers around AI models and are extensively used in queries. Here is an `illustrative AI function for classifying MNIST images `_. +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) To register an user-defined function, use the ``CREATE FUNCTION`` statement: @@ -156,6 +160,9 @@ After registering ``MnistImageClassifier`` function, you can call the function i WHERE id = 30; <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) --- Use the 'MnistImageClassifier' function's output to filter frames --- This query returns the frame ids of the frames with digit 6 --- We limit to the first five frames containing digit 6 @@ -165,6 +172,7 @@ After registering ``MnistImageClassifier`` function, you can call the function i LIMIT 5; <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD AI-Centric Query Optimization ----------------------------- @@ -176,6 +184,8 @@ Query optimization has powered SQL database systems for several decades. It is t ======= ======= ======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) +======= Save time and money ---------------------- @@ -233,7 +243,12 @@ That's it! You can now use the newly registered UDF anywhere in the query -- in response = query2.df() >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) diff --git a/docs/source/overview/getting-started.rst b/docs/source/overview/getting-started.rst index 1699a7d8df..979fb4b2ab 100644 --- a/docs/source/overview/getting-started.rst +++ b/docs/source/overview/getting-started.rst @@ -1,18 +1,27 @@ <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. _getting-started: ======= .. _Getting Started: >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) <<<<<<< HEAD +<<<<<<< HEAD ======= .. _getting-started: >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +======= +.. _getting-started: +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Getting Started ================= @@ -132,7 +141,7 @@ The program runs a SQL query for listing all the built-in functions in EvaDB. It cursor = evadb.connect().cursor() # List all the built-in functions in EvaDB - print(cursor.query("SHOW UDFS;").df()) + print(cursor.query("SHOW FUNCTIONS;").df()) Now, run the Python program: @@ -145,9 +154,9 @@ You should see a list of built-in functions including but not limited to the fol .. code-block:: bash name inputs ... impl metadata - 0 ArrayCount [Input_Array NDARRAY ANYTYPE (), Search_Key ANY] ... /home/jarulraj3/evadb/evadb/udfs/ndarray/array... [] - 1 Crop [Frame_Array NDARRAY UINT8 (3, None, None), bb... ... /home/jarulraj3/evadb/evadb/udfs/ndarray/crop.py [] - 2 ChatGPT [query NDARRAY STR (1,), content NDARRAY STR (... ... /home/jarulraj3/evadb/evadb/udfs/chatgpt.py [] + 0 ArrayCount [Input_Array NDARRAY ANYTYPE (), Search_Key ANY] ... /home/jarulraj3/evadb/evadb/functions/ndarray/array... [] + 1 Crop [Frame_Array NDARRAY UINT8 (3, None, None), bb... ... /home/jarulraj3/evadb/evadb/functions/ndarray/crop.py [] + 2 ChatGPT [query NDARRAY STR (1,), content NDARRAY STR (... ... /home/jarulraj3/evadb/evadb/functions/chatgpt.py [] [3 rows x 6 columns] @@ -204,14 +213,27 @@ The program runs a SQL query for listing all the built-in functions in EvaDB. It ======= EvaDB supports additional installation options for extending its functionality. Go over the :doc:`Installation Options ` for all the available options. -Illustrative AI App -------------------- +Illustrative AI Query +--------------------- +<<<<<<< HEAD Here is a simple, illustrative `MNIST image classification `_ AI app in EvaDB. >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +======= +Here is an illustrative `MNIST image classification `_ AI query in EvaDB. +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) -.. code-block:: python +.. code-block:: sql + + --- This AI query retrieves images in the loaded MNIST video with label 4 + --- We constrain the query to only search through the first 100 frames + --- We limit the query to only return the first five frames with label 4 + SELECT data, id, MnistImageClassifier(data) + FROM MnistVideo + WHERE MnistImageClassifier(data) = '4' AND id < 100 + LIMIT 5; +<<<<<<< HEAD # Import the EvaDB package import evadb @@ -366,6 +388,7 @@ Try out EvaDB by experimenting with the introductory `MNIST notebook on Colab `_. Try out EvaDB by experimenting with this introductory notebook. <<<<<<< HEAD +<<<<<<< HEAD >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= ======= @@ -377,3 +400,6 @@ Try out EvaDB by experimenting with the introductory `MNIST notebook on Colab ` to learn more about the functions used in this app. >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) diff --git a/docs/source/overview/getting-started/installation-options.rst b/docs/source/overview/getting-started/installation-options.rst index 7129484552..157edffd4f 100644 --- a/docs/source/overview/getting-started/installation-options.rst +++ b/docs/source/overview/getting-started/installation-options.rst @@ -68,6 +68,7 @@ EvaDB provides the following additional installation options for extending its f * ``pip install evadb[qdrant]`` for installing the Qdrant vector database system. It enables use-cases related to similarity search based on feature vectors. +<<<<<<< HEAD <<<<<<< HEAD * ``pip install evadb[ludwig]`` for installing the Ludwig model training framework. It enables use-cases related to training and fine-tuning AI models. @@ -75,6 +76,9 @@ EvaDB provides the following additional installation options for extending its f ======= * ``pip install evadb[ludwig]`` for installing the Ludwig model training framework. It enables use-cases related to training and fine-tunining AI models. >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +======= +* ``pip install evadb[ludwig]`` for installing the Ludwig model training framework. It enables use-cases related to training and fine-tuning AI models. +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) * ``pip install evadb[ray]`` for installing the Ray compute engine. It enables EvaDB to do more efficient query execution on CPUs and GPUs. diff --git a/docs/source/reference/ai/custom.rst b/docs/source/reference/ai/custom.rst index b528131b72..ab0f513638 100644 --- a/docs/source/reference/ai/custom.rst +++ b/docs/source/reference/ai/custom.rst @@ -1,5 +1,9 @@ .. _udf: +<<<<<<< HEAD +======= + +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) Functions ====================== diff --git a/docs/source/reference/ai/model-forecasting.rst b/docs/source/reference/ai/model-forecasting.rst index ca8af04ad4..6dfe0d5ecb 100644 --- a/docs/source/reference/ai/model-forecasting.rst +++ b/docs/source/reference/ai/model-forecasting.rst @@ -1,13 +1,19 @@ <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. _forecast: ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= .. _forecast: >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Time Series Forecasting ======================== @@ -31,6 +37,9 @@ First, we create a table to insert required data. <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Next, we create a function of `TYPE Forecasting`. We must enter the column name on which we wish to forecast using `PREDICT`. .. code-block:: sql @@ -43,6 +52,7 @@ Next, we create a UDF of `TYPE Forecasting`. We must enter the column name on wh CREATE UDF IF NOT EXISTS Forecast FROM >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= Next, we create a function of `TYPE Forecasting`. We must enter the column name on which we wish to forecast using `PREDICT`. @@ -50,6 +60,8 @@ Next, we create a function of `TYPE Forecasting`. We must enter the column name CREATE FUNCTION IF NOT EXISTS Forecast FROM >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) (SELECT y FROM AirData) TYPE Forecasting PREDICT 'y'; @@ -60,6 +72,9 @@ This trains a forecasting model. The model can be called by providing the horizo <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) SELECT Forecast(12); Here, the horizon is `12`, which represents the forecast 12 steps into the future. @@ -104,6 +119,7 @@ Below is an example query specifying the above parameters: TIME 'saledate' ID 'type' Frequency 'W'; +<<<<<<< HEAD Below is an example query with `neuralforecast` with `trend` column as exogenous and without automatic hyperparameter optimization: @@ -117,11 +133,14 @@ Below is an example query with `neuralforecast` with `trend` column as exogenous LIBRARY 'neuralforecast' AUTO 'f' FREQUENCY 'M'; +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= SELECT Forecast(12) FROM AirData; Here, the horizon is `12`. >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= SELECT Forecast(12); @@ -181,3 +200,5 @@ Below is an example query with `neuralforecast` with `trend` column as exogenous AUTO 'f' FREQUENCY 'M'; >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) diff --git a/docs/source/reference/api.rst b/docs/source/reference/api.rst index 28282f67fd..3503959e42 100644 --- a/docs/source/reference/api.rst +++ b/docs/source/reference/api.rst @@ -3,18 +3,27 @@ <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Python API ======= Basic API >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) <<<<<<< HEAD +<<<<<<< HEAD ======= Python API >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +======= +Python API +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ========== To begin a querying session in EvaDB, obtain a connection with a cursor using ``connect`` and ``cursor`` functions. After getting the cursor, you can run queries with the ``query`` function in this manner: @@ -105,9 +114,12 @@ EvaDBQuery Interface ``cursor.query("...")`` only construct the query and not run the query. ``cursor.query("...").df()`` will both construct and run the query. +<<<<<<< HEAD <<<<<<< HEAD >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) <<<<<<< HEAD ======= EvaDBCursor Interface @@ -152,4 +164,9 @@ EvaDBQuery Interface ~evadb.EvaDBQuery.sql_query ~evadb.EvaDBQuery.execute >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) diff --git a/docs/source/reference/evaql/create.rst b/docs/source/reference/evaql/create.rst index 28d29c352d..be2f0a2c7b 100644 --- a/docs/source/reference/evaql/create.rst +++ b/docs/source/reference/evaql/create.rst @@ -49,6 +49,7 @@ To create a table, specify the schema of the table. object_id INTEGER ); +<<<<<<< HEAD <<<<<<< HEAD CREATE INDEX ------------ @@ -93,6 +94,11 @@ You can check out :ref:`similarity search use case` about how to u CREATE FUNCTION --------------- +======= +CREATE FUNCTION +--------------- + +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) To register an user-defined function, specify the implementation details of the function. .. code-block:: sql @@ -110,14 +116,15 @@ CREATE FUNCTION via Type ======= .. _create-udf-train: -CREATE UDF via Training ------------------------ +CREATE FUNCTION via Training +---------------------------- To register an user-defined function by training a predication model. >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) .. code-block:: sql +<<<<<<< HEAD CREATE [OR REPALCE] FUNCTION [IF NOT EXISTS] function_name [ FROM ( select ) ] TYPE function_type @@ -132,6 +139,14 @@ Where the `parameter` is ``key value`` pair. .. note:: Go over :ref:`hf`, :ref:`ludwig`, and :ref:`forecast` to check examples for creating function via type. +======= + CREATE FUNCTION IF NOT EXISTS PredictHouseRent FROM + (SELECT * FROM HomeRentals) + TYPE Ludwig + PREDICT 'rental_price' + TIME_LIST 120; + TUNE_FOR_MEMORY False; +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) CREATE MATERIALIZED VIEW ------------------------ diff --git a/docs/source/reference/evaql/drop.rst b/docs/source/reference/evaql/drop.rst index 7bf67fca8b..0ca3339f76 100644 --- a/docs/source/reference/evaql/drop.rst +++ b/docs/source/reference/evaql/drop.rst @@ -9,6 +9,7 @@ DROP TABLE DROP TABLE DETRACVideo; +<<<<<<< HEAD <<<<<<< HEAD DROP FUNCTION ------------- @@ -16,10 +17,15 @@ DROP FUNCTION DROP UDF -------- >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +======= +DROP FUNCTION +------------- +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) .. code:: mysql DROP FUNCTION FastRCNNObjectDetector; +<<<<<<< HEAD DROP DATABASE ------------- @@ -27,4 +33,6 @@ DROP DATABASE .. code:: mysql DROP DATABASE postgres_db; - DROP DATABASE IF EXISTS postgres_db; \ No newline at end of file + DROP DATABASE IF EXISTS postgres_db; +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) diff --git a/docs/source/reference/evaql/select.rst b/docs/source/reference/evaql/select.rst index 3e3be6144d..1cba7235da 100644 --- a/docs/source/reference/evaql/select.rst +++ b/docs/source/reference/evaql/select.rst @@ -40,6 +40,7 @@ Search for frames containing greater than 3 cars <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD SELECT TUPLES WITH MULTIPLE PREDICATES -------------------------------------- ======= @@ -51,13 +52,22 @@ SELECT WITH MULTIPLE FUNCTIONS ------------------------------ >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) SELECT WITH MULTIPLE FUNCTIONS ------------------------------ ======= SELECT WITH MULTIPLE UDFS ------------------------- >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +======= +SELECT WITH MULTIPLE FUNCTIONS +------------------------------ +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Compose multiple user-defined functions in a single query to construct semantically complex queries. diff --git a/docs/source/reference/evaql/show.rst b/docs/source/reference/evaql/show.rst index 4352f3abb3..427cc734af 100644 --- a/docs/source/reference/evaql/show.rst +++ b/docs/source/reference/evaql/show.rst @@ -1,6 +1,7 @@ SHOW ==== +<<<<<<< HEAD <<<<<<< HEAD SHOW FUNCTIONS -------------- @@ -8,6 +9,10 @@ SHOW FUNCTIONS SHOW UDFS --------- >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +======= +SHOW FUNCTIONS +-------------- +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) List the registered user-defined functions diff --git a/docs/source/reference/evaql/udf.rst b/docs/source/reference/evaql/udf.rst index e27cef510b..7514f17d41 100644 --- a/docs/source/reference/evaql/udf.rst +++ b/docs/source/reference/evaql/udf.rst @@ -8,12 +8,17 @@ SHOW FUNCTIONS -------------- ======= -UDF -=== +Functions +========= +<<<<<<< HEAD SHOW UDFS --------- >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +======= +SHOW FUNCTIONS +-------------- +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) Here is a list of built-in user-defined functions in EvaDB. @@ -32,6 +37,7 @@ FastRCNNObjectDetector is a model for detecting objects. MVITActionRecognition i ArrayCount and Crop are utility functions for counting the number of objects in an array and cropping a bounding box from an image, respectively. +<<<<<<< HEAD <<<<<<< HEAD SELECT WITH MULTIPLE FUNCTIONS ------------------------------ @@ -39,6 +45,10 @@ SELECT WITH MULTIPLE FUNCTIONS SELECT WITH MULTIPLE UDFS ------------------------- >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +======= +SELECT WITH MULTIPLE FUNCTIONS +------------------------------ +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) Here is a query that illustrates how to use multiple functions in a single query. diff --git a/docs/source/reference/evaql/use.rst b/docs/source/reference/evaql/use.rst index 84e585c04b..a9f2f1a444 100644 --- a/docs/source/reference/evaql/use.rst +++ b/docs/source/reference/evaql/use.rst @@ -11,6 +11,7 @@ The USE statement allows us to run arbitrary native queries in the connected dat USE === +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD The USE statement allows us to run arbitary native queries in the connected database. @@ -19,11 +20,19 @@ The USE statement allows us to run arbitary native queries in the connected data The USE statement allows us to run arbitrary native queries in the connected database. >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) The USE statement allows us to run arbitrary native queries in the connected database. ======= The USE statement allows us to run arbitary native queries in the connected database. >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +======= +The USE statement allows us to run arbitrary native queries in the connected database. +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. code:: text @@ -33,18 +42,27 @@ The USE statement allows us to run arbitary native queries in the connected data <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) * [native_query] is an arbitrary SQL query supported by the [database_connection]. ======= * [native_query] is an arbitary SQL query supprted by the [database_connection]. >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) <<<<<<< HEAD +<<<<<<< HEAD ======= * [native_query] is an arbitrary SQL query supported by the [database_connection]. >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +======= +* [native_query] is an arbitrary SQL query supported by the [database_connection]. +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. warning:: diff --git a/docs/source/reference/udfs/model-train.rst b/docs/source/reference/udfs/model-train.rst deleted file mode 100644 index de6e84eeee..0000000000 --- a/docs/source/reference/udfs/model-train.rst +++ /dev/null @@ -1,48 +0,0 @@ -Training and Finetuning -======================== - -1. You can train a predication model easily in EvaDB - -.. note:: - - Install Ludwig in your EvaDB virtual environment: ``pip install evadb[ludwig]``. - -.. code-block:: sql - - CREATE UDF IF NOT EXISTS PredictHouseRent FROM - ( SELECT sqft, location, rental_price FROM HomeRentals ) - TYPE Ludwig - 'predict' 'rental_price' - 'time_limit' 120; - -In the above query, you are creating a new customized UDF by automatically training a model from the `HomeRentals` table. The `rental_price` column will be the target column for predication, while `sqft` and `location` are the inputs. - -You can also simply give all other columns in `HomeRentals` as inputs and let the underlying automl framework to figure it out. Below is an example query: - -.. code-block:: sql - - CREATE UDF IF NOT EXISTS PredictHouseRent FROM - ( SELECT * FROM HomeRentals ) - TYPE Ludwig - 'predict' 'rental_price' - 'time_limit' 120; - -.. note:: - - Check :ref:`create-udf-train` for available configurations for training models. - -2. After training completes, you can use the `PredictHouseRent` like all other UDFs in EvaDB - -.. code-block:: sql - - CREATE PredictHouseRent(sqft, location) FROM HomeRentals; - -You can also simply give all columns in `HomeRentals` as inputs for inference. The customized UDF with the underlying model can figure out the proper inference columns via the training columns. - -.. code-block:: sql - - CREATE PredictHouseRent(*) FROM HomeRentals; - -Check out our `Integration Tests `_ for working example. - - diff --git a/docs/source/reference/udfs/openai.rst b/docs/source/reference/udfs/openai.rst deleted file mode 100644 index d229eff44a..0000000000 --- a/docs/source/reference/udfs/openai.rst +++ /dev/null @@ -1,27 +0,0 @@ -OpenAI Models -===================== - -This section provides an overview of how you can use OpenAI models in EvaDB. - - -Chat Completion UDFs --------------------- - -To create a chat completion UDF in EvaDB, use the following SQL command: - -.. code-block:: sql - - CREATE UDF IF NOT EXISTS OpenAIChatCompletion - IMPL 'evadb/udfs/openai_chat_completion_udf.py' - 'model' 'gpt-3.5-turbo' - -EvaDB supports the following models for chat completion task: - -- "gpt-4" -- "gpt-4-0314" -- "gpt-4-32k" -- "gpt-4-32k-0314" -- "gpt-3.5-turbo" -- "gpt-3.5-turbo-0301" - -The chat completion UDF can be composed in interesting ways with other UDFs. Please check the `Google Colab `_ for an example of combining chat completion task with caption extraction and video summarization models from Hugging Face and feeding it to chat completion to ask questions about the results. diff --git a/docs/source/shared/postgresql.rst b/docs/source/shared/postgresql.rst index da8fca4a10..7f786fc7f8 100644 --- a/docs/source/shared/postgresql.rst +++ b/docs/source/shared/postgresql.rst @@ -5,6 +5,7 @@ We will assume that you have a ``PostgreSQL`` database server running locally th EvaDB lets you connect to your favorite databases, data warehouses, data lakes, etc., via the ``CREATE DATABASE`` statement. In this query, we connect EvaDB to an existing ``PostgreSQL`` server: +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD .. code-block:: @@ -52,6 +53,8 @@ EvaDB lets you connect to your favorite databases, data warehouses, data lakes, } >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. code-block:: text CREATE DATABASE postgres_data @@ -63,4 +66,40 @@ EvaDB lets you connect to your favorite databases, data warehouses, data lakes, "port": "5432", "database": "evadb" } +<<<<<<< HEAD >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +.. tab-set:: + + .. tab-item:: Python + + .. code-block:: python + + params = { + "user": "eva", + "password": "password", + "host": "localhost", + "port": "5432", + "database": "evadb", + } + query = f"CREATE DATABASE postgres_data + WITH ENGINE = 'postgres', + PARAMETERS = {params};" + cursor.query(query).df() + + .. tab-item:: SQL + + .. code-block:: text + + CREATE DATABASE postgres_data + WITH ENGINE = 'postgres', + PARAMETERS = { + "user": "eva", + "password": "password", + "host": "localhost", + "port": "5432", + "database": "evadb" + } +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) diff --git a/docs/source/usecases/emotion-analysis.rst b/docs/source/usecases/emotion-analysis.rst index a1e1a79a6b..39e61c404f 100644 --- a/docs/source/usecases/emotion-analysis.rst +++ b/docs/source/usecases/emotion-analysis.rst @@ -86,10 +86,13 @@ This query returns the faces detected in the first ten frames of the video: .. code-block:: +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) +----------+---------------------+-------------------------+ | happy.id | facedetector.bboxes | facedetector.scores | +----------+---------------------+-------------------------+ @@ -110,6 +113,9 @@ This query returns the faces detected in the first ten frames of the video: | | [235 309 325 412]] | | +----------+---------------------+-------------------------+ <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= +-----------------------------------------------------------------------------------------------------+ | objectdetectionvideos.id | yolo.labels | @@ -118,8 +124,11 @@ This query returns the faces detected in the first ten frames of the video: | 1 | [car, car, car, car, car, car, car, car, car, ... | +-----------------------------------------------------------------------------------------------------+ >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Chaining Functions in a Single AI Query --------------------------------------- @@ -139,10 +148,13 @@ Now, the ``DataFrame`` only contains the emotions of the detected faces: .. code-block:: +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) +----------+-------------------+------------------------+------------------------+ | happy.id | Face.bbox | emotiondetector.labels | emotiondetector.scores | +----------+-------------------+------------------------+------------------------+ @@ -169,6 +181,9 @@ Now, the ``DataFrame`` only contains the emotions of the detected faces: | 14 | [515 88 790 454] | happy | 0.9997135996818542 | +----------+-------------------+------------------------+------------------------+ <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= +------------------------------+ | objectdetectionvideos.label | @@ -177,7 +192,10 @@ Now, the ``DataFrame`` only contains the emotions of the detected faces: | 6 | +------------------------------+ >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. include:: ../shared/footer.rst diff --git a/docs/source/usecases/food-review.rst b/docs/source/usecases/food-review.rst deleted file mode 100644 index 8f688b0e83..0000000000 --- a/docs/source/usecases/food-review.rst +++ /dev/null @@ -1,147 +0,0 @@ -ChatGPT + Postgres Tutorial -=========================== - -.. raw:: html - - - - - - -
    - Run on Google Colab - - View source on GitHub - - Download notebook -


    - - -In this tutorial, we demonstrate how to use EvaDB + ChatGPT to analyze the tone of food reviews stored in PostgreSQL. Then, based on the analysis, we further use -EvaDB + ChatGPT to address negative reviews by proposing a solution to the customer. - -For this use case, we assume user has a Postgres server running locally. You can also check our notebook above to skip Postgres setup. - -1. Connect to EvaDB ---------------------- - -.. code-block:: python - - import evadb - cursor = evadb.connect().cursor() - -2. Connect to an Existing Postgres Database ---------------------------------------------- - -.. tab-set:: - - .. tab-item:: Python - - .. code-block:: python - - params = { - "user": "eva", - "password": "password", - "host": "localhost", - "port": "5432", - "database": "evadb", - } - query = f"CREATE DATABASE postgres_data WITH ENGINE = 'postgres', PARAMETERS = {params};" - cursor.query(query).df() - - .. tab-item:: SQL - - .. code-block:: text - - CREATE DATABASE postgres_data WITH ENGINE = 'postgres', PARAMETERS = { - "user": "eva", - "password": "password", - "host": "localhost", - "port": "5432", - "database": "evadb" - } - -3. Sentiment Analysis of Food Review using ChatGPT ---------------------------------------------------- - -We then use EvaDB + ChatGPT to analyze whether the review is "positive" or "negative" with customized ChatGPT prompt. For this use case, -we assume reviews have been already loaded into the table inside PostgreSQL. -You can check our `Jupyter Notebook `__ for how to load data. - -.. tab-set:: - - .. tab-item:: Python - - .. code-block:: python - - cursor.query(""" - SELECT ChatGPT( - "Is the review positive or negative. Only reply 'positive' or 'negative'. Here are examples. The food is very bad: negative. The food is very good: postive.", - review) - FROM postgres_data.review_table; - """).df() - - .. tab-item:: SQL - - .. code-block:: sql - - SELECT ChatGPT( - "Is the review positive or negative. Only reply 'positive' or 'negative'. Here are examples. The food is very bad: negative. The food is very good: postive.", - review) - FROM postgres_data.review_table; - -This will return tone analysis results for existing reviews. - -.. code-block:: - - +------------------------------+ - | chatgpt.response | - |------------------------------| - | negative | - | positive | - | negative | - +------------------------------+ - -4. Response to Negative Reviews using ChatGPT ---------------------------------------------- - -.. tab-set:: - - .. tab-item:: Python - - .. code-block:: python - - cursor.query(""" - SELECT ChatGPT( - "Respond the the review with solution to address the review's concern", - review) - FROM postgres_data.review_table - WHERE ChatGPT( - "Is the review positive or negative. Only reply 'positive' or 'negative'. Here are examples. The food is very bad: negative. The food is very good: postive.", - review) = "negative"; - """).df() - - .. tab-item:: SQL - - .. code-block:: sql - - SELECT ChatGPT( - "Respond the the review with solution to address the review's concern", - review) - FROM postgres_data.review_table - WHERE ChatGPT( - "Is the review positive or negative. Only reply 'positive' or 'negative'. Here are examples. The food is very bad: negative. The food is very good: postive.", - review) = "negative"; - -This query will first filter out positive reviews and then apply ChatGPT again to create response to negative reviews. This will give results. - -.. code-block:: - - +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - | chatgpt.response | - |----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| - | Dear valued customer, Thank you for bringing this matter to our attention. We apologize for the inconvenience caused by the excessive saltiness of your fried rice. We understand how important it is to have a satisfying dining experience, and we would like to make it right for you ... | - | Dear [Customer's Name], Thank you for bringing this issue to our attention. We apologize for the inconvenience caused by the missing chicken sandwich in your takeout order. We understand how frustrating it can be when an item is missing from your meal. To address this concern, we ... | - +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -Check out our `Jupyter Notebook `__ for working example. \ No newline at end of file diff --git a/docs/source/usecases/image-classification.rst b/docs/source/usecases/image-classification.rst index 2351b5d0c1..db83bfb2de 100644 --- a/docs/source/usecases/image-classification.rst +++ b/docs/source/usecases/image-classification.rst @@ -1,6 +1,7 @@ <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD .. _image-classification: ======= .. _image classification: @@ -82,6 +83,8 @@ In the following query, we call the classifier on every image in the video. The ======= ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. _image-classification: ======= .. _image classification: @@ -160,6 +163,67 @@ In the following query, we call the classifier on every image in the video. The SELECT MnistImageClassifier(data).label FROM mnist_video; +<<<<<<< HEAD +======= +======= +.. _image-classification: + +Image Classification +==================== + +.. raw:: html + + + + + + +
    + Run on Google Colab + + View source on GitHub + + Download notebook +


    + + +Introduction +------------ + +In this tutorial, we present how to use ``PyTorch`` models in EvaDB to classify images. In particular, we focus on classifying images from the ``MNIST`` dataset that contains ``digits``. EvaDB makes it easy to do image classification using its built-in support for ``PyTorch`` models. + +In this tutorial, besides classifying images, we will also showcase a query where the model's output is used to retrieve images with the digit ``6``. + +.. include:: ../shared/evadb.rst + +We will assume that the input ``MNIST`` video is loaded into ``EvaDB``. To download the video and load it into ``EvaDB``, see the complete `image classification notebook on Colab `_. + +Create Image Classification Function +------------------------------------ + +To create a custom ``MnistImageClassifier`` function, use the ``CREATE FUNCTION`` statement. The code for the custom classification model is available `here `_. + +We will assume that the file is downloaded and stored as ``mnist_image_classifier.py``. Now, run the following query to register the AI function: + +.. code-block:: sql + + CREATE FUNCTION + IF NOT EXISTS MnistImageClassifier + IMPL 'mnist_image_classifier.py'; + +Image Classification Queries +---------------------------- + +After the function is registered in ``EvaDB``, you can use it subsequent SQL queries in different ways. + +In the following query, we call the classifier on every image in the video. The output of the function is stored in the ``label`` column (i.e., the digit associated with the given frame) of the output ``DataFrame``. + +.. code-block:: sql + + SELECT MnistImageClassifier(data).label + FROM mnist_video; + +>>>>>>> 53dafecf (feat: sync master staging (#1050)) >>>>>>> 2dacff69 (feat: sync master staging (#1050)) This query returns the label of all the images: @@ -179,8 +243,11 @@ This query returns the label of all the images: <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) +------------------------------+ ======= @@ -212,11 +279,17 @@ Like normal SQL, you can also specify conditions to filter out some frames of th WHERE id < 2 >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) <<<<<<< HEAD +<<<<<<< HEAD ======= +------------------------------+ >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +======= + +------------------------------+ +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Filtering Based on AI Function @@ -244,6 +317,7 @@ Now, the ``DataFrame`` only contains images of the digit ``6``. <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD .. include:: ../shared/footer.rst .. include:: ../shared/designs/design7.rst @@ -254,8 +328,16 @@ Check out our `Jupyter Notebook >>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. include:: ../shared/footer.rst ======= Check out our `Jupyter Notebook `_ for working example. >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +======= +.. include:: ../shared/footer.rst +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) diff --git a/docs/source/usecases/object-detection.rst b/docs/source/usecases/object-detection.rst index c2c64c76c5..0868bfb6a2 100644 --- a/docs/source/usecases/object-detection.rst +++ b/docs/source/usecases/object-detection.rst @@ -73,10 +73,13 @@ This query returns the label of all the images: .. code-block:: +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) +--------------------------+--------------------------------+--------------------------------+--------------------------------+ | objectdetectionvideos.id | yolo.labels | yolo.bboxes | yolo.scores | +--------------------------+--------------------------------+--------------------------------+--------------------------------+ @@ -87,6 +90,9 @@ This query returns the label of all the images: | 4 | ['car', 'car', 'car', 'car ... | [[843.2254028320312, 280.8 ... | [0.9, 0.85, 0.83, 0.8, 0.7 ... | +--------------------------+--------------------------------+--------------------------------+--------------------------------+ <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= +-----------------------------------------------------------------------------------------------------+ | objectdetectionvideos.id | yolo.labels | @@ -95,8 +101,11 @@ This query returns the label of all the images: | 1 | [car, car, car, car, car, car, car, car, car, ... | +-----------------------------------------------------------------------------------------------------+ >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Filtering Based on YOLO Function -------------------------------- @@ -107,6 +116,9 @@ In the following query, we use the output of the object detector to retrieve a s <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) SELECT id, Yolo(data).labels FROM ObjectDetectionVideos WHERE ['person', 'car'] <@ Yolo(data).labels @@ -116,21 +128,27 @@ In the following query, we use the output of the object detector to retrieve a s FROM ObjectDetectionVideos WHERE ['pedestrian', 'car'] <@ Yolo(data).label; >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= SELECT id, Yolo(data).labels FROM ObjectDetectionVideos WHERE ['person', 'car'] <@ Yolo(data).labels LIMIT 5; >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Now, the ``DataFrame`` only contains frames with the desired objects: .. code-block:: +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) +--------------------------+--------------------------------------------------------------+ | objectdetectionvideos.id | yolo.labels | +--------------------------+--------------------------------------------------------------+ @@ -141,6 +159,17 @@ Now, the ``DataFrame`` only contains frames with the desired objects: | 4 | ['car', 'car', 'car', 'car', 'car', 'car', 'car', 'car', ... | +--------------------------+--------------------------------------------------------------+ <<<<<<< HEAD +<<<<<<< HEAD +======= +======= + +------------------------------+ + | objectdetectionvideos.label | + |------------------------------| + | 6 | + | 6 | + +------------------------------+ +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. include:: ../shared/footer.rst diff --git a/docs/source/usecases/qa-video.rst b/docs/source/usecases/qa-video.rst deleted file mode 100644 index 133eefca24..0000000000 --- a/docs/source/usecases/qa-video.rst +++ /dev/null @@ -1,89 +0,0 @@ -Q&A Application on Videos -========================= - -1. Connect to EvaDB -------------------- - -.. code-block:: python - - import evadb - cursor = evadb.connect().cursor() - -2. Register Functions ---------------------- - -Register speech-to-text **whisper** model from `HuggingFace` - -.. code-block:: python - - cursor.query(""" - CREATE UDF SpeechRecognizer - TYPE HuggingFace - 'task' 'automatic-speech-recognition' - 'model' 'openai/whisper-base'; - """).execute() - -.. note:: - - EvaDB allows users to register any model in HuggingFace as a function. - -Register **OpenAI** LLM model - -.. code-block:: python - - cursor.query(""" - CREATE UDF ChatGPT - IMPL 'evadb/udfs/chatgpt.py' - """).execute() - - # Set OpenAI token - import os - os.environ["OPENAI_KEY"] = "sk-..." - -.. note:: - - ChatGPT function is a wrapper around OpenAI API call. You can also switch to other LLM models that can run locally. - -3. Summarize Video in Text --------------------------- - -Create a table with text summary of the video. Text summarization is generated by running speech-to-text ``Whisper`` model from ``HuggingFace``. - -.. code-block:: python - - cursor.query(""" - CREATE TABLE text_summary AS - SELECT SpeechRecognizer(audio) FROM ukraine_video; - """).execute() - -This results a table shown below. - -.. code-block:: - - +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - | text_summary.text | - |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| - | The war in Ukraine has been on for 415 days. Who is winning it? Not Russia. Certainly not Ukraine. It is the US oil companies. US oil companies have reached $200 billion in pure profits. The earnings are still on. They are still milking this war and sharing the spoils. Let us look at how Exxon mobile has been doing. In 2022, the company made $56 billion in profits. Oil companies capitalized on instability and they are profiting from pain. American oil companies are masters of this art. You may remember the war in Iraq. The US went to war in Iraq by selling a lie. The Americans did not find any weapons of mass destruction but they did find lots of oil. And in the year since, American officials have admitted this. And this story is not over. It's repeating itself in Ukraine. They are feeding another war and filling the coffers of US oil companies. | - +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - -4. Q&A using ChatGPT ---------------------- - -We can now embed the ChatGPT prompt inside SQL with text summary from the table as its knowledge base. - -.. code-block:: python - - cursor.query(""" - SELECT ChatGPT('Is this video summary related to Ukraine russia war', text) - FROM text_summary; - """).df() - -This query returns a projected ``DataFrame``. - -.. code-block:: - - +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ - | chatgpt.response | - |-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| - | Based on the provided context, it seems that the video summary is related to the Ukraine-Russia war. It discusses how US oil companies are allegedly profiting from the war in Ukraine, similar to how they allegedly benefited from the war in Iraq. | - +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ diff --git a/docs/source/usecases/question-answering.rst b/docs/source/usecases/question-answering.rst index 099dadc32c..f4b001d869 100644 --- a/docs/source/usecases/question-answering.rst +++ b/docs/source/usecases/question-answering.rst @@ -97,16 +97,22 @@ Here is the query's output ``DataFrame``: .. code-block:: +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) +-------------------------------------------------------------------------------------------------------------------------+ | text_summary.text | +-------------------------------------------------------------------------------------------------------------------------+ | The war in Ukraine has been on for 415 days. Who is winning it? Not Russia. Certainly not Ukraine. It is the US oil ... | +-------------------------------------------------------------------------------------------------------------------------+ <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | text_summary.text | @@ -114,8 +120,11 @@ Here is the query's output ``DataFrame``: | The war in Ukraine has been on for 415 days. Who is winning it? Not Russia. Certainly not Ukraine. It is the US oil companies. US oil companies have reached $200 billion in pure profits. The earnings are still on. They are still milking this war and sharing the spoils. Let us look at how Exxon mobile has been doing. In 2022, the company made $56 billion in profits. Oil companies capitalized on instability and they are profiting from pain. American oil companies are masters of this art. You may remember the war in Iraq. The US went to war in Iraq by selling a lie. The Americans did not find any weapons of mass destruction but they did find lots of oil. And in the year since, American officials have admitted this. And this story is not over. It's repeating itself in Ukraine. They are feeding another war and filling the coffers of US oil companies. | +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Question Answering using ChatGPT -------------------------------- @@ -133,16 +142,22 @@ Here is the query's output ``DataFrame``: .. code-block:: +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) +--------------------------------------------------------------------------------------------------------------------------+ | chatgpt.response | +--------------------------------------------------------------------------------------------------------------------------+ | No, the video summary provided does not appear to be related to the Ukraine-Russia war. It seems to be a conversatio ... | +--------------------------------------------------------------------------------------------------------------------------+ <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | chatgpt.response | @@ -150,8 +165,11 @@ Here is the query's output ``DataFrame``: | Based on the provided context, it seems that the video summary is related to the Ukraine-Russia war. It discusses how US oil companies are allegedly profiting from the war in Ukraine, similar to how they allegedly benefited from the war in Iraq. | +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. include:: ../shared/nlp.rst diff --git a/docs/source/usecases/sentiment-analysis.rst b/docs/source/usecases/sentiment-analysis.rst index 8e8bc13027..9afd7f226c 100644 --- a/docs/source/usecases/sentiment-analysis.rst +++ b/docs/source/usecases/sentiment-analysis.rst @@ -91,6 +91,7 @@ While running this query, EvaDB first retrieves the negative reviews and then ap | Dear [Customer's Name], Thank you for bringing this issue to our attention. We apologize for the inconvenience caused by the missing chicken sandwich in your takeout order. We understand how frustrating it can be when an item is missing from your meal. To address this concern, we ... | +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD .. include:: ../shared/nlp.rst @@ -106,3 +107,11 @@ Check out our `Jupyter Notebook >>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +.. include:: ../shared/nlp.rst + +.. include:: ../shared/footer.rst +======= +Check out our `Jupyter Notebook `__ for working example. +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) diff --git a/docs/source/usecases/similar-image-search.rst b/docs/source/usecases/similar-image-search.rst deleted file mode 100644 index 91563cc2cd..0000000000 --- a/docs/source/usecases/similar-image-search.rst +++ /dev/null @@ -1,83 +0,0 @@ -Image Similarity Search Pipeline using EvaDB on Images -====================================================== - -In this use case, we want to search similar images based on an image provided by the user. To implement this use case, we leverage EvaDB's capability of easily expressing feature extraction pipeline. Additionally, we also leverage EvaDB's capability of building a similarity search index and searching the index to -locate similar images through ``FAISS`` library. - -For this use case, we use a reddit image dataset that can be downloaded from `Here `_. -We populate a table in the database that contains all images. - -1. Connect to EvaDB -------------------- - -.. code-block:: python - - import evadb - cursor = evadb.connect().cursor() - -2. Register SIFT as Function ----------------------------- - -.. code-block:: python - - cursor.query(""" - CREATE UDF IF NOT EXISTS SiftFeatureExtractor - IMPL 'evadb/udfs/sift_feature_extractor.py' - """).execute() - -3. Search Similar Images ------------------------- - -To locate images that have similar appearance, we will first build an index based on embeddings of images. -Then, for the given image, EvaDB can find similar images by searching in the index. - -Build Index using ``FAISS`` -*************************** - -The below query creates a new index on the projected column ``SiftFeatureExtractor(data)`` from the ``reddit_dataset`` table. - -.. code-block:: python - - cursor.query(""" - CREATE INDEX reddit_sift_image_index - ON reddit_dataset (SiftFeatureExtractor(data)) - USING FAISS - """).execute() - -Search Index for a Given Image -******************************* - -EvaDB leverages the ``ORDER BY ... LIMIT ...`` SQL syntax to retrieve the top 5 similar images. -In this example, ``Similarity(x, y)`` is a built-in function to calculate distance between ``x`` and ``y``. -In current version, ``x`` is a single tuple and ``y`` is a column that contains multiple tuples. -By default EvaDB does pairwise distance calculation between ``x`` and all tuples from ``y``. -In this case, EvaDB leverages the index that we have already built. - -.. code-block:: python - - query = cursor.query(""" - SELECT name FROM reddit_dataset ORDER BY - Similarity( - SiftFeatureExtractor(Open('reddit-images/g1074_d4mxztt.jpg')), - SiftFeatureExtractor(data) - ) - LIMIT 5 - """) - query.df() - -The ``DataFrame`` contains the top 5 similar images. - -.. code-block:: - - +---------------------------------+ - | reddit_dataset.name | - |---------------------------------| - | reddit-images/g1074_d4mxztt.jpg | - | reddit-images/g348_d7ju7dq.jpg | - | reddit-images/g1209_ct6bf1n.jpg | - | reddit-images/g1190_cln9xzr.jpg | - | reddit-images/g1190_clna2x2.jpg | - +---------------------------------+ - -Check out our `Jupyter Notebook `_ for working example. -We also demonstrate more complicated features of EvaDB for similarity search. diff --git a/docs/source/usecases/text-summarization.rst b/docs/source/usecases/text-summarization.rst index eba82bb6f6..ef9b3e226c 100644 --- a/docs/source/usecases/text-summarization.rst +++ b/docs/source/usecases/text-summarization.rst @@ -85,10 +85,13 @@ Here is the query's output ``DataFrame``: .. code-block:: +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) +--------------------------------------------------------------+--------------------------------------------------------------+ | mypdfs.data | textsummarizer.summary_text | +--------------------------------------------------------------+--------------------------------------------------------------+ @@ -96,6 +99,9 @@ Here is the query's output ``DataFrame``: | PHYSICAL CHARACTERISTICS ( 1 ) COLOUR -- Red ( 2 ) ... | The temperature is 38° C / 100.4° F. The body weight is ... | +--------------------------------------------------------------+--------------------------------------------------------------+ <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= +--------------------------------------------------------+--------------------------------------------------------+ | mypdfs.data | mypdfs.summary_text | @@ -104,8 +110,11 @@ Here is the query's output ``DataFrame``: | PHYSICAL CHARACTERISTICS ( 1 ) COLOUR -- R... | The temperature is 38° C / 100.4° F. The body ... | +--------------------------------------------------------+--------------------------------------------------------+ >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) .. include:: ../shared/nlp.rst diff --git a/evadb/binder/statement_binder.py b/evadb/binder/statement_binder.py index 1b08f7c2d9..fb8deab919 100644 --- a/evadb/binder/statement_binder.py +++ b/evadb/binder/statement_binder.py @@ -41,13 +41,19 @@ from evadb.parser.create_index_statement import CreateIndexStatement <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) from evadb.parser.create_statement import ColumnDefinition, CreateTableStatement ======= from evadb.parser.create_statement import CreateTableStatement >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= from evadb.parser.create_statement import ColumnDefinition, CreateTableStatement >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) from evadb.parser.delete_statement import DeleteTableStatement from evadb.parser.explain_statement import ExplainStatement from evadb.parser.rename_statement import RenameTableStatement @@ -58,18 +64,27 @@ from evadb.third_party.huggingface.binder import assign_hf_function <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) from evadb.utils.generic_utils import ( load_function_class_from_file, string_comparison_case_insensitive, ) <<<<<<< HEAD +<<<<<<< HEAD ======= from evadb.utils.generic_utils import load_function_class_from_file >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +from evadb.utils.generic_utils import load_function_class_from_file +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) from evadb.utils.logging_manager import logger @@ -109,6 +124,9 @@ def _bind_create_function_statement(self, node: CreateFunctionStatement): arg_map = {key: value for key, value in node.metadata} <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) inputs, outputs = [], [] if string_comparison_case_insensitive(node.function_type, "ludwig"): assert ( @@ -216,7 +234,27 @@ def _bind_create_function_statement(self, node: CreateFunctionStatement): raise BinderError( f"Unsupported type of function: {node.function_type}." ) +<<<<<<< HEAD >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= + assert ( + "predict" in arg_map + ), f"Creating {node.function_type} functions expects 'predict' metadata." + # We only support a single predict column for now + predict_columns = set([arg_map["predict"]]) + inputs, outputs = [], [] + for column in all_column_list: + if column.name in predict_columns: + if node.function_type != "Forecasting": + column.name = column.name + "_predictions" + else: + column.name = column.name + outputs.append(column) + else: + inputs.append(column) +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) assert ( len(node.inputs) == 0 and len(node.outputs) == 0 ), f"{node.function_type} functions' input and output are auto assigned" @@ -224,6 +262,9 @@ def _bind_create_function_statement(self, node: CreateFunctionStatement): <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= @bind.register(CreateIndexStatement) def _bind_create_index_statement(self, node: CreateIndexStatement): @@ -270,8 +311,11 @@ def _bind_create_index_statement(self, node: CreateIndexStatement): ), "Index input needs to be 2 dimensional." >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) @bind.register(SelectStatement) def _bind_select_statement(self, node: SelectStatement): if node.from_table: @@ -431,6 +475,9 @@ def _bind_func_expr(self, node: FunctionExpression): <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) if string_comparison_case_insensitive(function_obj.type, "HuggingFace"): node.function = assign_hf_function(function_obj) @@ -441,12 +488,15 @@ def _bind_func_expr(self, node: FunctionExpression): elif function_obj.type == "Ludwig": >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= if string_comparison_case_insensitive(function_obj.type, "HuggingFace"): node.function = assign_hf_function(function_obj) elif string_comparison_case_insensitive(function_obj.type, "Ludwig"): >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) function_class = load_function_class_from_file( function_obj.impl_file_path, "GenericLudwigModel", diff --git a/evadb/executor/create_function_executor.py b/evadb/executor/create_function_executor.py index dd40e0e6d6..7c67cdd4a0 100644 --- a/evadb/executor/create_function_executor.py +++ b/evadb/executor/create_function_executor.py @@ -14,17 +14,23 @@ # limitations under the License. <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) import hashlib import os import pickle ======= import os >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= import hashlib import os import pickle >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) from pathlib import Path from typing import Dict, List @@ -53,11 +59,20 @@ load_function_class_from_file, <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) string_comparison_case_insensitive, try_to_import_ludwig, try_to_import_neuralforecast, try_to_import_sklearn, +<<<<<<< HEAD try_to_import_statsforecast, +======= +======= + try_to_import_ludwig, +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) try_to_import_torch, try_to_import_ultralytics, try_to_import_xgboost, @@ -84,6 +99,9 @@ def __init__(self, db: EvaDBDatabase, node: CreateFunctionPlan): def handle_huggingface_function(self): <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) """Handle HuggingFace functions HuggingFace functions are special functions that are not loaded from a file. @@ -94,12 +112,15 @@ def handle_huggingface_function(self): HuggingFace Functions are special Functions that are not loaded from a file. So we do not need to call the setup method on them like we do for other Functions. >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= """Handle HuggingFace functions HuggingFace functions are special functions that are not loaded from a file. So we do not need to call the setup method on them like we do for other functions. >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) """ # We need at least one deep learning framework for HuggingFace # Torch or Tensorflow @@ -117,6 +138,9 @@ def handle_huggingface_function(self): def handle_ludwig_function(self): <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) """Handle ludwig functions Use Ludwig's auto_train engine to train/tune models. @@ -125,11 +149,14 @@ def handle_ludwig_function(self): Use ludwig's auto_train engine to train/tune models. >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= """Handle ludwig functions Use Ludwig's auto_train engine to train/tune models. >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) """ try_to_import_ludwig() from ludwig.automl import auto_train @@ -173,10 +200,13 @@ def handle_ludwig_function(self): self.node.metadata, ) +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def handle_sklearn_function(self): """Handle sklearn functions @@ -290,10 +320,13 @@ def handle_ultralytics_function(self): def handle_ultralytics_function(self): """Handle Ultralytics Functions""" >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= def handle_ultralytics_function(self): """Handle Ultralytics functions""" >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) try_to_import_ultralytics() impl_path = ( @@ -311,10 +344,13 @@ def handle_ultralytics_function(self): self.node.metadata, ) +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def handle_forecasting_function(self): """Handle forecasting functions""" os.environ["CUDA_VISIBLE_DEVICES"] = "" @@ -576,6 +612,7 @@ def handle_generic_function(self): Generic Functions are loaded from a file. We check for inputs passed by the user during CREATE or try to load io from decorators. >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= arg_map = {arg.key: arg.value for arg in self.node.metadata} @@ -823,6 +860,8 @@ def handle_generic_function(self): Generic functions are loaded from a file. We check for inputs passed by the user during CREATE or try to load io from decorators. >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) """ impl_path = self.node.impl_path.absolute().as_posix() function = self._try_initializing_function(impl_path) @@ -843,8 +882,11 @@ def exec(self, *args, **kwargs): """ <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) assert ( self.node.if_not_exists and self.node.or_replace ) is False, ( @@ -853,10 +895,15 @@ def exec(self, *args, **kwargs): overwrite = False <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) # check catalog if it already has this function entry if self.catalog().get_function_catalog_entry_by_name(self.node.name): if self.node.if_not_exists: @@ -865,8 +912,11 @@ def exec(self, *args, **kwargs): return <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) elif self.node.or_replace: # We use DropObjectExecutor to avoid bookkeeping the code. The drop function should be moved to catalog. from evadb.executor.drop_object_executor import DropObjectExecutor @@ -885,10 +935,15 @@ def exec(self, *args, **kwargs): else: overwrite = True <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) else: msg = f"Function {self.node.name} already exists." logger.error(msg) @@ -897,13 +952,19 @@ def exec(self, *args, **kwargs): # if it's a type of HuggingFaceModel, override the impl_path <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) if string_comparison_case_insensitive(self.node.function_type, "HuggingFace"): ======= if self.node.function_type == "HuggingFace": >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= if string_comparison_case_insensitive(self.node.function_type, "HuggingFace"): >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ( name, impl_path, @@ -913,13 +974,19 @@ def exec(self, *args, **kwargs): ) = self.handle_huggingface_function() <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) elif string_comparison_case_insensitive(self.node.function_type, "ultralytics"): ======= elif self.node.function_type == "ultralytics": >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= elif string_comparison_case_insensitive(self.node.function_type, "ultralytics"): >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ( name, impl_path, @@ -929,13 +996,19 @@ def exec(self, *args, **kwargs): ) = self.handle_ultralytics_function() <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) elif string_comparison_case_insensitive(self.node.function_type, "Ludwig"): ======= elif self.node.function_type == "Ludwig": >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= elif string_comparison_case_insensitive(self.node.function_type, "Ludwig"): >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ( name, impl_path, @@ -945,8 +1018,11 @@ def exec(self, *args, **kwargs): ) = self.handle_ludwig_function() <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) elif string_comparison_case_insensitive(self.node.function_type, "Sklearn"): ( name, @@ -975,10 +1051,15 @@ def exec(self, *args, **kwargs): metadata, ) = self.handle_forecasting_function() <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) else: ( name, @@ -993,8 +1074,11 @@ def exec(self, *args, **kwargs): ) <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) if overwrite: msg = f"Function {self.node.name} overwritten." @@ -1002,6 +1086,9 @@ def exec(self, *args, **kwargs): msg = f"Function {self.node.name} added to the database." yield Batch(pd.DataFrame([msg])) <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= yield Batch( pd.DataFrame( @@ -1009,8 +1096,11 @@ def exec(self, *args, **kwargs): ) ) >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def _try_initializing_function( self, impl_path: str, function_args: Dict = {} @@ -1021,6 +1111,9 @@ def _try_initializing_function( impl_path (str): The file path of the function implementation file. <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) function_args (Dict, optional): Dictionary of arguments to pass to the function. Defaults to {}. Returns: @@ -1030,6 +1123,7 @@ def _try_initializing_function( RuntimeError: If an error occurs while initializing the function. ======= function_args (Dict, optional): Dictionary of arguments to pass to the Function. Defaults to {}. +<<<<<<< HEAD ======= function_args (Dict, optional): Dictionary of arguments to pass to the function. Defaults to {}. >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) @@ -1044,6 +1138,15 @@ def _try_initializing_function( ======= RuntimeError: If an error occurs while initializing the function. >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= + + Returns: + FunctionCatalogEntry: A FunctionCatalogEntry object that represents the initialized Function. + + Raises: + RuntimeError: If an error occurs while initializing the Function. +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) """ # load the function class from the file @@ -1055,13 +1158,19 @@ def _try_initializing_function( except Exception as e: <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) err_msg = f"Error creating function {self.node.name}: {str(e)}" ======= err_msg = f"Error creating Function: {str(e)}" >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= err_msg = f"Error creating function {self.node.name}: {str(e)}" >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) # logger.error(err_msg) raise RuntimeError(err_msg) @@ -1072,13 +1181,19 @@ def _resolve_function_io( ) -> List[FunctionIOCatalogEntry]: <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) """Private method that resolves the input/output definitions for a given function. ======= """Private method that resolves the input/output definitions for a given Function. >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= """Private method that resolves the input/output definitions for a given function. >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) It first searches for the input/outputs in the CREATE statement. If not found, it resolves them using decorators. If not found there as well, it raises an error. Args: @@ -1088,13 +1203,19 @@ def _resolve_function_io( A List of FunctionIOCatalogEntry objects that represent the resolved input and <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) output definitions for the function. ======= output definitions for the Function. >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= output definitions for the function. >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Raises: RuntimeError: If an error occurs while resolving the function input/output @@ -1122,13 +1243,19 @@ def _resolve_function_io( err_msg = ( <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) f"Error creating function, input/output definition incorrect: {str(e)}" ======= f"Error creating Function, input/output definition incorrect: {str(e)}" >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= f"Error creating function, input/output definition incorrect: {str(e)}" >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ) logger.error(err_msg) raise RuntimeError(err_msg) diff --git a/evadb/executor/create_udf_executor.py b/evadb/executor/create_udf_executor.py index 97c937c5b7..8772379dea 100644 --- a/evadb/executor/create_udf_executor.py +++ b/evadb/executor/create_udf_executor.py @@ -13,12 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. <<<<<<< HEAD +<<<<<<< HEAD import hashlib import os import pickle ======= import os >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +import hashlib +import os +import pickle +>>>>>>> 53dafecf (feat: sync master staging (#1050)) from pathlib import Path from typing import Dict, List @@ -41,10 +47,14 @@ from evadb.utils.errors import UDFIODefinitionError from evadb.utils.generic_utils import ( load_udf_class_from_file, +<<<<<<< HEAD <<<<<<< HEAD try_to_import_forecast, ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= + try_to_import_forecast, +>>>>>>> 53dafecf (feat: sync master staging (#1050)) try_to_import_ludwig, try_to_import_torch, try_to_import_ultralytics, @@ -79,11 +89,15 @@ def handle_huggingface_udf(self): def handle_ludwig_udf(self): """Handle ludwig UDFs +<<<<<<< HEAD <<<<<<< HEAD Use Ludwig's auto_train engine to train/tune models. ======= Use ludwig's auto_train engine to train/tune models. >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= + Use Ludwig's auto_train engine to train/tune models. +>>>>>>> 53dafecf (feat: sync master staging (#1050)) """ try_to_import_ludwig() from ludwig.automl import auto_train @@ -143,6 +157,9 @@ def handle_ultralytics_udf(self): ) <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def handle_forecasting_udf(self): """Handle forecasting UDFs""" aggregated_batch_list = [] @@ -256,8 +273,11 @@ def handle_forecasting_udf(self): metadata_here, ) +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def handle_generic_udf(self): """Handle generic UDFs @@ -298,11 +318,16 @@ def exec(self, *args, **kwargs): name, impl_path, udf_type, io_list, metadata = self.handle_ultralytics_udf() elif self.node.udf_type == "Ludwig": name, impl_path, udf_type, io_list, metadata = self.handle_ludwig_udf() +<<<<<<< HEAD <<<<<<< HEAD elif self.node.udf_type == "Forecasting": name, impl_path, udf_type, io_list, metadata = self.handle_forecasting_udf() ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= + elif self.node.udf_type == "Forecasting": + name, impl_path, udf_type, io_list, metadata = self.handle_forecasting_udf() +>>>>>>> 53dafecf (feat: sync master staging (#1050)) else: name, impl_path, udf_type, io_list, metadata = self.handle_generic_udf() diff --git a/evadb/executor/drop_object_executor.py b/evadb/executor/drop_object_executor.py index 774c93095c..10d4a78e58 100644 --- a/evadb/executor/drop_object_executor.py +++ b/evadb/executor/drop_object_executor.py @@ -42,15 +42,20 @@ def exec(self, *args, **kwargs): elif self.node.object_type == ObjectType.FUNCTION: yield self._handle_drop_function(self.node.name, self.node.if_exists) +<<<<<<< HEAD <<<<<<< HEAD elif self.node.object_type == ObjectType.DATABASE: yield self._handle_drop_database(self.node.name, self.node.if_exists) ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) elif self.node.object_type == ObjectType.DATABASE: yield self._handle_drop_database(self.node.name, self.node.if_exists) +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) def _handle_drop_table(self, table_name: str, if_exists: bool): if not self.catalog().check_table_exists(table_name): diff --git a/evadb/functions/chatgpt.py b/evadb/functions/chatgpt.py index 3ac76e9a10..c4f42796ac 100644 --- a/evadb/functions/chatgpt.py +++ b/evadb/functions/chatgpt.py @@ -82,13 +82,19 @@ def name(self) -> str: <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) @setup(cacheable=True, function_type="chat-completion", batchable=True) ======= @setup(cacheable=False, function_type="chat-completion", batchable=True) >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= @setup(cacheable=True, function_type="chat-completion", batchable=True) >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def setup( self, model="gpt-3.5-turbo", diff --git a/evadb/functions/forecast.py b/evadb/functions/forecast.py index 3428b07236..6ae6f80f8c 100644 --- a/evadb/functions/forecast.py +++ b/evadb/functions/forecast.py @@ -20,6 +20,9 @@ <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) from evadb.functions.abstract.abstract_function import AbstractFunction from evadb.functions.decorators.decorators import setup @@ -34,6 +37,7 @@ class ForecastModel(AbstractFunction): class ForecastModel(AbstractUDF): >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= from evadb.functions.abstract.abstract_function import AbstractFunction from evadb.functions.decorators.decorators import setup @@ -41,14 +45,19 @@ class ForecastModel(AbstractUDF): class ForecastModel(AbstractFunction): >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) @property def name(self) -> str: return "ForecastModel" +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) @setup(cacheable=False, function_type="Forecasting", batchable=True) def setup( self, @@ -61,12 +70,18 @@ def setup( library: str, ): <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= @setup(cacheable=False, udf_type="Forecasting", batchable=True) def setup(self, model_name: str, model_path: str): >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) f = open(model_path, "rb") loaded_model = pickle.load(f) f.close() @@ -74,8 +89,11 @@ def setup(self, model_name: str, model_path: str): self.model_name = model_name <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) self.predict_column_rename = predict_column_rename self.time_column_rename = time_column_rename self.id_column_rename = id_column_rename @@ -83,11 +101,35 @@ def setup(self, model_name: str, model_path: str): self.library = library <<<<<<< HEAD +======= + + @forward( + input_signatures=[], + output_signatures=[ + PandasDataframe( + columns=["y"], + column_types=[ + NdArrayType.FLOAT32, + ], + column_shapes=[(None,)], + ) + ], + ) +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) def forward(self, data) -> pd.DataFrame: +<<<<<<< HEAD if self.library == "statsforecast": forecast_df = self.model.predict(h=self.horizon) else: forecast_df = self.model.predict() +======= + horizon = list(data.iloc[:, -1])[0] + assert ( + type(horizon) is int + ), "Forecast UDF expects integral horizon in parameter." + forecast_df = self.model.predict(h=horizon) +<<<<<<< HEAD +>>>>>>> 53dafecf (feat: sync master staging (#1050)) forecast_df.reset_index(inplace=True) forecast_df = forecast_df.rename( columns={ @@ -98,6 +140,7 @@ def forward(self, data) -> pd.DataFrame: )[: self.horizon * forecast_df["unique_id"].nunique()] return forecast_df ======= +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) @@ -108,6 +151,8 @@ def forward(self, data) -> pd.DataFrame: type(horizon) is int ), "Forecast UDF expects integral horizon in parameter." forecast_df = self.model.predict(h=horizon) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) forecast_df = forecast_df.rename(columns={self.model_name: "y"}) return pd.DataFrame( forecast_df, @@ -116,6 +161,7 @@ def forward(self, data) -> pd.DataFrame: ], ) >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= if self.library == "statsforecast": forecast_df = self.model.predict(h=self.horizon) @@ -131,3 +177,5 @@ def forward(self, data) -> pd.DataFrame: )[: self.horizon * forecast_df["unique_id"].nunique()] return forecast_df >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) diff --git a/evadb/functions/function_bootstrap_queries.py b/evadb/functions/function_bootstrap_queries.py index c44fc3d73e..ad58fef6b0 100644 --- a/evadb/functions/function_bootstrap_queries.py +++ b/evadb/functions/function_bootstrap_queries.py @@ -49,10 +49,13 @@ EvaDB_INSTALLATION_DIR ) +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) DummyNoInputFunction_function_query = """CREATE FUNCTION IF NOT EXISTS DummyNoInputFunction IMPL '{}/../test/util.py'; @@ -67,11 +70,16 @@ EvaDB_INSTALLATION_DIR ) +<<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) fuzzy_function_query = """CREATE FUNCTION IF NOT EXISTS FuzzDistance INPUT (Input_Array1 NDARRAY ANYTYPE, Input_Array2 NDARRAY ANYTYPE) OUTPUT (distance FLOAT(32, 7)) @@ -240,13 +248,19 @@ def init_builtin_functions(db: EvaDBDatabase, mode: str = "debug") -> None: <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Args:G ======= Args: >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= Args:G >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) mode (str, optional): The mode for loading functions, either 'debug' or 'release'. Defaults to 'debug'. @@ -298,14 +312,20 @@ def init_builtin_functions(db: EvaDBDatabase, mode: str = "debug") -> None: DummyFeatureExtractor_function_query, <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) DummyNoInputFunction_function_query, DummyLLM_function_query, ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= DummyNoInputFunction_function_query, DummyLLM_function_query, >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ] ) diff --git a/evadb/optimizer/operators.py b/evadb/optimizer/operators.py index 4684082cc9..7184d071fa 100644 --- a/evadb/optimizer/operators.py +++ b/evadb/optimizer/operators.py @@ -24,12 +24,18 @@ from evadb.catalog.models.table_catalog import TableCatalogEntry <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD from evadb.catalog.models.utils import IndexCatalogEntry ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= from evadb.catalog.models.utils import IndexCatalogEntry >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +from evadb.catalog.models.utils import IndexCatalogEntry +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) from evadb.expression.abstract_expression import AbstractExpression from evadb.expression.constant_value_expression import ConstantValueExpression from evadb.expression.function_expression import FunctionExpression @@ -651,6 +657,9 @@ class LogicalCreateFunction(Operator): function_name provided by the user required <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) or_replace: bool if true should overwrite if function with same name exists if_not_exists: bool @@ -660,12 +669,15 @@ class LogicalCreateFunction(Operator): if true should throw an error if function with same name exists else will replace the existing >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= or_replace: bool if true should overwrite if function with same name exists if_not_exists: bool if true should skip if function with same name exists >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) inputs: List[FunctionIOCatalogEntry] function inputs, annotated list similar to table columns outputs: List[FunctionIOCatalogEntry] diff --git a/evadb/optimizer/optimizer_utils.py b/evadb/optimizer/optimizer_utils.py index 4d3f94a8d3..0604755d5b 100644 --- a/evadb/optimizer/optimizer_utils.py +++ b/evadb/optimizer/optimizer_utils.py @@ -309,8 +309,11 @@ def enable_cache_on_expression_tree( def check_expr_validity_for_cache(expr: FunctionExpression): <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) valid = expr.name in CACHEABLE_FUNCTIONS and not expr.has_cache() if len(expr.children) == 1: # Normal function that only takes one parameter. @@ -322,6 +325,9 @@ def check_expr_validity_for_cache(expr: FunctionExpression): ) return valid <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= return ( expr.name in CACHEABLE_FUNCTIONS @@ -330,8 +336,11 @@ def check_expr_validity_for_cache(expr: FunctionExpression): and isinstance(expr.children[0], TupleValueExpression) ) >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def get_expression_execution_cost( diff --git a/evadb/optimizer/rules/rules.py b/evadb/optimizer/rules/rules.py index 1184fb36ac..88a4a6271a 100644 --- a/evadb/optimizer/rules/rules.py +++ b/evadb/optimizer/rules/rules.py @@ -20,12 +20,18 @@ from evadb.catalog.catalog_utils import is_video_table <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD from evadb.catalog.models.utils import IndexCatalogEntry ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= from evadb.catalog.models.utils import IndexCatalogEntry >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +from evadb.catalog.models.utils import IndexCatalogEntry +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) from evadb.constants import CACHEABLE_FUNCTIONS from evadb.executor.execution_context import Context from evadb.expression.expression_utils import ( @@ -580,6 +586,9 @@ def _exists_predicate(opr): column_catalog_entry = tv_expr.col_object <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) # Only check the index existence when building on EvaDB data. if not is_postgres_data_source: @@ -609,6 +618,7 @@ def _exists_predicate(opr): if isinstance(base_func_expr, TupleValueExpression) else base_func_expr.signature() ) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) @@ -619,6 +629,15 @@ def _exists_predicate(opr): None if isinstance(base_func_expr, TupleValueExpression) else base_func_expr.signature() +======= + + # Get index catalog. Check if an index exists for matching + # function signature and table columns. + index_catalog_entry = ( + catalog_manager().get_index_catalog_entry_by_column_and_function_signature( + column_catalog_entry, function_signature +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ) # Get index catalog. Check if an index exists for matching diff --git a/evadb/optimizer/statement_to_opr_converter.py b/evadb/optimizer/statement_to_opr_converter.py index 5d344ac5c0..c30be39722 100644 --- a/evadb/optimizer/statement_to_opr_converter.py +++ b/evadb/optimizer/statement_to_opr_converter.py @@ -57,16 +57,22 @@ from evadb.parser.statement import AbstractStatement <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) from evadb.parser.table_ref import JoinNode, TableRef, TableValuedExpression from evadb.parser.types import FunctionType, JoinType ======= from evadb.parser.table_ref import TableRef from evadb.parser.types import FunctionType >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= from evadb.parser.table_ref import JoinNode, TableRef, TableValuedExpression from evadb.parser.types import FunctionType, JoinType >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) from evadb.utils.logging_manager import logger diff --git a/evadb/parser/create_function_statement.py b/evadb/parser/create_function_statement.py index a358f081db..f47c5cce11 100644 --- a/evadb/parser/create_function_statement.py +++ b/evadb/parser/create_function_statement.py @@ -72,8 +72,11 @@ def __init__( def __str__(self) -> str: <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) s = "CREATE" if self._or_replace: @@ -81,11 +84,17 @@ def __str__(self) -> str: s += " " + "FUNCTION" <<<<<<< HEAD +<<<<<<< HEAD ======= s = "CREATE FUNCTION" >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= + s = "CREATE FUNCTION" +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) if self._if_not_exists: s += " IF NOT EXISTS" diff --git a/evadb/parser/evadb.lark b/evadb/parser/evadb.lark index 3ea2e6bfa9..364b4a6433 100644 --- a/evadb/parser/evadb.lark +++ b/evadb/parser/evadb.lark @@ -37,21 +37,30 @@ rename_table: RENAME TABLE table_name TO table_name // Create Functions <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) create_function: CREATE or_replace? FUNCTION if_not_exists? function_name INPUT create_definitions OUTPUT create_definitions TYPE function_type IMPL function_impl function_metadata* | CREATE or_replace? FUNCTION if_not_exists? function_name IMPL function_impl function_metadata* | CREATE or_replace? FUNCTION if_not_exists? function_name TYPE function_type function_metadata* | CREATE or_replace? FUNCTION if_not_exists? function_name FROM LR_BRACKET select_statement RR_BRACKET TYPE function_type function_metadata* <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= create_function: CREATE FUNCTION if_not_exists? function_name INPUT create_definitions OUTPUT create_definitions TYPE function_type IMPL function_impl function_metadata* | CREATE FUNCTION if_not_exists? function_name IMPL function_impl function_metadata* | CREATE FUNCTION if_not_exists? function_name TYPE function_type function_metadata* | CREATE FUNCTION if_not_exists? function_name FROM LR_BRACKET select_statement RR_BRACKET TYPE function_type function_metadata* >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) // Details function_name: uid @@ -301,13 +310,19 @@ function_call: function ->function_call <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) function: simple_id "(" (STAR | function_args)? ")" dotted_id? ======= function: simple_id "(" (STAR | function_args) ")" dotted_id? >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= function: simple_id "(" (STAR | function_args)? ")" dotted_id? >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) aggregate_windowed_function: aggregate_function_name "(" function_arg ")" | COUNT "(" (STAR | function_arg) ")" diff --git a/evadb/parser/lark_visitor/_create_statements.py b/evadb/parser/lark_visitor/_create_statements.py index 6a309b8af8..e683216c16 100644 --- a/evadb/parser/lark_visitor/_create_statements.py +++ b/evadb/parser/lark_visitor/_create_statements.py @@ -295,6 +295,7 @@ def create_index(self, tree): return CreateIndexStatement( <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) index_name, @@ -309,6 +310,12 @@ def create_index(self, tree): >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= + index_name, if_not_exists, table_ref, col_list, vector_store_type, function +======= + index_name, table_ref, col_list, vector_store_type, function +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ) def vector_store_type(self, tree): diff --git a/evadb/parser/lark_visitor/_drop_statement.py b/evadb/parser/lark_visitor/_drop_statement.py index ddbdc7fbdb..d421adb770 100644 --- a/evadb/parser/lark_visitor/_drop_statement.py +++ b/evadb/parser/lark_visitor/_drop_statement.py @@ -61,8 +61,11 @@ def drop_function(self, tree): return DropObjectStatement(ObjectType.FUNCTION, function_name, if_exists) <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) # Drop Database def drop_database(self, tree): @@ -78,7 +81,12 @@ def drop_database(self, tree): return DropObjectStatement(ObjectType.DATABASE, database_name, if_exists) <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) diff --git a/evadb/parser/lark_visitor/_functions.py b/evadb/parser/lark_visitor/_functions.py index 19ff712a1a..b84d26f304 100644 --- a/evadb/parser/lark_visitor/_functions.py +++ b/evadb/parser/lark_visitor/_functions.py @@ -32,13 +32,19 @@ def function(self, tree): function_output = None <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) function_args = [] ======= function_args = None >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= function_args = [] >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) for child in tree.children: if isinstance(child, Token): @@ -69,6 +75,7 @@ def function_args(self, tree): def create_function(self, tree): function_name = None <<<<<<< HEAD +<<<<<<< HEAD <<<<<<< HEAD or_replace = False ======= @@ -76,6 +83,11 @@ def create_function(self, tree): ======= or_replace = False >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= + or_replace = False +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) if_not_exists = False input_definitions = [] output_definitions = [] @@ -91,14 +103,20 @@ def create_function(self, tree): function_name = self.visit(child) <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) elif child.data == "or_replace": or_replace = True ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= elif child.data == "or_replace": or_replace = True >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) elif child.data == "if_not_exists": if_not_exists = True elif child.data == "create_definitions": @@ -130,6 +148,7 @@ def create_function(self, tree): return CreateFunctionStatement( function_name, <<<<<<< HEAD +<<<<<<< HEAD <<<<<<< HEAD or_replace, ======= @@ -137,6 +156,11 @@ def create_function(self, tree): ======= or_replace, >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= + or_replace, +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) if_not_exists, impl_path, input_definitions, diff --git a/evadb/plan_nodes/create_function_plan.py b/evadb/plan_nodes/create_function_plan.py index 7e59d919b2..4096f23458 100644 --- a/evadb/plan_nodes/create_function_plan.py +++ b/evadb/plan_nodes/create_function_plan.py @@ -30,6 +30,9 @@ class CreateFunctionPlan(AbstractPlan): function_name provided by the user required <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) or_replace: bool if true should overwrite if function with same name exists if_not_exists: bool @@ -39,12 +42,15 @@ class CreateFunctionPlan(AbstractPlan): if true should throw an error if function with same name exists else will replace the existing >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= or_replace: bool if true should overwrite if function with same name exists if_not_exists: bool if true should skip if function with same name exists >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) inputs: List[FunctionIOCatalogEntry] function inputs, annotated list similar to table columns outputs: List[FunctionIOCatalogEntry] @@ -111,6 +117,7 @@ def metadata(self): def __str__(self): return "CreateFunctionPlan(name={}, \ <<<<<<< HEAD +<<<<<<< HEAD <<<<<<< HEAD or_replace={}, \ ======= @@ -118,6 +125,11 @@ def __str__(self): ======= or_replace={}, \ >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= + or_replace={}, \ +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) if_not_exists={}, \ inputs={}, \ outputs={}, \ diff --git a/evadb/third_party/databases/mariadb/__init__.py b/evadb/third_party/databases/mariadb/__init__.py index 4a840209af..ad80e49fac 100644 --- a/evadb/third_party/databases/mariadb/__init__.py +++ b/evadb/third_party/databases/mariadb/__init__.py @@ -12,4 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +<<<<<<<< HEAD:evadb/third_party/databases/mariadb/__init__.py """mariadb integrations""" +======== +"""user defined test functions operating on ndarrays functions""" +>>>>>>>> 2dacff69 (feat: sync master staging (#1050)):test/integration_tests/long/functions/ndarray/__init__.py diff --git a/evadb/utils/generic_utils.py b/evadb/utils/generic_utils.py index 9c3d755491..ea74611bf1 100644 --- a/evadb/utils/generic_utils.py +++ b/evadb/utils/generic_utils.py @@ -379,6 +379,7 @@ def is_forecast_available() -> bool: return False +<<<<<<< HEAD def try_to_import_sklearn(): try: import sklearn # noqa: F401 @@ -423,6 +424,8 @@ def is_xgboost_available() -> bool: return False +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) ############################## ## VISION ############################## diff --git a/script/formatting/spelling.txt b/script/formatting/spelling.txt index f7ba8e9ba6..466350751e 100644 --- a/script/formatting/spelling.txt +++ b/script/formatting/spelling.txt @@ -1,6 +1,7 @@ <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD personal_ws-1.1 en 1776 ======= personal_ws-1.1 en 1467 @@ -9,11 +10,19 @@ personal_ws-1.1 en 1467 personal_ws-1.1 en 1563 >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) personal_ws-1.1 en 1563 ======= personal_ws-1.1 en 1467 >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +<<<<<<< HEAD >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +======= +personal_ws-1.1 en 1563 +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ABCD ABCMeta ANYDIM @@ -621,6 +630,7 @@ MydbHandler <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD MysqlHandler NBEATS NCHAR @@ -634,6 +644,14 @@ MysqlHandler ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +MysqlHandler +======= +>>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +======= +MysqlHandler +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) NEQ NHITS NLP @@ -817,19 +835,28 @@ SamplePlan <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) SampleTable SampleVideoTable ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= SampleTable SampleVideoTable >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) Scalability ScanPlan SchemaUtils @@ -1316,17 +1343,23 @@ feedstock <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) fetchall ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) ======= fetchall >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= fetchall ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ffill ffmpeg fileFormat @@ -1721,18 +1754,24 @@ sk <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD sklearn +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) smallint ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) ======= smallint >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= smallint ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) smi softmax spacy diff --git a/setup.py b/setup.py index 930159f90b..fd31de6d31 100644 --- a/setup.py +++ b/setup.py @@ -158,6 +158,10 @@ def read(path, encoding="utf-8"): ======= >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) +forecasting_libs = [ + "statsforecast" # MODEL TRAIN AND FINE TUNING +] + ### NEEDED FOR DEVELOPER TESTING ONLY dev_libs = [ @@ -198,6 +202,7 @@ def read(path, encoding="utf-8"): "postgres": postgres_libs, "ludwig": ludwig_libs, <<<<<<< HEAD +<<<<<<< HEAD <<<<<<< HEAD "sklearn": sklearn_libs, "xgboost": xgboost_libs, @@ -210,11 +215,21 @@ def read(path, encoding="utf-8"): "dev": dev_libs + vision_libs + document_libs + function_libs + notebook_libs + forecasting_libs, >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) "sklearn": sklearn_libs, "forecasting": forecasting_libs, # everything except ray, qdrant, ludwig and postgres. The first three fail on pyhton 3.11. "dev": dev_libs + vision_libs + document_libs + function_libs + notebook_libs + forecasting_libs + sklearn_libs, +<<<<<<< HEAD >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= + "forecasting": forecasting_libs, + # everything except ray, qdrant, ludwig and postgres. The first three fail on pyhton 3.11. + "dev": dev_libs + vision_libs + document_libs + function_libs + notebook_libs + forecasting_libs, +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) } setup( diff --git a/test/integration_tests/long/test_function_executor.py b/test/integration_tests/long/test_function_executor.py index da7fa2d927..84e8ed99ca 100644 --- a/test/integration_tests/long/test_function_executor.py +++ b/test/integration_tests/long/test_function_executor.py @@ -167,10 +167,13 @@ def test_create_function(self): ) self.assertEqual(actual, expected) +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def test_create_or_replace(self): function_name = "DummyObjectDetector" execute_query_fetch_all(self.evadb, f"DROP FUNCTION IF EXISTS {function_name};") @@ -195,11 +198,16 @@ def test_create_or_replace(self): expected = Batch(pd.DataFrame([f"Function {function_name} overwritten."])) self.assertEqual(actual, expected) +<<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def test_should_create_function_with_metadata(self): function_name = "DummyObjectDetector" execute_query_fetch_all(self.evadb, f"DROP FUNCTION {function_name};") diff --git a/test/integration_tests/long/test_model_forecasting.py b/test/integration_tests/long/test_model_forecasting.py index 91fd825265..4aef36b5d6 100644 --- a/test/integration_tests/long/test_model_forecasting.py +++ b/test/integration_tests/long/test_model_forecasting.py @@ -37,6 +37,7 @@ def setUpClass(cls): y INTEGER);""" execute_query_fetch_all(cls.evadb, create_table_query) +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= @@ -50,6 +51,8 @@ def setUpClass(cls): ylagged INTEGER);""" execute_query_fetch_all(cls.evadb, create_table_query) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) create_table_query = """ CREATE TABLE HomeData (\ saledate TEXT(30),\ @@ -58,15 +61,21 @@ def setUpClass(cls): bedrooms INTEGER);""" execute_query_fetch_all(cls.evadb, create_table_query) +<<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) path = f"{EvaDB_ROOT_DIR}/data/forecasting/air-passengers.csv" load_query = f"LOAD CSV '{path}' INTO AirData;" execute_query_fetch_all(cls.evadb, load_query) +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= @@ -75,15 +84,22 @@ def setUpClass(cls): load_query = f"LOAD CSV '{path}' INTO AirDataPanel;" execute_query_fetch_all(cls.evadb, load_query) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) path = f"{EvaDB_ROOT_DIR}/data/forecasting/home_sales.csv" load_query = f"LOAD CSV '{path}' INTO HomeData;" execute_query_fetch_all(cls.evadb, load_query) +<<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) @classmethod def tearDownClass(cls): shutdown_ray() @@ -91,26 +107,41 @@ def tearDownClass(cls): # clean up <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) execute_query_fetch_all(cls.evadb, "DROP TABLE IF EXISTS AirData;") execute_query_fetch_all(cls.evadb, "DROP TABLE IF EXISTS HomeData;") execute_query_fetch_all(cls.evadb, "DROP FUNCTION IF EXISTS AirForecast;") execute_query_fetch_all(cls.evadb, "DROP FUNCTION IF EXISTS HomeForecast;") <<<<<<< HEAD +<<<<<<< HEAD ======= execute_query_fetch_all(cls.evadb, "DROP TABLE IF EXISTS HomeRentals;") >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= + execute_query_fetch_all(cls.evadb, "DROP TABLE IF EXISTS HomeRentals;") +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) @forecast_skip_marker def test_forecast(self): create_predict_udf = """ <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) CREATE FUNCTION AirForecast FROM +======= + CREATE FUNCTION Forecast FROM +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) (SELECT unique_id, ds, y FROM AirData) TYPE Forecasting HORIZON 12 @@ -131,8 +162,12 @@ def test_forecast(self): predict_query = """ <<<<<<< HEAD +<<<<<<< HEAD <<<<<<< HEAD SELECT AirForecast() order by y; +======= + SELECT AirForecast(12) order by y; +>>>>>>> 53dafecf (feat: sync master staging (#1050)) """ result = execute_query_fetch_all(self.evadb, predict_query) self.assertEqual(len(result), 12) @@ -194,6 +229,7 @@ def test_forecast_with_column_rename(self): result = execute_query_fetch_all(self.evadb, predict_query) self.assertEqual(int(list(result.frames.iloc[:, -1])[-1]), 459) >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= SELECT AirForecast() order by y; """ @@ -252,6 +288,8 @@ def test_forecast_with_column_rename(self): ["homeforecast.type", "homeforecast.saledate", "homeforecast.ma"], ) >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) if __name__ == "__main__": diff --git a/test/integration_tests/long/test_model_train.py b/test/integration_tests/long/test_model_train.py index a14078cc57..4de1e21dde 100644 --- a/test/integration_tests/long/test_model_train.py +++ b/test/integration_tests/long/test_model_train.py @@ -69,13 +69,19 @@ def test_ludwig_automl(self): create_predict_function = """ <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) CREATE OR REPLACE FUNCTION PredictHouseRentLudwig FROM ======= CREATE FUNCTION IF NOT EXISTS PredictHouseRent FROM >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= CREATE OR REPLACE FUNCTION PredictHouseRentLudwig FROM >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ( SELECT * FROM HomeRentals ) TYPE Ludwig PREDICT 'rental_price' diff --git a/test/integration_tests/long/test_reuse.py b/test/integration_tests/long/test_reuse.py index ef412d756c..871050d729 100644 --- a/test/integration_tests/long/test_reuse.py +++ b/test/integration_tests/long/test_reuse.py @@ -57,8 +57,11 @@ def setUp(self): execute_query_fetch_all(self.evadb, f"LOAD VIDEO '{ua_detrac}' INTO DETRAC;") <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) execute_query_fetch_all(self.evadb, "CREATE TABLE fruitTable (data TEXT(100))") data_list = [ "The color of apple is red", @@ -69,10 +72,15 @@ def setUp(self): self.evadb, f"INSERT INTO fruitTable (data) VALUES ('{data}')" ) <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) load_functions_for_testing(self.evadb) self._load_hf_model() diff --git a/test/integration_tests/short/test_drop_executor.py b/test/integration_tests/short/test_drop_executor.py index ce6ad94ea2..4fc8e3e7de 100644 --- a/test/integration_tests/short/test_drop_executor.py +++ b/test/integration_tests/short/test_drop_executor.py @@ -112,20 +112,28 @@ def test_should_drop_table(self): self.evadb, drop_query, do_not_print_exceptions=True ) +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) # we should be able to re-create the table execute_query_fetch_all(self.evadb, query) # clean up execute_query_fetch_all(self.evadb, drop_query) +<<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def run_create_function_query(self): create_function_query = """CREATE FUNCTION DummyObjectDetector INPUT (Frame_Array NDARRAY UINT8(3, 256, 256)) @@ -150,10 +158,13 @@ def test_should_drop_function(self): ) self.assertTrue(function is None) +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) # We should be able to re-create the function self.run_create_function_query() # clean up diff --git a/test/integration_tests/short/test_select_executor.py b/test/integration_tests/short/test_select_executor.py index ef8f58bcdb..7d17cb2a5b 100644 --- a/test/integration_tests/short/test_select_executor.py +++ b/test/integration_tests/short/test_select_executor.py @@ -432,13 +432,19 @@ def test_expression_tree_signature(self): ) <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) signature = next(plan.find_all(LogicalFilter)).predicate.children[0].signature() ======= signature = plan.target_list[0].signature() >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= signature = next(plan.find_all(LogicalFilter)).predicate.children[0].signature() >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) function_id = ( self.evadb.catalog() .get_function_catalog_entry_by_name("DummyMultiObjectDetector") diff --git a/test/markers.py b/test/markers.py index 7f21635635..728a83a639 100644 --- a/test/markers.py +++ b/test/markers.py @@ -20,6 +20,7 @@ from evadb.utils.generic_utils import ( <<<<<<< HEAD +<<<<<<< HEAD <<<<<<< HEAD is_chromadb_available, ======= @@ -27,6 +28,11 @@ ======= is_chromadb_available, >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= + is_chromadb_available, +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) is_forecast_available, is_gpu_available, is_ludwig_available, diff --git a/test/unit_tests/binder/test_statement_binder.py b/test/unit_tests/binder/test_statement_binder.py index d73c6d5407..7452dcf9f0 100644 --- a/test/unit_tests/binder/test_statement_binder.py +++ b/test/unit_tests/binder/test_statement_binder.py @@ -407,6 +407,9 @@ def test_bind_create_index(self): <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def test_bind_create_function_should_raise_without_predict_for_ludwig(self): with patch.object(StatementBinder, "bind"): create_function_statement = MagicMock() @@ -416,12 +419,15 @@ def test_bind_create_function_should_raise(self): with patch.object(StatementBinder, "bind"): create_function_statement = MagicMock() >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= def test_bind_create_function_should_raise_without_predict_for_ludwig(self): with patch.object(StatementBinder, "bind"): create_function_statement = MagicMock() create_function_statement.function_type = "ludwig" >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) create_function_statement.query.target_list = [] create_function_statement.metadata = [] binder = StatementBinder(StatementBinderContext(MagicMock())) @@ -430,6 +436,9 @@ def test_bind_create_function_should_raise_without_predict_for_ludwig(self): <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def test_bind_create_function_should_drop_row_id_for_select_star(self): with patch.object(StatementBinder, "bind"): create_function_statement = MagicMock() @@ -439,12 +448,15 @@ def test_bind_create_function_should_drop_row_id(self): with patch.object(StatementBinder, "bind"): create_function_statement = MagicMock() >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= def test_bind_create_function_should_drop_row_id_for_select_star(self): with patch.object(StatementBinder, "bind"): create_function_statement = MagicMock() create_function_statement.function_type = "ludwig" >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) row_id_col_obj = ColumnCatalogEntry( name=IDENTIFIER_COLUMN, type=MagicMock(), @@ -512,8 +524,11 @@ def test_bind_create_function_should_drop_row_id_for_select_star(self): self.assertEqual(create_function_statement.outputs, expected_outputs) <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def test_bind_create_function_should_bind_forecast_with_default_columns(self): with patch.object(StatementBinder, "bind"): @@ -674,7 +689,12 @@ def test_bind_create_function_should_raise_forecast_missing_required_columns(sel err_msg = "Missing required {'ma'} columns for forecasting function." self.assertEqual(str(cm.exception), err_msg) <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) diff --git a/test/unit_tests/executor/test_create_udf_executor.py b/test/unit_tests/executor/test_create_udf_executor.py index d3c2a08702..9b4ce90765 100644 --- a/test/unit_tests/executor/test_create_udf_executor.py +++ b/test/unit_tests/executor/test_create_udf_executor.py @@ -57,10 +57,13 @@ def test_should_create_function(self, load_function_class_from_file_mock): {"key1": "value1", "key2": "value2"}, ) +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def test_should_raise_or_replace_if_not_exists(self): plan = type( "CreateFunctionPlan", @@ -85,6 +88,9 @@ def test_should_skip_if_not_exists(self, load_function_class_from_file_mock): catalog_instance = MagicMock() catalog_instance().get_function_catalog_entry_by_name.return_value = True <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= @patch("evadb.executor.create_function_executor.load_function_class_from_file") def test_should_raise_error_on_incorrect_io_definition( @@ -93,8 +99,11 @@ def test_should_raise_error_on_incorrect_io_definition( catalog_instance = MagicMock() catalog_instance().get_function_catalog_entry_by_name.return_value = None >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) catalog_instance().insert_function_catalog_entry.return_value = "function" impl_path = MagicMock() abs_path = impl_path.absolute.return_value = MagicMock() @@ -102,8 +111,11 @@ def test_should_raise_error_on_incorrect_io_definition( load_function_class_from_file_mock.return_value.return_value = "mock_class" <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) plan = type( "CreateFunctionPlan", (), @@ -256,13 +268,19 @@ def test_should_raise_error_on_incorrect_io_definition( self.assertIn( <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) "Error creating function, input/output definition incorrect:", ======= "Error creating Function, input/output definition incorrect:", >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= "Error creating function, input/output definition incorrect:", >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) str(exc.exception), ) diff --git a/test/unit_tests/executor/test_plan_executor.py b/test/unit_tests/executor/test_plan_executor.py index a99ec1289e..0261dade94 100644 --- a/test/unit_tests/executor/test_plan_executor.py +++ b/test/unit_tests/executor/test_plan_executor.py @@ -183,13 +183,19 @@ def test_execute_plan_for_create_insert_load_upload_plans(self, mock_build): <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) tree = MagicMock(node=CreateFunctionPlan(None, False, False, [], [], None)) ======= tree = MagicMock(node=CreateFunctionPlan(None, False, [], [], None)) >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= tree = MagicMock(node=CreateFunctionPlan(None, False, False, [], [], None)) >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) mock_build.return_value = tree actual = list(PlanExecutor(MagicMock(), None).execute_plan()) tree.exec.assert_called_once() diff --git a/test/unit_tests/optimizer/test_statement_to_opr_converter.py b/test/unit_tests/optimizer/test_statement_to_opr_converter.py index 796cce4f13..0935d60f12 100644 --- a/test/unit_tests/optimizer/test_statement_to_opr_converter.py +++ b/test/unit_tests/optimizer/test_statement_to_opr_converter.py @@ -137,10 +137,13 @@ def test_visit_select_should_not_call_visits_for_null_values(self): converter._visit_projection.assert_not_called() converter._visit_select_predicate.assert_not_called() +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def test_visit_select_without_table_ref(self): converter = StatementToPlanConverter() converter.visit_table_ref = MagicMock() @@ -162,11 +165,16 @@ def test_visit_select_without_table_ref(self): converter._visit_orderby.assert_not_called() converter._visit_limit.assert_not_called() +<<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) @patch("evadb.optimizer.statement_to_opr_converter.LogicalCreateFunction") @patch( "evadb.optimizer.\ @@ -339,12 +347,18 @@ def test_check_plan_equality(self): ) <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= + +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) create_plan.append_child(create_function_plan) plans.append(dummy_plan) diff --git a/test/unit_tests/parser/test_parser.py b/test/unit_tests/parser/test_parser.py index a0199b96aa..71d807de6b 100644 --- a/test/unit_tests/parser/test_parser.py +++ b/test/unit_tests/parser/test_parser.py @@ -126,6 +126,7 @@ def test_create_index_statement(self): <<<<<<< HEAD ======= +<<<<<<< HEAD # create if_not_exists expected_stmt = CreateIndexStatement( "testindex", @@ -147,6 +148,7 @@ def test_create_index_statement(self): self.assertEqual(actual_stmt.index_def, create_index_query) >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +<<<<<<< HEAD <<<<<<< HEAD # create if_not_exists expected_stmt = CreateIndexStatement( @@ -168,6 +170,8 @@ def test_create_index_statement(self): self.assertEqual(actual_stmt, expected_stmt) self.assertEqual(actual_stmt.index_def, create_index_query) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) # create index on Function expression @@ -674,13 +678,19 @@ def test_select_function_star(self): <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) query = "SELECT DemoFunc(*) FROM DemoDB.DemoTable;" ======= query = "SELECT DemoFunc(*) FROM DemoDB.DemoTable" >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= query = "SELECT DemoFunc(*) FROM DemoDB.DemoTable;" >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) evadb_stmt_list = parser.parse(query) # check stmt itself @@ -792,10 +802,13 @@ def test_delete_statement(self): self.assertEqual(delete_stmt, expected_stmt) +<<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def test_set_statement(self): parser = Parser() set_statement = """SET OPENAIKEY = 'ABCD'""" @@ -1028,11 +1041,16 @@ def test_should_return_false_for_unequal_expression(self): insert_stmt = InsertTableStatement(table) create_func = CreateFunctionStatement( "func", +<<<<<<< HEAD <<<<<<< HEAD False, ======= >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) False, +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) False, Path("data/fastrcnn.py"), [ @@ -1191,6 +1209,9 @@ def test_class_equality(self): self.assertNotEqual(table_ref, table_info) <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) ======= def test_lark(self): @@ -1204,5 +1225,8 @@ def test_lark(self): parser = Parser() parser.parse(query) >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) diff --git a/test/unit_tests/plan_nodes/test_plan.py b/test/unit_tests/plan_nodes/test_plan.py index 9ba6f15f90..102b9c92b2 100644 --- a/test/unit_tests/plan_nodes/test_plan.py +++ b/test/unit_tests/plan_nodes/test_plan.py @@ -73,6 +73,7 @@ def test_insert_plan(self): def test_create_function_plan(self): function_name = "function" <<<<<<< HEAD +<<<<<<< HEAD <<<<<<< HEAD or_replace = False ======= @@ -80,6 +81,11 @@ def test_create_function_plan(self): ======= or_replace = False >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= + or_replace = False +======= +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) if_not_exists = True functionIO = "functionIO" inputs = [functionIO, functionIO] @@ -89,6 +95,9 @@ def test_create_function_plan(self): node = CreateFunctionPlan( <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) function_name, or_replace, if_not_exists, inputs, outputs, impl_path, ty ) self.assertEqual(node.opr_type, PlanOprType.CREATE_FUNCTION) @@ -100,6 +109,7 @@ def test_create_function_plan(self): self.assertEqual(node.opr_type, PlanOprType.CREATE_FUNCTION) self.assertEqual(node.if_not_exists, True) >>>>>>> 2dacff69 (feat: sync master staging (#1050)) +<<<<<<< HEAD ======= function_name, or_replace, if_not_exists, inputs, outputs, impl_path, ty ) @@ -107,6 +117,8 @@ def test_create_function_plan(self): self.assertEqual(node.or_replace, or_replace) self.assertEqual(node.if_not_exists, if_not_exists) >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) self.assertEqual(node.inputs, [functionIO, functionIO]) self.assertEqual(node.outputs, [functionIO]) self.assertEqual(node.impl_path, impl_path) diff --git a/test/unit_tests/storage/test_sqlite_native_storage_engine.py b/test/unit_tests/storage/test_sqlite_native_storage_engine.py index 52157bfafa..4b52030332 100644 --- a/test/unit_tests/storage/test_sqlite_native_storage_engine.py +++ b/test/unit_tests/storage/test_sqlite_native_storage_engine.py @@ -33,18 +33,27 @@ def __init__(self): <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) class SQLiteNativeStorageEngineTest(unittest.TestCase): ======= class SQLiiteNativeStorageEngineTest(unittest.TestCase): >>>>>>> 8c5b63dc (release: merge staging into master (#1032)) <<<<<<< HEAD +<<<<<<< HEAD ======= class SQLiteNativeStorageEngineTest(unittest.TestCase): >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> c63abee7 (release: merge staging into master (#1032)) +======= +======= +class SQLiteNativeStorageEngineTest(unittest.TestCase): +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/test/util.py b/test/util.py index 64962dd6cb..004bfb3301 100644 --- a/test/util.py +++ b/test/util.py @@ -36,18 +36,27 @@ from evadb.expression.function_expression import FunctionExpression <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +>>>>>>> 53dafecf (feat: sync master staging (#1050)) from evadb.functions.abstract.abstract_function import ( AbstractClassifierFunction, AbstractFunction, ) <<<<<<< HEAD +<<<<<<< HEAD ======= from evadb.functions.abstract.abstract_function import AbstractClassifierFunction >>>>>>> 2dacff69 (feat: sync master staging (#1050)) ======= >>>>>>> 40a10ce1 (Bump v0.3.4+ dev) +======= +======= +from evadb.functions.abstract.abstract_function import AbstractClassifierFunction +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) +>>>>>>> 53dafecf (feat: sync master staging (#1050)) from evadb.functions.decorators import decorators from evadb.functions.decorators.io_descriptors.data_types import ( NumpyArray, diff --git a/tutorials/11-similarity-search-for-motif-mining.ipynb b/tutorials/11-similarity-search-for-motif-mining.ipynb index 39f7fd2123..8b176aa735 100644 --- a/tutorials/11-similarity-search-for-motif-mining.ipynb +++ b/tutorials/11-similarity-search-for-motif-mining.ipynb @@ -269,11 +269,17 @@ } ], "source": [ +<<<<<<< HEAD "cursor.query(\"DROP FUNCTION IF EXISTS SiftFeatureExtractor;\").df()\n", "cursor.query(\"\"\"\n", " CREATE FUNCTION SiftFeatureExtractor\n", " IMPL '../evadb/functions/sift_feature_extractor.py'\n", "\"\"\").df()" +======= + "cursor.query(\"DROP UDF IF EXISTS SiftFeatureExtractor;\").df()\n", + "cursor.query(\"\"\"CREATE UDF IF NOT EXISTS SiftFeatureExtractor\n", + " IMPL '../evadb/functions/sift_feature_extractor.py'\"\"\").df()" +>>>>>>> 2dacff69 (feat: sync master staging (#1050)) ] }, {