Merge pull request #94 from ploomber/pipeline

Setting up Ploomber pipeline components

.gitignore

@@ -201,3 +201,7 @@ pipeline/products/eda-pipeline.ipynb
 
 !docs/intro-to-dashboards-and-apps/data/database/car_data.*
 !pipeline/data/database/car_data.*
+
+# Mini project ploomber output
+mini-projects/movie_rec_system/movie_rec_system/products/etl/eda-pipeline.ipynb
+mini-projects/movie_rec_system/movie_rec_system/products/etl/extract-pipeline.ipynb

mini-projects/movie-rec-system/.dockerignore

@@ -0,0 +1,4 @@
+.movies_data.duckdb.metadata
+movies_data.duckdb
+movie_rec_system/products/eda-pipeline.ipynb
+movie_rec_system/products/extract-pipeline.ipynb

mini-projects/movie-rec-system/Dockerfile

@@ -0,0 +1,22 @@
+# Use the official Python image as the base image
+FROM python:3.10
+
+# Set the working directory
+WORKDIR /app
+
+# Copy the poetry files
+COPY pyproject.toml poetry.lock /app/
+
+# Install poetry
+RUN pip install poetry
+
+# Install project dependencies
+RUN poetry install
+
+# Copy the rest of the application code
+COPY . .
+
+# Expose the port that the app runs on
+EXPOSE 8000
+
+RUN poetry run ploomber build

mini-projects/movie-rec-system/README.md

@@ -25,9 +25,9 @@ poetry lock
 poetry install
 ```
 
-5. Run the app
+5. Run the as a Ploomber pipeline
 
 ```
 cd mini-projects/
-poetry run python movie_rec_system/src/extract.py
+poetry run ploomber build
 ```

mini-projects/movie-rec-system/movie_rec_system/eda/eda.ipynb → mini-projects/movie-rec-system/movie_rec_system/etl/eda.ipynb

@@ -9,7 +9,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -19,28 +19,59 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 9,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<span style=\"None\">Found pyproject.toml from &#x27;C:\\Users\\jpjon\\Documents\\Ploomber\\sql\\mini-projects\\movie-rec-system&#x27;</span>"
+      ],
+      "text/plain": [
+       "Found pyproject.toml from 'C:\\Users\\jpjon\\Documents\\Ploomber\\sql\\mini-projects\\movie-rec-system'"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "%reload_ext sql\n",
-    "%sql duckdb:///../../movies_data.duckdb"
+    "%sql duckdb:///movies_data.duckdb"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 10,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "TableNotFoundError: There is no table with name 'movies' in the default schema\n",
+      "If you need help solving this issue, send us a message: https://ploomber.io/community\n"
+     ]
+    }
+   ],
    "source": [
     "%sqlcmd columns -t movies"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 11,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "TableNotFoundError: There is no table with name 'genres' in the default schema\n",
+      "If you need help solving this issue, send us a message: https://ploomber.io/community\n"
+     ]
+    }
+   ],
    "source": [
     "%sqlcmd columns -t genres"
    ]
@@ -278,9 +309,8 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.0"
-  },
-  "orig_nbformat": 4
+   "version": "3.10.8"
+  }
  },
  "nbformat": 4,
  "nbformat_minor": 2

mini-projects/movie-rec-system/pipeline.yaml

@@ -0,0 +1,9 @@
+tasks:
+  - source: movie_rec_system/etl/extract.py
+    product:
+      nb: movie_rec_system/products/extract-pipeline.ipynb
+      data: movies_data.duckdb
+  - source: movie_rec_system/etl/eda.ipynb
+    static_analysis: disable
+    product: 
+      nb: movie_rec_system/products/eda-pipeline.ipynb