usage/get_pipeline.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "0684e7d6-02a9-4a50-8a4b-c0c6e6b7795e",
   "metadata": {},
   "source": [
    "# Get Pipeline Information\n",
    "\n",
    "This example shows how to get a list of pipelines and interact with one of them.\n",
    "If you do not have any pipelines created, follow one of our other tutorials or examples to create a pipeline, or create one in the [webapp](https://app.glassflow.dev)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c07f1c8c-a7e9-4c02-bc09-0738ca762781",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-02T02:13:50.005236Z",
     "start_time": "2024-10-02T02:13:49.956734Z"
    }
   },
   "outputs": [],
   "source": [
    "from glassflow import GlassFlowClient\n",
    "import pandas as pd\n",
    "\n",
    "# Please edit this variable with your own personal access token from https://app.glassflow.dev/profile\n",
    "personal_access_token = \"< your-personal-access-token >\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a1f8ce60-4e2e-4996-a603-1ac7669f8200",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-02T02:15:12.578831Z",
     "start_time": "2024-10-02T02:15:12.576414Z"
    }
   },
   "outputs": [],
   "source": [
    "## Setup glassflow client\n",
    "client = GlassFlowClient(\n",
    "    personal_access_token=personal_access_token\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2603886-4cbf-4722-be60-6b87e46d9610",
   "metadata": {},
   "source": [
    "## List all the pipelines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2f4b84c49f85ed1b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-02T02:15:13.790956Z",
     "start_time": "2024-10-02T02:15:13.788488Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "glassflow.models.operations.pipeline_crud.ListPipelinesResponse"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline_resp = client.list_pipelines()\n",
    "type(pipeline_resp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0f5203ed-010f-4fae-89e0-f00e1c837bfb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-02T02:15:20.010055Z",
     "start_time": "2024-10-02T02:15:20.007331Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline_list = pipeline_resp.pipelines\n",
    "type(pipeline_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b694e682-9aa9-41c5-b145-03fbbf1bea1f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-02T02:15:23.719096Z",
     "start_time": "2024-10-02T02:15:23.716208Z"
    }
   },
   "outputs": [],
   "source": [
    "if len(pipeline_list) == 0:\n",
    "    print(\"You currently do not have any pipelines. Please create a pipeline first before continuing\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4f7c6e66-a212-4841-8a1d-e6d32e0ba9f6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-02T02:15:27.080341Z",
     "start_time": "2024-10-02T02:15:27.077792Z"
    }
   },
   "source": [
    "Show the list of pipelines as a table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3f9c60d3-e295-4f76-9289-cb2403af8f65",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-02T02:15:30.295826Z",
     "start_time": "2024-10-02T02:15:30.290419Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>metadata</th>\n",
       "      <th>name</th>\n",
       "      <th>space_id</th>\n",
       "      <th>created_at</th>\n",
       "      <th>id</th>\n",
       "      <th>state</th>\n",
       "      <th>space_name</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>None</td>\n",
       "      <td>classified-ads-enrichment</td>\n",
       "      <td>4119e23c-c09f-4153-810c-6160ac8581eb</td>\n",
       "      <td>2024-07-23T09:59:19.425754Z</td>\n",
       "      <td>2b276e80-f1e8-4551-b652-b4a9a74edc87</td>\n",
       "      <td>running</td>\n",
       "      <td>examples</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'sourceConnector': {'type': 'webhook'}}</td>\n",
       "      <td>gfl</td>\n",
       "      <td>41f3ef24-01d8-4d76-bf84-b11c4abbc6a3</td>\n",
       "      <td>2024-08-20T08:22:34.807997Z</td>\n",
       "      <td>701d6cc6-0b42-42be-96e0-c8c0a748ef89</td>\n",
       "      <td>running</td>\n",
       "      <td>use-cases</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'sourceConnector': {'type': 'webhook'}}</td>\n",
       "      <td>airbyte-destination-integration-test</td>\n",
       "      <td>4119e23c-c09f-4153-810c-6160ac8581eb</td>\n",
       "      <td>2024-08-28T11:38:14.32958Z</td>\n",
       "      <td>f796ecc5-7fd2-4dcf-86eb-88914733469b</td>\n",
       "      <td>running</td>\n",
       "      <td>examples</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{}</td>\n",
       "      <td>example pipeline</td>\n",
       "      <td>7a093f5d-d7f5-460b-89cf-f6472cb6418a</td>\n",
       "      <td>2024-10-04T09:00:14.480015Z</td>\n",
       "      <td>79d420c8-b8b8-4fe9-aed2-0763a0d067e2</td>\n",
       "      <td>paused</td>\n",
       "      <td>tutorials</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   metadata  \\\n",
       "0                                      None   \n",
       "1  {'sourceConnector': {'type': 'webhook'}}   \n",
       "2  {'sourceConnector': {'type': 'webhook'}}   \n",
       "3                                        {}   \n",
       "\n",
       "                                   name                              space_id  \\\n",
       "0             classified-ads-enrichment  4119e23c-c09f-4153-810c-6160ac8581eb   \n",
       "1                                   gfl  41f3ef24-01d8-4d76-bf84-b11c4abbc6a3   \n",
       "2  airbyte-destination-integration-test  4119e23c-c09f-4153-810c-6160ac8581eb   \n",
       "3                      example pipeline  7a093f5d-d7f5-460b-89cf-f6472cb6418a   \n",
       "\n",
       "                    created_at                                    id    state  \\\n",
       "0  2024-07-23T09:59:19.425754Z  2b276e80-f1e8-4551-b652-b4a9a74edc87  running   \n",
       "1  2024-08-20T08:22:34.807997Z  701d6cc6-0b42-42be-96e0-c8c0a748ef89  running   \n",
       "2   2024-08-28T11:38:14.32958Z  f796ecc5-7fd2-4dcf-86eb-88914733469b  running   \n",
       "3  2024-10-04T09:00:14.480015Z  79d420c8-b8b8-4fe9-aed2-0763a0d067e2   paused   \n",
       "\n",
       "  space_name  \n",
       "0   examples  \n",
       "1  use-cases  \n",
       "2   examples  \n",
       "3  tutorials  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pipelines_df = pd.DataFrame(pipeline_list)\n",
    "display(pipelines_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d8696942-c28d-4272-accf-b43652aba0a2",
   "metadata": {},
   "source": [
    "## Get pipeline\n",
    "\n",
    "Take the latest pipeline from the list and get additional details of the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "bc407981-280e-45eb-bea4-9d924e3b1b5d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-02T02:16:10.021367Z",
     "start_time": "2024-10-02T02:16:10.018492Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'metadata': {},\n",
       " 'name': 'example pipeline',\n",
       " 'space_id': '7a093f5d-d7f5-460b-89cf-f6472cb6418a',\n",
       " 'created_at': '2024-10-04T09:00:14.480015Z',\n",
       " 'id': '79d420c8-b8b8-4fe9-aed2-0763a0d067e2',\n",
       " 'state': 'paused',\n",
       " 'space_name': 'tutorials'}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pipeline_selected = pipeline_list[-1] \n",
    "display(pipeline_selected)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d07fedb6-4f0e-46b5-af02-bd62eb38cafa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-02T02:16:18.309818Z",
     "start_time": "2024-10-02T02:16:15.648828Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "glassflow.pipeline.Pipeline"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipeline = client.get_pipeline(pipeline_id=pipeline_selected['id'])\n",
    "type(pipeline)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1567553b-df20-48ff-bf05-975de8f2f094",
   "metadata": {},
   "source": [
    "Display any environment variables set at the pipeline "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0b163d70-0fc3-4ef5-a8fe-b4204474f6f5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-02T02:20:28.194113Z",
     "start_time": "2024-10-02T02:20:28.191160Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'name': 'MY_VAR', 'value': 'my env var value'}]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "display(pipeline.env_vars)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "174e18b8-637c-461e-bbd2-2293009e086f",
   "metadata": {},
   "source": [
    "Display the code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a73750b9-3ba3-406e-882a-580c24596445",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-10-02T02:29:13.365671Z",
     "start_time": "2024-10-02T02:29:13.360952Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "```python\n",
       "import os\n",
       "\n",
       "\n",
       "def handler(data, log):\n",
       "    env = os.getenv(\"MY_VAR\")\n",
       "    log.info(\"My env variable: %s\", env)\n",
       "    return data\n",
       "\n",
       "```"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython.display import Markdown as md\n",
    "\n",
    "md(f\"```python\\n{pipeline.transformation_code}\\n```\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}