Tickets/dm 45892 (#9)

* WIP * WIP * WIP * WIP * WIP * WIP * WIPO * WIP * WIP * ignore as list * WIP * WIP * WIP * WIP
lsst-ts · Aug 28, 2024 · c6c2a83 · c6c2a83
1 parent e2d30ef
commit c6c2a83
Show file tree

Hide file tree

Showing 4 changed files with 80 additions and 90 deletions.
diff --git a/notebooks_tsqr/TEMPLATE_logrep.ipynb b/notebooks_tsqr/TEMPLATE_logrep.ipynb
@@ -35,7 +35,8 @@
     "from pprint import pp, pformat\n",
     "from urllib.parse import urlencode\n",
     "from IPython.display import FileLink, display_markdown\n",
-    "from matplotlib import pyplot as plt"
+    "from matplotlib import pyplot as plt\n",
+    "import os"
    ]
   },
   {
@@ -54,19 +55,8 @@
     "\n",
     "timeout = (float(response_timeout), float(read_timeout))\n",
     "\n",
-    "# Env list comes from drop-down menu top of:\n",
-    "# https://rsp.lsst.io/v/usdfdev/guides/times-square/\n",
-    "envs = dict(\n",
-    "    #rubin_usdf_dev = '',\n",
-    "    #data_lsst_cloud = '',\n",
-    "    #usdf = '',\n",
-    "    #base_data_facility = '',\n",
-    "    summit = 'https://summit-lsp.lsst.codes',\n",
-    "    usdf_dev = 'https://usdf-rsp-dev.slac.stanford.edu',\n",
-    "    #rubin_idf_int = '',\n",
-    "    tucson = 'https://tucson-teststand.lsst.codes',\n",
-    ")\n",
-    "server = envs[env]\n",
+    "server = os.environ.get('EXTERNAL_INSTANCE_URL', \n",
+    "                         'https://tucson-teststand.lsst.codes')\n",
     "service = f'{server}/{log}'\n",
     "service"
    ]
@@ -301,28 +291,35 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "display(recs[-1])"
+    "rec = recs[-1]\n",
+    "\n",
+    "msg = rec[\"message_text\"]\n",
+    "md = f'Message text from log:\\n> {msg}'\n",
+    "display_markdown(md, raw=True)\n",
+    "\n",
+    "display(rec)"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
+   "cell_type": "markdown",
    "id": "22",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "msg = rec[\"message_text\"]\n",
-    "md = f'Message text from log:\\n> {msg}'\n",
-    "display_markdown(md, raw=True)"
+    "<a class=\"anchor\" id=\"elicitation\"></a>\n",
+    "## Stakeholder Elicitation"
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "id": "23",
    "metadata": {},
+   "outputs": [],
    "source": [
-    "<a class=\"anchor\" id=\"elicitation\"></a>\n",
-    "## Stakeholder Elicitation"
+    "#EXTERNAL_INSTANCE_URL\n",
+    "ed = dict(os.environ.items())\n",
+    "with pd.option_context('display.max_rows', None,):\n",
+    "    print(pd.DataFrame(ed.values(), index=ed.keys()))"
    ]
   },
   {

diff --git a/notebooks_tsqr/TEMPLATE_logrep.yaml b/notebooks_tsqr/TEMPLATE_logrep.yaml
@@ -1,5 +1,5 @@
 # For use with a Times Square notebook
-title: LR mix
+title: TEMPLATE for LR
 description: Prototype 1
 authors:
   - name: Steve Pothier

diff --git a/notebooks_tsqr/exposurelog.ipynb b/notebooks_tsqr/exposurelog.ipynb
@@ -34,7 +34,7 @@
     "import pandas as pd\n",
     "from pprint import pp\n",
     "from urllib.parse import urlencode\n",
-    "from IPython.display import FileLink\n",
+    "from IPython.display import FileLink, display_markdown\n",
     "from matplotlib import pyplot as plt\n",
     "import os"
    ]
@@ -46,7 +46,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "\n",
     "limit = int(record_limit)\n",
     "\n",
     "response_timeout = 3.05  # seconds, how long to wait for connection\n",
@@ -194,7 +193,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "cols = ['date_added', 'time_lost']\n",
+    "cols = ['obs_id', 'user_id', 'user_agent','is_human','is_valid','exposure_flag']\n",
     "df = pd.DataFrame(recs)[cols]\n",
     "\n",
     "# Allow download of CSV version of DataFrame\n",
@@ -218,9 +217,21 @@
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "id": "16",
    "metadata": {},
+   "outputs": [],
+   "source": [
+    "cols = ['obs_id', 'site_id', 'instrument', 'message_text', 'tags','user_id', 'user_agent','is_human','is_valid','exposure_flag']\n",
+    "df = pd.DataFrame(recs, columns=None)\n",
+    "df"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "17",
+   "metadata": {},
    "source": [
     "<a class=\"anchor\" id=\"plot\"></a>\n",
     "## Plots from log"
@@ -229,19 +240,20 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "17",
+   "id": "18",
    "metadata": {},
    "outputs": [],
    "source": [
     "x = [r['date_added'] for r in recs]\n",
-    "y = [r['time_lost'] for r in recs]\n",
+    "y = [int(r['is_valid']) for r in recs]\n",
+    "display_markdown('### IsValid vs DateAdded', raw=True)\n",
     "plt.plot(x, y) \n",
     "plt.show()"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "18",
+   "id": "19",
    "metadata": {},
    "source": [
     "<a class=\"anchor\" id=\"raw_analysis\"></a>\n",
@@ -250,7 +262,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "19",
+   "id": "20",
    "metadata": {},
    "source": [
     "### Example of one record"
@@ -259,45 +271,42 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "20",
+   "id": "21",
    "metadata": {},
    "outputs": [],
    "source": [
-    "rec = recs[0]\n",
-    "rec"
+    "rec = recs[-1]\n",
+    "\n",
+    "msg = rec[\"message_text\"]\n",
+    "md = f'Message text from log:\\n> {msg}'\n",
+    "display_markdown(md, raw=True)\n",
+    "\n",
+    "md = f'One full record (the last one retrieved):\\n> {rec}'\n",
+    "display_markdown(md, raw=True)\n",
+    "\n",
+    "display(rec)"
    ]
   },
   {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "21",
+   "cell_type": "markdown",
+   "id": "22",
    "metadata": {},
-   "outputs": [],
    "source": [
-    "msg = rec[\"message_text\"]\n",
-    "print(msg)"
+    "<a class=\"anchor\" id=\"elicitation\"></a>\n",
+    "## Stakeholder Elicitation"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "22",
+   "id": "23",
    "metadata": {},
    "outputs": [],
    "source": [
-    "import os\n",
     "#EXTERNAL_INSTANCE_URL\n",
+    "ed = dict(os.environ.items())\n",
     "with pd.option_context('display.max_rows', None,):\n",
-    "    print(pd.DataFrame(os.environ))"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "23",
-   "metadata": {},
-   "source": [
-    "<a class=\"anchor\" id=\"elicitation\"></a>\n",
-    "## Stakeholder Elicitation"
+    "    print(pd.DataFrame(ed.values(), index=ed.keys()))"
    ]
   },
   {

diff --git a/notebooks_tsqr/narrativelog.ipynb b/notebooks_tsqr/narrativelog.ipynb
@@ -25,11 +25,7 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "2",
-   "metadata": {
-    "jupyter": {
-     "source_hidden": true
-    }
-   },
+   "metadata": {},
    "outputs": [],
    "source": [
     "# Only use packages available in the Rubin Science Platform\n",
@@ -38,8 +34,9 @@
     "import pandas as pd\n",
     "from pprint import pp\n",
     "from urllib.parse import urlencode\n",
-    "from IPython.display import FileLink\n",
-    "from matplotlib import pyplot as plt"
+    "from IPython.display import FileLink, display_markdown\n",
+    "from matplotlib import pyplot as plt\n",
+    "import os"
    ]
   },
   {
@@ -58,19 +55,8 @@
     "\n",
     "timeout = (float(response_timeout), float(read_timeout))\n",
     "\n",
-    "# Env list comes from drop-down menu top of:\n",
-    "# https://rsp.lsst.io/v/usdfdev/guides/times-square/\n",
-    "envs = dict(\n",
-    "    #rubin_usdf_dev = '',\n",
-    "    #data_lsst_cloud = '',\n",
-    "    #usdf = '',\n",
-    "    #base_data_facility = '',\n",
-    "    summit = 'https://summit-lsp.lsst.codes',\n",
-    "    usdf_dev = 'https://usdf-rsp-dev.slac.stanford.edu',\n",
-    "    #rubin_idf_int = '',\n",
-    "    tucson = 'https://tucson-teststand.lsst.codes',\n",
-    ")\n",
-    "server = envs[env]\n",
+    "server = os.environ.get('EXTERNAL_INSTANCE_URL', \n",
+    "                         'https://tucson-teststand.lsst.codes')\n",
     "service = f'{server}/{log}'\n",
     "service"
    ]
@@ -219,7 +205,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "cols = ['date_added', 'time_lost']\n",
+    "cols = ['date_added', 'time_lost', 'time_lost_type']\n",
     "df = pd.DataFrame(recs)[cols]\n",
     "\n",
     "# Allow download of CSV version of DataFrame\n",
@@ -238,7 +224,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "df = pd.DataFrame(recs)\n",
+    "cols = ['message_text','tags','user_id', 'components','date_end']\n",
+    "df = pd.DataFrame(recs, columns=cols)\n",
     "df"
    ]
   },
@@ -288,24 +275,21 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "rec = recs[0]\n",
-    "rec"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "21",
-   "metadata": {},
-   "outputs": [],
-   "source": [
+    "rec = recs[-1]\n",
+    "\n",
     "msg = rec[\"message_text\"]\n",
-    "print(msg)"
+    "md = f'Message text from log:\\n> {msg}'\n",
+    "display_markdown(md, raw=True)\n",
+    "\n",
+    "md = f'One full record (the last one retrieved):\\n> {rec}'\n",
+    "display_markdown(md, raw=True)\n",
+    "\n",
+    "display(rec)"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "22",
+   "id": "21",
    "metadata": {},
    "source": [
     "<a class=\"anchor\" id=\"elicitation\"></a>\n",
@@ -315,7 +299,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "23",
+   "id": "22",
    "metadata": {},
    "outputs": [],
    "source": []