Tickets/dm 45960 (#14)

* WIP * WIP * WIP * WIP * WIP * WIP * WIPO * WIP * WIP * ignore as list * WIP * WIP * WIP * WIP * WIP * WIP * WIP * plot exposure gap
lsst-ts · Aug 30, 2024 · f397860 · f397860
1 parent 91df4e1
commit f397860
Show file tree

Hide file tree

Showing 2 changed files with 115 additions and 24 deletions.
diff --git a/notebooks_tsqr/exposurelog.ipynb b/notebooks_tsqr/exposurelog.ipynb
@@ -89,9 +89,100 @@
    ]
   },
   {
-   "cell_type": "markdown",
+   "cell_type": "code",
+   "execution_count": null,
    "id": "6",
    "metadata": {},
+   "outputs": [],
+   "source": [
+    "# logrep_utils.py\n",
+    "############################################\n",
+    "# Python Standard Library\n",
+    "from urllib.parse import urlencode\n",
+    "import itertools\n",
+    "from datetime import datetime\n",
+    "############################################\n",
+    "# External Packages\n",
+    "import requests\n",
+    "\n",
+    "\n",
+    "MAX_CONNECT_TIMEOUT = 3.1    # seconds\n",
+    "MAX_READ_TIMEOUT = 90 * 60   # seconds\n",
+    "\n",
+    "class ApiAdapter:\n",
+    "    def __init__(self, *,\n",
+    "                 server_url='https://tucson-teststand.lsst.codes',\n",
+    "                 connect_timeout=3.05,  # seconds\n",
+    "                 read_timeout=10 * 60,  # seconds\n",
+    "                 ):\n",
+    "        self.server = server_url\n",
+    "        self.c_timeout = min(MAX_CONNECT_TIMEOUT,\n",
+    "                             float(connect_timeout))  # seconds\n",
+    "        self.r_timeout = min(MAX_READ_TIMEOUT,  # seconds\n",
+    "                             float(read_timeout))\n",
+    "        self.timeout = (self.c_timeout, self.r_timeout)\n",
+    "\n",
+    "\n",
+    "class ExposurelogAdapter(ApiAdapter):\n",
+    "    service = 'exposurelog'\n",
+    "\n",
+    "    def get_exposures(self, instrument, registry=1):\n",
+    "        qparams = dict(instrument=instrument, registery=registry)\n",
+    "        url = f'{self.server}/{self.service}/exposures?{urlencode(qparams)}'\n",
+    "        recs = requests.get(url, timeout=self.timeout).json()\n",
+    "        return recs\n",
+    "\n",
+    "    def get_observation_gaps(self, instrument, rollup='day'):\n",
+    "        recs = self.get_exposures(instrument)\n",
+    "        instrument_gaps = dict()\n",
+    "        for day,dayrecs in itertools.groupby(recs, key=lambda r: r['day_obs']):\n",
+    "            gaps = list()\n",
+    "            begin = end = None\n",
+    "            for rec in dayrecs:\n",
+    "                begin = rec['timespan_begin']\n",
+    "                if end:\n",
+    "                    # span in minutes\n",
+    "                    diff = (datetime.fromisoformat(begin)\n",
+    "                            - datetime.fromisoformat(end)\n",
+    "                            ).total_seconds() / 60.0\n",
+    "\n",
+    "                    gaps.append((\n",
+    "                        datetime.fromisoformat(end).time().isoformat(),\n",
+    "                        datetime.fromisoformat(begin).time().isoformat(),\n",
+    "                        diff\n",
+    "                    ))\n",
+    "                end = rec['timespan_end']\n",
+    "            instrument_gaps[day] = gaps\n",
+    "\n",
+    "            roll = dict()\n",
+    "            if rollup == 'day':\n",
+    "                for day,tuples in instrument_gaps.items():\n",
+    "                    roll[day] = sum([t[2] for t in tuples])\n",
+    "            else:\n",
+    "                roll = instrument_gaps\n",
+    "        #!return instrument_gaps, recs\n",
+    "        return roll, recs\n",
+    "\n",
+    "\n",
+    "\n",
+    "# gaps,recs=logrep_utils.ExposurelogAdapter().get_observation_gaps('LSSTComCam')\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "gaps,recs = ExposurelogAdapter().get_observation_gaps('LSSTComCam')\n",
+    "gaps"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "8",
+   "metadata": {},
    "source": [
     "<a class=\"anchor\" id=\"get_records\"></a>\n",
     "## Get Records"
@@ -100,7 +191,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "7",
+   "id": "9",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -133,7 +224,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "8",
+   "id": "10",
    "metadata": {},
    "source": [
     "<a class=\"anchor\" id=\"table\"></a>\n",
@@ -142,7 +233,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "9",
+   "id": "11",
    "metadata": {},
    "source": [
     "### Fields names provided in records from log."
@@ -151,7 +242,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "10",
+   "id": "12",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -160,7 +251,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "11",
+   "id": "13",
    "metadata": {},
    "source": [
     "### Facets from log records.\n",
@@ -170,7 +261,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "12",
+   "id": "14",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -180,7 +271,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "13",
+   "id": "15",
    "metadata": {},
    "source": [
     "### Table of selected log record fields.\n",
@@ -190,7 +281,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "14",
+   "id": "16",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -209,7 +300,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "15",
+   "id": "17",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -220,7 +311,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "16",
+   "id": "18",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -231,7 +322,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "17",
+   "id": "19",
    "metadata": {},
    "source": [
     "<a class=\"anchor\" id=\"plot\"></a>\n",
@@ -241,20 +332,19 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "18",
+   "id": "20",
    "metadata": {},
    "outputs": [],
    "source": [
-    "x = [r['date_added'] for r in recs]\n",
-    "y = [int(r['is_valid']) for r in recs]\n",
-    "display_markdown('### IsValid vs DateAdded', raw=True)\n",
-    "plt.plot(x, y) \n",
-    "plt.show()"
+    "x,y = zip(*gaps.items())\n",
+    "display_markdown('### Date vs ExposureGap (minutes)', raw=True)\n",
+    "df = pd.DataFrame(dict(day=x,minutes=y))\n",
+    "df.plot.bar(x='day', y='minutes')"
    ]
   },
   {
    "cell_type": "markdown",
-   "id": "19",
+   "id": "21",
    "metadata": {},
    "source": [
     "<a class=\"anchor\" id=\"raw_analysis\"></a>\n",
@@ -263,7 +353,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "20",
+   "id": "22",
    "metadata": {},
    "source": [
     "### Example of one record"
@@ -272,7 +362,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "21",
+   "id": "23",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -290,7 +380,7 @@
   },
   {
    "cell_type": "markdown",
-   "id": "22",
+   "id": "24",
    "metadata": {},
    "source": [
     "<a class=\"anchor\" id=\"elicitation\"></a>\n",
@@ -300,7 +390,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "23",
+   "id": "25",
    "metadata": {},
    "outputs": [],
    "source": [
@@ -313,7 +403,7 @@
   {
    "cell_type": "code",
    "execution_count": null,
-   "id": "24",
+   "id": "26",
    "metadata": {},
    "outputs": [],
    "source": []

diff --git a/times-square.yaml b/times-square.yaml
@@ -2,5 +2,6 @@ enabled: true
 root: notebooks_tsqr
 ignore:
   - SAVE_*
+  - TEMPLATE_*
 description: >
   Times Square for project-wide Logging and Reporting.