From f397860edac585c78a7d9ebcd998b93c3a60bb99 Mon Sep 17 00:00:00 2001 From: "S. Pothier" Date: Thu, 29 Aug 2024 18:38:56 -0700 Subject: [PATCH] Tickets/dm 45960 (#14) * WIP * WIP * WIP * WIP * WIP * WIP * WIPO * WIP * WIP * ignore as list * WIP * WIP * WIP * WIP * WIP * WIP * WIP * plot exposure gap --- notebooks_tsqr/exposurelog.ipynb | 138 +++++++++++++++++++++++++------ times-square.yaml | 1 + 2 files changed, 115 insertions(+), 24 deletions(-) diff --git a/notebooks_tsqr/exposurelog.ipynb b/notebooks_tsqr/exposurelog.ipynb index 20b68b7..7c1fe42 100644 --- a/notebooks_tsqr/exposurelog.ipynb +++ b/notebooks_tsqr/exposurelog.ipynb @@ -89,9 +89,100 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": null, "id": "6", "metadata": {}, + "outputs": [], + "source": [ + "# logrep_utils.py\n", + "############################################\n", + "# Python Standard Library\n", + "from urllib.parse import urlencode\n", + "import itertools\n", + "from datetime import datetime\n", + "############################################\n", + "# External Packages\n", + "import requests\n", + "\n", + "\n", + "MAX_CONNECT_TIMEOUT = 3.1 # seconds\n", + "MAX_READ_TIMEOUT = 90 * 60 # seconds\n", + "\n", + "class ApiAdapter:\n", + " def __init__(self, *,\n", + " server_url='https://tucson-teststand.lsst.codes',\n", + " connect_timeout=3.05, # seconds\n", + " read_timeout=10 * 60, # seconds\n", + " ):\n", + " self.server = server_url\n", + " self.c_timeout = min(MAX_CONNECT_TIMEOUT,\n", + " float(connect_timeout)) # seconds\n", + " self.r_timeout = min(MAX_READ_TIMEOUT, # seconds\n", + " float(read_timeout))\n", + " self.timeout = (self.c_timeout, self.r_timeout)\n", + "\n", + "\n", + "class ExposurelogAdapter(ApiAdapter):\n", + " service = 'exposurelog'\n", + "\n", + " def get_exposures(self, instrument, registry=1):\n", + " qparams = dict(instrument=instrument, registery=registry)\n", + " url = f'{self.server}/{self.service}/exposures?{urlencode(qparams)}'\n", + " recs = requests.get(url, timeout=self.timeout).json()\n", + " return recs\n", + "\n", + " def get_observation_gaps(self, instrument, rollup='day'):\n", + " recs = self.get_exposures(instrument)\n", + " instrument_gaps = dict()\n", + " for day,dayrecs in itertools.groupby(recs, key=lambda r: r['day_obs']):\n", + " gaps = list()\n", + " begin = end = None\n", + " for rec in dayrecs:\n", + " begin = rec['timespan_begin']\n", + " if end:\n", + " # span in minutes\n", + " diff = (datetime.fromisoformat(begin)\n", + " - datetime.fromisoformat(end)\n", + " ).total_seconds() / 60.0\n", + "\n", + " gaps.append((\n", + " datetime.fromisoformat(end).time().isoformat(),\n", + " datetime.fromisoformat(begin).time().isoformat(),\n", + " diff\n", + " ))\n", + " end = rec['timespan_end']\n", + " instrument_gaps[day] = gaps\n", + "\n", + " roll = dict()\n", + " if rollup == 'day':\n", + " for day,tuples in instrument_gaps.items():\n", + " roll[day] = sum([t[2] for t in tuples])\n", + " else:\n", + " roll = instrument_gaps\n", + " #!return instrument_gaps, recs\n", + " return roll, recs\n", + "\n", + "\n", + "\n", + "# gaps,recs=logrep_utils.ExposurelogAdapter().get_observation_gaps('LSSTComCam')\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "gaps,recs = ExposurelogAdapter().get_observation_gaps('LSSTComCam')\n", + "gaps" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, "source": [ "\n", "## Get Records" @@ -100,7 +191,7 @@ { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -133,7 +224,7 @@ }, { "cell_type": "markdown", - "id": "8", + "id": "10", "metadata": {}, "source": [ "\n", @@ -142,7 +233,7 @@ }, { "cell_type": "markdown", - "id": "9", + "id": "11", "metadata": {}, "source": [ "### Fields names provided in records from log." @@ -151,7 +242,7 @@ { "cell_type": "code", "execution_count": null, - "id": "10", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -160,7 +251,7 @@ }, { "cell_type": "markdown", - "id": "11", + "id": "13", "metadata": {}, "source": [ "### Facets from log records.\n", @@ -170,7 +261,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -180,7 +271,7 @@ }, { "cell_type": "markdown", - "id": "13", + "id": "15", "metadata": {}, "source": [ "### Table of selected log record fields.\n", @@ -190,7 +281,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -209,7 +300,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -220,7 +311,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -231,7 +322,7 @@ }, { "cell_type": "markdown", - "id": "17", + "id": "19", "metadata": {}, "source": [ "\n", @@ -241,20 +332,19 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "20", "metadata": {}, "outputs": [], "source": [ - "x = [r['date_added'] for r in recs]\n", - "y = [int(r['is_valid']) for r in recs]\n", - "display_markdown('### IsValid vs DateAdded', raw=True)\n", - "plt.plot(x, y) \n", - "plt.show()" + "x,y = zip(*gaps.items())\n", + "display_markdown('### Date vs ExposureGap (minutes)', raw=True)\n", + "df = pd.DataFrame(dict(day=x,minutes=y))\n", + "df.plot.bar(x='day', y='minutes')" ] }, { "cell_type": "markdown", - "id": "19", + "id": "21", "metadata": {}, "source": [ "\n", @@ -263,7 +353,7 @@ }, { "cell_type": "markdown", - "id": "20", + "id": "22", "metadata": {}, "source": [ "### Example of one record" @@ -272,7 +362,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -290,7 +380,7 @@ }, { "cell_type": "markdown", - "id": "22", + "id": "24", "metadata": {}, "source": [ "\n", @@ -300,7 +390,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -313,7 +403,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "26", "metadata": {}, "outputs": [], "source": [] diff --git a/times-square.yaml b/times-square.yaml index ed3fa57..a8877e0 100644 --- a/times-square.yaml +++ b/times-square.yaml @@ -2,5 +2,6 @@ enabled: true root: notebooks_tsqr ignore: - SAVE_* + - TEMPLATE_* description: > Times Square for project-wide Logging and Reporting.