From f860de610fec865572bfc5e01a22dbd9fa29cb21 Mon Sep 17 00:00:00 2001 From: Steve Pothier Date: Mon, 26 Aug 2024 15:24:57 -0700 Subject: [PATCH 01/10] WIP --- README.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.md b/README.md index 231b75b..cc53e85 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,11 @@ # ts_logging_and_reporting Project-wide Logging and Reporting + +## Development + +This repository uses Pre-commit to keep notebooks formatted and clean. Install Pre-commit by running: + +```bash +pip install pre-commit +pre-commit install +``` From f3e1d454e35ec57f418d81868c71fed098e451f9 Mon Sep 17 00:00:00 2001 From: Steve Pothier Date: Mon, 26 Aug 2024 15:33:52 -0700 Subject: [PATCH 02/10] WIP --- .pre-commit-config.yaml | 14 + notebooks_tsqr/README.md | 5 + notebooks_tsqr/logrep_proto_1.ipynb | 987 ++++++++++++++++++++++++++++ notebooks_tsqr/logrep_proto_1.yaml | 16 + notebooks_tsqr/requirements.txt | 6 + notebooks_tsqr/scaffolding.org | 71 ++ requirements.txt | 7 + times-square.yaml | 4 + 8 files changed, 1110 insertions(+) create mode 100644 .pre-commit-config.yaml create mode 100644 notebooks_tsqr/README.md create mode 100644 notebooks_tsqr/logrep_proto_1.ipynb create mode 100644 notebooks_tsqr/logrep_proto_1.yaml create mode 100644 notebooks_tsqr/requirements.txt create mode 100644 notebooks_tsqr/scaffolding.org create mode 100644 requirements.txt create mode 100644 times-square.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d302149 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,14 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-toml + - id: check-yaml + - id: trailing-whitespace + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.5.5 + hooks: + - id: ruff + args: [--fix, --exit-non-zero-on-fix] + - id: ruff-format diff --git a/notebooks_tsqr/README.md b/notebooks_tsqr/README.md new file mode 100644 index 0000000..3efda5a --- /dev/null +++ b/notebooks_tsqr/README.md @@ -0,0 +1,5 @@ +Times Square notebooks for (potential) use in project-wide Logging & Reporting + +See [offical +documentation](https://rsp.lsst.io/v/usdfdev/guides/times-square/index.html) +on creating notebooks for use by Times Square. diff --git a/notebooks_tsqr/logrep_proto_1.ipynb b/notebooks_tsqr/logrep_proto_1.ipynb new file mode 100644 index 0000000..563c013 --- /dev/null +++ b/notebooks_tsqr/logrep_proto_1.ipynb @@ -0,0 +1,987 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "e5b65509-9611-4493-8916-77a4c37489f0", + "metadata": {}, + "outputs": [], + "source": [ + "# Parameters. Set defaults here.\n", + "# Times Square replaces this cell with the user's parameters.\n", + "env = 'usdf_dev' # usdf-dev, tucson, slac, summit\n", + "log_name = 'narrativelog'\n", + "record_limit = '999'" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "c2017a15-e054-4a4e-a522-28deff6d2a7a", + "metadata": {}, + "outputs": [], + "source": [ + "# Only use packages available in the Rubin Science Platform\n", + "import requests\n", + "from collections import defaultdict\n", + "import pandas as pd\n", + "from pprint import pp\n", + "from urllib.parse import urlencode\n", + "from IPython.display import FileLink\n", + "from matplotlib import pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "2a310c90-7c5a-4cc2-a3b2-cc9d8291d245", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'https://usdf-rsp-dev.slac.stanford.edu'" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "log = log_name\n", + "limit = int(record_limit)\n", + "response_timeout = 3.05 # seconds, how long to wait for connection\n", + "read_timeout = 20 # seconds\n", + "\n", + "timeout = (float(response_timeout), float(read_timeout))\n", + "\n", + "# Env list comes from drop-down menu top of:\n", + "# https://rsp.lsst.io/v/usdfdev/guides/times-square/\n", + "envs = dict(\n", + " #rubin_usdf_dev = '',\n", + " #data_lsst_cloud = '',\n", + " #usdf = '',\n", + " #base_data_facility = '',\n", + " summit = 'https://summit-lsp.lsst.codes',\n", + " usdf_dev = 'https://usdf-rsp-dev.slac.stanford.edu',\n", + " #rubin_idf_int = '',\n", + " tucson = 'https://tucson-teststand.lsst.codes',\n", + ")\n", + "server = envs[env]\n", + "server" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "59b7f58f-7a39-42dc-a57d-cc890f5d3375", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Attempt to get logs from url='https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=either&limit=999'\n", + "numr=999 numf=24\n" + ] + } + ], + "source": [ + "recs = None\n", + "ok = True\n", + "\n", + "# is_human=either&is_valid=either&offset=0&limit=50' \n", + "# site_ids=tucson&message_text=wubba&min_level=0&max_level=999&user_ids=spothier&user_agents=LOVE\n", + "# tags=love&exclude_tags=ignore_message\n", + "qparams = dict(is_human='either',\n", + " is_valid='either',\n", + " limit=limit,\n", + " )\n", + "qstr = urlencode(qparams)\n", + "url = f'{server}/{log}/messages?{qstr}'\n", + "print(f'\\nAttempt to get logs from {url=}')\n", + "ignore_fields = set(['tags', 'urls', 'message_text', 'id', 'date_added', \n", + " 'obs_id', 'day_obs', 'seq_num', 'parent_id', 'user_id',\n", + " 'date_invalidated', 'date_begin', 'date_end',\n", + " 'time_lost', # float\n", + " #'systems','subsystems','cscs', # values are lists, special handling\n", + " ])\n", + "try:\n", + " response = requests.get(url, timeout=timeout)\n", + " response.raise_for_status()\n", + " recs = response.json()\n", + " flds = set(recs[0].keys())\n", + " facflds = flds - ignore_fields\n", + " # facets(field) = set(value-1, value-2, ...)\n", + " facets = {fld: set([str(r[fld])\n", + " for r in recs if not isinstance(r[fld], list)]) \n", + " for fld in facflds}\n", + "except Exception as err:\n", + " ok = False\n", + " print(f'ERROR getting {log} from {env=} using {url=}: {err=}')\n", + "numf = len(flds) if ok else 0\n", + "numr = len(recs) if ok else 0\n", + "print(f'{numr=} {numf=}')" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "d7fedb0b-02fc-4843-8607-cf593ea3fb91", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'category',\n", + " 'components',\n", + " 'cscs',\n", + " 'date_added',\n", + " 'date_begin',\n", + " 'date_end',\n", + " 'date_invalidated',\n", + " 'id',\n", + " 'is_human',\n", + " 'is_valid',\n", + " 'level',\n", + " 'message_text',\n", + " 'parent_id',\n", + " 'primary_hardware_components',\n", + " 'primary_software_components',\n", + " 'site_id',\n", + " 'subsystems',\n", + " 'systems',\n", + " 'tags',\n", + " 'time_lost',\n", + " 'time_lost_type',\n", + " 'urls',\n", + " 'user_agent',\n", + " 'user_id'}" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "flds" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "d0432b3b-4270-4702-b79d-ed7a4a2800e3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'level': {'0', '100'},\n", + " 'category': {'None', 'SCIENCE'},\n", + " 'subsystems': {'None'},\n", + " 'user_agent': {'LOVE'},\n", + " 'primary_software_components': {'None'},\n", + " 'is_valid': {'False', 'True'},\n", + " 'is_human': {'True'},\n", + " 'systems': {'None'},\n", + " 'primary_hardware_components': {'None'},\n", + " 'cscs': {'None'},\n", + " 'components': {'None'},\n", + " 'site_id': {'summit'},\n", + " 'time_lost_type': {'None', 'fault'}}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "facets" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "0720e2cc-d68c-4830-92c1-74ae5180f057", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': '0005d0e0-6de7-4857-b658-cfca7147374a',\n", + " 'site_id': 'summit',\n", + " 'message_text': 'The moveShutterMainDoor command Fail\\r\\n\\r\\n\\r\\nwe try the parameter with uppercase now\\r\\n\\r\\n\\r\\n open: False\\r\\n\\r\\n\\r\\n\\r\\n\\r\\nError in run\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 500, in wait_for\\r\\n return fut.result()\\r\\n ^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 239, in _get_next_ackcmd\\r\\n await self._next_ack_event.wait()\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/locks.py\", line 213, in wait\\r\\n await fut\\r\\nasyncio.exceptions.CancelledError\\r\\n\\r\\n\\r\\nThe above exception was the direct cause of the following exception:\\r\\n\\r\\n\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 189, in next_ackcmd\\r\\n ackcmd = await self._wait_task\\r\\n ^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 214, in _basic_next_ackcmd\\r\\n ackcmd = await asyncio.wait_for(\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 502, in wait_for\\r\\n raise exceptions.TimeoutError() from exc\\r\\nTimeoutError\\r\\n\\r\\n\\r\\nDuring handling of the above exception, another exception occurred:\\r\\n\\r\\n\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\\r\\n await self._run_task\\r\\n File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/run_command.py\", line 170, in run\\r\\n await getattr(self.remote, f\"cmd_{self.cmd}\").start(\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\\r\\n return await cmd_info.next_ackcmd(timeout=timeout)\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 205, in next_ackcmd\\r\\n raise base.AckTimeoutError(\\r\\nlsst.ts.salobj.base.AckTimeoutError: msg=\\'Timed out waiting for command acknowledgement\\', ackcmd=(ackcmd private_seqNum=1142844592, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result=\\'No command acknowledgement seen\\')\\r\\n',\n", + " 'level': 0,\n", + " 'tags': [],\n", + " 'urls': [],\n", + " 'time_lost': 0.0,\n", + " 'date_begin': '2024-03-20T04:15:26.769000',\n", + " 'user_id': 'cmorales@summit-lsp.lsst.codes',\n", + " 'user_agent': 'LOVE',\n", + " 'is_human': True,\n", + " 'is_valid': True,\n", + " 'date_added': '2024-03-20T04:16:06.976595',\n", + " 'date_invalidated': None,\n", + " 'parent_id': None,\n", + " 'systems': None,\n", + " 'subsystems': None,\n", + " 'cscs': None,\n", + " 'date_end': '2024-03-20T04:15:26.769000',\n", + " 'components': ['AuxTel'],\n", + " 'primary_software_components': ['None'],\n", + " 'primary_hardware_components': ['None'],\n", + " 'category': 'None',\n", + " 'time_lost_type': 'fault'}" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "recs[0]" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "343f9f9c-ffc2-4b06-b18c-3086fe17885f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
date_addedtime_lost
02024-03-20T04:16:06.9765950.0
12024-04-24T20:54:19.3466460.0
22024-06-19T01:59:58.7013430.0
32024-08-21T04:22:26.2326530.0
42023-11-21T19:55:36.7871050.0
.........
9942023-11-16T02:57:08.8861320.0
9952024-04-19T23:27:40.2327910.0
9962024-01-05T20:16:07.1721730.0
9972024-07-15T19:40:15.7077530.0
9982024-02-21T16:11:05.7723510.0
\n", + "

999 rows × 2 columns

\n", + "
" + ], + "text/plain": [ + " date_added time_lost\n", + "0 2024-03-20T04:16:06.976595 0.0\n", + "1 2024-04-24T20:54:19.346646 0.0\n", + "2 2024-06-19T01:59:58.701343 0.0\n", + "3 2024-08-21T04:22:26.232653 0.0\n", + "4 2023-11-21T19:55:36.787105 0.0\n", + ".. ... ...\n", + "994 2023-11-16T02:57:08.886132 0.0\n", + "995 2024-04-19T23:27:40.232791 0.0\n", + "996 2024-01-05T20:16:07.172173 0.0\n", + "997 2024-07-15T19:40:15.707753 0.0\n", + "998 2024-02-21T16:11:05.772351 0.0\n", + "\n", + "[999 rows x 2 columns]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "cols = ['date_added', 'time_lost']\n", + "df = pd.DataFrame(recs)[cols]\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "72f1a445-c5b9-4efa-a5fc-77e1c4cee84e", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "tl.csv
" + ], + "text/plain": [ + "/home/pothiers/sandbox/ts_notebooks/logging/tl.csv" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# Allow download of CSV version of DataFrame\n", + "csvfile = 'tl.csv'\n", + "df.to_csv(csvfile)\n", + "myfile = FileLink(csvfile)\n", + "display(myfile)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "a74d281f-952d-42d3-bf35-949a17000b49", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqYAAAGdCAYAAADALrFsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAeVklEQVR4nO3df3DdZb3g8U9K27SFJlBpG0qDFnHR+gPcihjwR3E6FIZ15coyOq5KlQHXbd0LdXToiDCg2FmGQdRbKd6FVkdYXPeuoFy2UisWdymgdbsI0l6rSGtr0vKjSVtofjRn/3B7khNSmtPk5Hlyzus1c2ZyTs735IG0ybvP833Ot65QKBQCAAASG5d6AAAAECFMAQDIhDAFACALwhQAgCwIUwAAsiBMAQDIgjAFACALwhQAgCyMTz2AWtHb2xs7d+6MqVOnRl1dXerhAABDUCgUYu/evTFr1qwYN858XqUJ01Gyc+fOaG5uTj0MAOAobN++PWbPnp16GFVPmI6SqVOnRsTf/mA3NDQkHg0AMBQdHR3R3Nxc/D1OZQnTUXJo+b6hoUGYAsAY4zS80eFkCQAAsiBMAQDIgjAFACALwhQAgCwIUwAAsiBMAQDIgjAFACALwhQAgCwIUwAAsiBMAQDIgjAFACALwhQAgCwIU6rGge6D8Y+P/Cm27tp7VMc//qcX4pdbdo3wqBgpu/YeiHuf2BYvd/WkHgoAFSJMqRrf+eUf46YHn4kFtz5S9rGFQiE++t3HYtGqX8fz+zorMDqG66N3PBbX/I/fxVcfeCb1UACoEGFK1fg/21466mMLhb6PX9zfNQKjYaQ9+/z+iIj4+TNtiUcCQKUIUwAAsiBMISIKR34KAFBhwhTib+eYAgBpCVMAALIgTCEs5QNADoQpROmufAAgDWEKAEAWhClERMFiPgAkJ0whLOUDQA6EKQAAWRCmAABkQZhCWMoHgBwIUwAAsiBMIezKB4AcCFMIS/kAkANhCuGSpACQA2EKAEAWhClERMFaPgAkJ0whLOUDQA6EKQAAWRCmEHblA0AOhClEWMsHgAwI04hYvnx5nHXWWTF16tSYMWNGXHzxxbFly5aS5xw4cCAWL14cr3vd6+K4446LSy65JNra2hKNGACg+gjTiFi/fn0sXrw4HnvssVi7dm10d3fH+eefH/v37y8+5+qrr46f/vSn8aMf/SjWr18fO3fujI985CMJR81IcuUnAEhvfOoB5GDNmjUl91evXh0zZsyIjRs3xvvf//5ob2+PO++8M+6555744Ac/GBERq1atire85S3x2GOPxXve854Uw2YEOccUANIzYzqI9vb2iIiYNm1aRERs3Lgxuru7Y8GCBcXnvPnNb45TTjklNmzYkGSMAADVxozpAL29vXHVVVfFueeeG29729siIqK1tTUmTpwYxx9/fMlzZ86cGa2trYO+TmdnZ3R2dhbvd3R0VGzMDJ8JUwBIz4zpAIsXL46nnnoq7r333mG9zvLly6OxsbF4a25uHqERUgmu/AQA6QnTfpYsWRIPPPBAPPzwwzF79uzi401NTdHV1RV79uwpeX5bW1s0NTUN+lrLli2L9vb24m379u2VHDoAwJgnTONvs2VLliyJH//4x/GLX/wi5syZU/L5efPmxYQJE2LdunXFx7Zs2RLbtm2LlpaWQV+zvr4+GhoaSm7ky3wpAKTnHNP42/L9PffcE/fff39MnTq1eN5oY2NjTJ48ORobG+Pyyy+PpUuXxrRp06KhoSE+//nPR0tLix35VaL/Sr5VfQBIQ5hGxO233x4REfPnzy95fNWqVbFo0aKIiPjGN74R48aNi0suuSQ6Oztj4cKF8Z3vfGeUR0qleB9TAEhPmMbQNr5MmjQpVqxYEStWrBiFEZGSSAWANJxjChElJ5laygeANIQphM1PAJADYQoDmDEFgDSEKcSAXfnmTwEgCWEKIUYBIAfCFAawlA8AaQhTCDEKADkQphB25QNADoQpDGD2FADSEKYQpVf/shEKANIQphAD3i5KlwJAEsIUAIAsCFMYwIQpAKQhTCEGLuVLUwBIQZhC2PAEADkQpjCARAWANIQphF35AJADYQphlhQAciBM4VVkKgCkIEwhBlz5SZcCQBLCFMIcKQDkQJjCACIVANIQphB25QNADoQpRIR5UgBIT5jCAC5JCgBpCFOIAUv56YYBADVNmEKIUQDIgTCFsPkJAHIgTGGAgvlTAEhCmEIMiFFdCgBJCFMIy/cAkANhCgNoVABIQ5hC2PwEADkQphA2PAFADoQpDCBSASANYQphKR8AciBMAQDIgjCFAUyYAkAawhRi4FK+NAWAFIQphA1PAJADYQoDSFQASEOYQgzYia9MASAJYQqhRQEgB8IUonTDk/NNASANYQoD2JQPAGkIUwhL+QCQA2EK4ZKkAJADYQoD6FIASEOYQkTIUQBIT5hCuCQpAORAmMIAshQA0hCmEKUxasIUANIQphBiFAByIEzhVVQqAKQgTCEGXJJUlwJAEsIUwhwpAORAmEIMeLuodMMAgJomTGEAS/kAkIYwhYgomCcFgOSEaUQ88sgj8aEPfShmzZoVdXV1cd9995V8vlAoxHXXXRcnnXRSTJ48ORYsWBB/+MMf0gyWyihZyhepAJCCMI2I/fv3xxlnnBErVqwY9PM333xzfOtb34qVK1fG448/Hscee2wsXLgwDhw4MMojZTRYygeANManHkAOLrzwwrjwwgsH/VyhUIjbbrstrr322vjwhz8cERHf//73Y+bMmXHffffFxz72sdEcKhWiRQEgPTOmR/Dss89Ga2trLFiwoPhYY2NjnH322bFhw4aEI2Mk2ZUPAOmZMT2C1tbWiIiYOXNmyeMzZ84sfm4wnZ2d0dnZWbzf0dFRmQEy4grW8gEgCTOmFbJ8+fJobGws3pqbm1MPiddgwxMApCdMj6CpqSkiItra2koeb2trK35uMMuWLYv29vbibfv27RUdJ8NjkhQA0hOmRzBnzpxoamqKdevWFR/r6OiIxx9/PFpaWg57XH19fTQ0NJTcGBtEKgCk4RzTiNi3b19s3bq1eP/ZZ5+NTZs2xbRp0+KUU06Jq666Kr72ta/Fm970ppgzZ0585StfiVmzZsXFF1+cbtCMKC0KAOkJ04j4zW9+E+edd17x/tKlSyMi4rLLLovVq1fHl770pdi/f39ceeWVsWfPnnjve98ba9asiUmTJqUaMiOs/4Yn55sCQBrCNCLmz5//mjux6+rq4sYbb4wbb7xxFEdFKpbyASAN55hSlcp9y6f+zxamAJCGMKUqlR2XYhQAkhOmVKXyu7T/OaYAQArCFAZw5ScASEOYUpXKPsdUiwJAcsKUqlT2Un6/AzQqAKQhTGEgZQoASQhTqlK5S/NaFADSE6ZUpXKv3uTKTwCQnjCFAWyEAoA0hClVyVI+AIw9whTCrnwAyIEwhQEs5QNAGsKUqlR+XKpRAEhNmFKVyt+Vf/THAgAjQ5jCAJbyASANYUpVsisfAMYeYUpVKjc07coHgPSEKVWpUOaUacl5pdbyASAJYQoAQBaEKVXJUj4AjD3ClKo0nM1PVvIBIA1hCgOUe34qADAyhCnVqdwZUzEKAMkJU6rScK7eJFEBIA1hCgOYPAWANIQpVanszU9iFACSE6ZUpbLfLqrfERoV4PCck08lCVMYwA9dgMHt3tsZZ399XSz/n8+kHgpVSphSlcq+JKkWBTii7z7yx9i1tzPuWP+n1EOhSglTqtJwrvwEwOD8rKTShClVo+Syoq78BABjjjAFACALwpSqUbqzvtxzTI/+WCrPhjSA2iBMqRol7WIpv6r4ngDUBmEKAEAWhClVo2TzU9kHD+NYKs73BKA2CFOqRsk5pmUv5R/9sVSec0wBaoMwBQAgC8KUqlG6lH/0V36yKz8/viMAtUGYUjWGs7Pervy8+Z4A1AZhCgBAFoQp1WMYO+vNyOXN6RUAtUGYUjVKd9aXeY7pMI6l8nxLOJLe3kJc809Pxt2PP5d6KDXDz0oqQZjCAH7WwtizbvOuuPfX2+PLP34q9VBqxsFePywZecKUqlGys77czU9+vsKY1v5Kd+oh1JyDfnBSAcKUqjGcH5GFw3xMHvz+g/z09qYeAdVImFI1+p/vVHbIDOdYKs7mJ8hPjzKlAoQpAFXFppzRoUupBGFK1Shdji93V/7RH0vl6QzK0WNTzqhwjimVIEypGiO1+cnP2vz4llCO7oOm8iqlf/NbyqcShCkAVaW7xz9lKqW337/cdSmVIEypGsPZWV+ycWpERsNIcs4gR9Lbbyqvy4xpxfR/71JL+VSCMKV6lOysP/pzTK3l58d35PCe/Mue+I93b4znXtifeihJdfebvrOUXzn9z989eNDfTEbe+NQDAODo/dt/+N8REfHcCy/HP/+n9yUeTTrdPcJ0NPSaMaXCzJhSNYa3lH/0x1J5fv8d2R927Us9hKJ/fORPcevafxnVr9ndb/ZOmFZO/xh1SVIqwYwpVWNYu/IP8zpkwvdkzOg+2Bs3PfhMRERcOm92NE+bMipft/95pV02P1VMyTmmwpQKMGMKwIjp6HfN+v1dPaP2dfvPktr8VDnClEoTplSNwjAW80t35Q9+7Lpn2uLWh7aUnGPF6HDRg7Gj40BfjO47kCZMj7SU710ejp4wpdKEaZlWrFgRb3jDG2LSpElx9tlnxxNPPJF6SPx/I/Um+Yc79vLv/Sa+9YutsfaZtqN/cY6Kjhg7+s+Ytvf7uNJKzjHtOXyYvrCvM1qW/yKuv/+p0RhW1fF2UVSaMC3DD3/4w1i6dGlcf/318dvf/jbOOOOMWLhwYezatSv10Kiwnn4zMNtffDnhSIbu/k074v5NO1IPgxrTcaAvRve8PHph2tUztKX8e3+9PVo7DsT3Njw3GsOqOjY/UWnCtAy33nprXHHFFfHpT3865s6dGytXrowpU6bEXXfdlXpoxPB21h/p2Of3dRU/7n6N9+5b89Rf44afPh2dPQfLHEF52l/pjvs37TjskuXz+zrj7+/dFH9/76bYvbezomMZDbn/+st9F/iTf9kTG597aURf80D3wUH/kdbxSt/y/ejOmPZfyj/8n5jO7r6/m/s7X32qQVdPb1y++tdxzT89ObIDrBKW8qk0u/KHqKurKzZu3BjLli0rPjZu3LhYsGBBbNiwIdm4tr/4ctzzxLZkXz8nu/YeKH5856+ejWnHTRzysZu27Sl+vOGPL8R/XrO55PPP94u7nz3dWjIr1N/tv/xjRET8+fn98eaTGob0tfe83BWTJ4yP+glD/3fif/nVn6L7YCGWP7g5/u5fn/yqz+/c80rx46/98+9j1vGTh/zaQ9FbKMT/3b4n6scfE285qSHq6kb05V/lla6+mNh3oOdV35+Ufv3si/H0zo44783T4/WvOzbZOLp6egf9//L83s740ca/RETEh8+cNeifhUIh4qX9XXH8lAkxbtyRv5mFQsR/fWJbtL/SHQveMiPeOP242PNyd5xw7MTY0tpRfN6d/+vZ2L1vdP5h1D+8//vG7fHbbYOH+KN/fKH48U0PPhONkyeUfL6t40Cs2/y3VbApE8v7e1kLtvZ7W7K7H38uHt6S14rh7BMmx78/+/Wph8Ew1BWcBT4kO3fujJNPPjkeffTRaGlpKT7+pS99KdavXx+PP/54yfM7Ozujs7PvB3JHR0c0NzdHe3t7NDQMLViG4jd/fjH+3cp0YQwAuTjrDSfEj/7DOSP6mh0dHdHY2Djiv78ZnBnTClm+fHnccMMNFf86MxsmxeXvnVPxrzNW7NzzSpzUOPmoZvD2d/ZET2/hVTMoh3T2HIw9L3fHzIZJh32N7oO98df2A3HKEN+7cf2/7C7OQMw/fXq8cfpxQzquUIj4w6698a9mTj3scw6d33f8lMH/e4Zrx0uvxMTx42L61PqKvP5gdu55ZcRnf4frYG8hnt/X+Zp/Liptx0uvxMknHP7/y0v7u6Knt/Ca36u6KO+Uie6DvfHcCy/HaTOOe9Xx9ePHxXGTxscL/U6BGQ0v7OuMY+vHx6QJx7zm81rbD8TrjpsYE44ZfDZ0997OmDRhXEydVJm/O2NdW8eBeN2xE2P8Yf7/pdT8Gn8PGBuE6RCdeOKJccwxx0RbW+mO7La2tmhqanrV85ctWxZLly4t3j80YzrSmqdNia/8m7kj/rqMjpf+26ZimP7dO0+OD5/56mV5AKgV+f1zJ1MTJ06MefPmxbp164qP9fb2xrp160qW9g+pr6+PhoaGkhsAAIdnxrQMS5cujcsuuyze9a53xbvf/e647bbbYv/+/fHpT3869dAYo+qi75yDukrvIAKAzAnTMnz0ox+N3bt3x3XXXRetra1x5plnxpo1a2LmzJmph8YY1b9FZSkAtU6YlmnJkiWxZMmS1MOgSvSPUROmANQ655gCAJAFYQoJlS7lmzIFoLYJU0iodPNTwoEAQAaEKQAAWRCmkJBd+QDQR5hCQiVhqkwBqHHCFACALAhTSKruMB8DQO0RppCQpXwA6CNMAQDIgjCFhCzkA0AfYQoJlS7lS1MAapswhYRKrvyUcBwAkANhCgBAFoQpJGRXPgD0EaaQUMnmJ2EKQI0TpgAAZEGYQkL9d+LX2f4EQI0TppALXQpAjROmAABkQZhCQiW78tMNAwCyIEwhoZI32LctH4AaJ0wBAMiCMIWELOUDQB9hCgl5g30A6CNMAQDIgjCFhEqX8k2ZAlDbhCkkVHLlJ10KQI0TppBQ3WE+BoBaJEwBAMiCMIWUTJkCQJEwhYRKrvykTAGoccIUAIAsCFNIqOTtokyYAlDjhCkk5BRTAOgjTAEAyIIwhYRKl/LNmQJQ24QpJFSyK1+XAlDjhCkAAFkQppBQyVJ+umEAQBaEKSRUsitfmQJQ44QpAABZEKaQUp13MgWAQ4QpJGQpHwD6CFNIyOYnAOgjTAEAyIIwhYRK32DfnCkAtU2YQkKW8gGgjzAFACALwhQSsisfAPoIU0iodClfmQJQ24QpAABZEKaQUP+d+JbyAah1whQAgCwIUwAAsiBMIaGSzU+W8gGoccIUEiq58pNd+QDUOGEKCZkxBYA+whQAgCwI04i46aab4pxzzokpU6bE8ccfP+hztm3bFhdddFFMmTIlZsyYEV/84hejp6dndAdK1XHlJwDoMz71AHLQ1dUVl156abS0tMSdd975qs8fPHgwLrroomhqaopHH300/vrXv8anPvWpmDBhQnz9619PMGKqhSs/AUAfM6YRccMNN8TVV18db3/72wf9/EMPPRS///3v4wc/+EGceeaZceGFF8ZXv/rVWLFiRXR1dY3yaAEAqpMwHYINGzbE29/+9pg5c2bxsYULF0ZHR0c8/fTTgx7T2dkZHR0dJTcYqGRXvglTAGqcMB2C1tbWkiiNiOL91tbWQY9Zvnx5NDY2Fm/Nzc0VHydjT+lSPgDUtqoN02uuuSbq6upe87Z58+aKff1ly5ZFe3t78bZ9+/aKfS0AgGpQtZufvvCFL8SiRYte8zmnnnrqkF6rqakpnnjiiZLH2traip8bTH19fdTX1w/p9SHCUj4AVG2YTp8+PaZPnz4ir9XS0hI33XRT7Nq1K2bMmBEREWvXro2GhoaYO3fuiHwNalNdSY0qUwBqW9WGaTm2bdsWL774Ymzbti0OHjwYmzZtioiI0047LY477rg4//zzY+7cufHJT34ybr755mhtbY1rr702Fi9ebFYUAGCECNOIuO666+J73/te8f473/nOiIh4+OGHY/78+XHMMcfEAw88EJ/73OeipaUljj322LjsssvixhtvTDVkqoQ32AeAPsI0IlavXh2rV69+zee8/vWvjwcffHB0BkTNsCsfAPpU7a58AADGFmEKCZUu5ZszBaC2CVNIqH+MylIAap0whYRKzjFVpgDUOGEKAEAWhCkkVPr2+qZMAahtwhRS6n+OqS4FoMYJUwAAsiBMISGTpADQR5hCQnblA0AfYQoAQBaEKSTUfye+Kz8BUOuEKSRUspSfbhgAkAVhCgBAFoQpJFTyBvumTAGoccIUEipdylemANQ2YQoAQBaEKSRUuis/4UAAIAPCFFKyKx8AioQpJFR32DsAUHuEKQAAWRCmkFD/qz3ZlQ9ArROmkJD3MQWAPsIUAIAsCFNIqM6ufAAoEqaQiTpr+QDUOGEKAEAWhClkwnwpALVOmEImrOQDUOuEKQAAWRCmkAlvsA9ArROmkAtdCkCNE6aQCeeYAlDrhCkAAFkQppAJE6YA1DphCplw5ScAap0wBQAgC8IUMmG+FIBaJ0whE1byAah1whQAgCwIU8iEKz8BUOuEKWTCUj4AtU6YAgCQBWEKAEAWhClkwlI+ALVOmAIAkAVhCpmwKx+AWidMIROW8gGodcIUMqFLAah1whQAgCwIU8hEnbV8AGqcMIVMyFIAap0wBQAgC8IUMmElH4BaJ0whE84xBaDWCVMAALIgTAEAyIIwBQAgC8IUAIAs1HyY/vnPf47LL7885syZE5MnT443vvGNcf3110dXV1fJ85588sl43/veF5MmTYrm5ua4+eabE40YAKA6jU89gNQ2b94cvb29cccdd8Rpp50WTz31VFxxxRWxf//+uOWWWyIioqOjI84///xYsGBBrFy5Mn73u9/FZz7zmTj++OPjyiuvTPxfAABQHWo+TC+44IK44IILivdPPfXU2LJlS9x+++3FML377rujq6sr7rrrrpg4cWK89a1vjU2bNsWtt94qTAEARkjNL+UPpr29PaZNm1a8v2HDhnj/+98fEydOLD62cOHC2LJlS7z00kuDvkZnZ2d0dHSU3AAAODxhOsDWrVvj29/+dnz2s58tPtba2hozZ84sed6h+62trYO+zvLly6OxsbF4a25urtygAQCqQNWG6TXXXBN1dXWvedu8eXPJMTt27IgLLrggLr300rjiiiuG9fWXLVsW7e3txdv27duH9XoAANWuas8x/cIXvhCLFi16zeeceuqpxY937twZ5513Xpxzzjnx3e9+t+R5TU1N0dbWVvLYoftNTU2DvnZ9fX3U19cfxcgBAGpT1Ybp9OnTY/r06UN67o4dO+K8886LefPmxapVq2LcuNKJ5JaWlvjyl78c3d3dMWHChIiIWLt2bZx++ulxwgknjPjYAQBqUdUu5Q/Vjh07Yv78+XHKKafELbfcErt3747W1taSc0c//vGPx8SJE+Pyyy+Pp59+On74wx/GN7/5zVi6dGnCkQMAVJeqnTEdqrVr18bWrVtj69atMXv27JLPFQqFiIhobGyMhx56KBYvXhzz5s2LE088Ma677jpvFQUAMIJqPkwXLVp0xHNRIyLe8Y53xK9+9avKDwgAoEbV/FI+AAB5EKYAAGRBmAIAkAVhCgBAFoQpAABZEKYAAGRBmAIAkAVhCgBAFoQpAABZEKYAAGRBmAIAkAVhCgBAFoQpAABZEKYAAGRBmAIAkAVhCgBAFoQpAABZEKYAAGRBmAIAkAVhCgBAFoQpAABZEKaQ0JwTj009BADIxvjUA4Ba9s5TTohvfuzMOGXalNRDAYDkhCkk9uEzT049BADIgqV8AACyIEwBAMiCMAUAIAvCFACALAhTAACyIEwBAMiCMAUAIAvCFACALAhTAACyIEwBAMiCMAUAIAvCFACALAhTAACyMD71AGpFoVCIiIiOjo7EIwEAhurQ7+1Dv8epLGE6Svbu3RsREc3NzYlHAgCUa+/evdHY2Jh6GFWvruCfAKOit7c3du7cGVOnTo26uroRfe0dO3bE3LlzR/Q1AWAs2r59ezQ0NIzY6xUKhdi7d2/MmjUrxo1zBmSlmTEdJePGjYvZs2dX5LWdHgAAf9PQ0DCiYRoRZkpHkfQHACALwhQAgCxYyq8CDQ0Nce6550ZPT0/xXNZD58IMvB8RQ3rsaI8bydcaK8flMIaxclwOYxgrx+UwhrFyXA5jGCvH5TCGSh03fvz4+MAHPhD19fXB2GXzEwAAWbCUDwBAFoQpAABZEKYAAGRBmAIAkIWyduUvX748brjhhujs7KzUeAAAqAIXXHBB3HnnnTFr1qwhH1PWjOn69eujp6cnjjnmmLIHBwBA7fj5z38el1xySVnHDOvtonbv3h0zZsw42sMBAKhidXV10dnZGRMmTBjS84f1Bvvt7e3DORwAgCo1YcKEOOuss4YcpRHD2PzU29sbn//854/2cAAAqthpp50WP/nJT8o65qjDdPHixfGzn/3saA8HAKCKPffcc/GJT3wiyjlr9KjCdMmSJXHHHXeU9YUAAKgdBw8ejDVr1sRjjz025GPKCtNCoRCf+9znYsWKFaIUAIBXOXRO6aFWLOdtRssK08WLF8fKlSvLOQQAgBrS3d0dERE9PT1x0kknxemnnz7kY8t6u6i6urryRwcAQM1atWpVLFq0aEjPLevtoizfAwBQKUe9Kx8AAEaSMAUAIAvCFACALAhTAACyIEwBAMiCMAUAIAvCFACALAhTAACyIEwBAMiCMAUAIAvCFACALAhTAACy8P8AoPdysIIaqGcAAAAASUVORK5CYII=", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "x = [r['date_added'] for r in recs]\n", + "y = [r['time_lost'] for r in recs]\n", + "plt.plot(x, y) \n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "138e5496-e058-417a-881e-8450e20af0f9", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "0cb5feca-620b-49a0-b25b-835a6d38843a", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'id': '0005d0e0-6de7-4857-b658-cfca7147374a',\n", + " 'site_id': 'summit',\n", + " 'message_text': 'The moveShutterMainDoor command Fail\\r\\n\\r\\n\\r\\nwe try the parameter with uppercase now\\r\\n\\r\\n\\r\\n open: False\\r\\n\\r\\n\\r\\n\\r\\n\\r\\nError in run\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 500, in wait_for\\r\\n return fut.result()\\r\\n ^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 239, in _get_next_ackcmd\\r\\n await self._next_ack_event.wait()\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/locks.py\", line 213, in wait\\r\\n await fut\\r\\nasyncio.exceptions.CancelledError\\r\\n\\r\\n\\r\\nThe above exception was the direct cause of the following exception:\\r\\n\\r\\n\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 189, in next_ackcmd\\r\\n ackcmd = await self._wait_task\\r\\n ^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 214, in _basic_next_ackcmd\\r\\n ackcmd = await asyncio.wait_for(\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 502, in wait_for\\r\\n raise exceptions.TimeoutError() from exc\\r\\nTimeoutError\\r\\n\\r\\n\\r\\nDuring handling of the above exception, another exception occurred:\\r\\n\\r\\n\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\\r\\n await self._run_task\\r\\n File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/run_command.py\", line 170, in run\\r\\n await getattr(self.remote, f\"cmd_{self.cmd}\").start(\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\\r\\n return await cmd_info.next_ackcmd(timeout=timeout)\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 205, in next_ackcmd\\r\\n raise base.AckTimeoutError(\\r\\nlsst.ts.salobj.base.AckTimeoutError: msg=\\'Timed out waiting for command acknowledgement\\', ackcmd=(ackcmd private_seqNum=1142844592, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result=\\'No command acknowledgement seen\\')\\r\\n',\n", + " 'level': 0,\n", + " 'tags': [],\n", + " 'urls': [],\n", + " 'time_lost': 0.0,\n", + " 'date_begin': '2024-03-20T04:15:26.769000',\n", + " 'user_id': 'cmorales@summit-lsp.lsst.codes',\n", + " 'user_agent': 'LOVE',\n", + " 'is_human': True,\n", + " 'is_valid': True,\n", + " 'date_added': '2024-03-20T04:16:06.976595',\n", + " 'date_invalidated': None,\n", + " 'parent_id': None,\n", + " 'systems': None,\n", + " 'subsystems': None,\n", + " 'cscs': None,\n", + " 'date_end': '2024-03-20T04:15:26.769000',\n", + " 'components': ['AuxTel'],\n", + " 'primary_software_components': ['None'],\n", + " 'primary_hardware_components': ['None'],\n", + " 'category': 'None',\n", + " 'time_lost_type': 'fault'},\n", + " {'id': '00067b6a-1741-41f3-87af-57261dff30ac',\n", + " 'site_id': 'summit',\n", + " 'message_text': 'David updated environment. But no need to cycle Scheduler 1. Waiting for completion of the current block. \\r\\n',\n", + " 'level': 0,\n", + " 'tags': [],\n", + " 'urls': [],\n", + " 'time_lost': 0.0,\n", + " 'date_begin': '2024-04-24T20:53:41.843000',\n", + " 'user_id': 'ykang@summit-lsp.lsst.codes',\n", + " 'user_agent': 'LOVE',\n", + " 'is_human': True,\n", + " 'is_valid': True,\n", + " 'date_added': '2024-04-24T20:54:19.346646',\n", + " 'date_invalidated': None,\n", + " 'parent_id': None,\n", + " 'systems': None,\n", + " 'subsystems': None,\n", + " 'cscs': None,\n", + " 'date_end': '2024-04-24T20:53:41.843000',\n", + " 'components': ['MainTel'],\n", + " 'primary_software_components': ['None'],\n", + " 'primary_hardware_components': ['None'],\n", + " 'category': 'None',\n", + " 'time_lost_type': 'fault'},\n", + " {'id': '000a3ba8-b1a9-4b04-9c4a-263da3dac1d1',\n", + " 'site_id': 'summit',\n", + " 'message_text': 'Runing BLOCK-T22 AuxTel filter Scans, the auxtel/calibrations/run_calibration_sequence.py went to fault.\\r\\n\\r\\n\\r\\nsalIndex: 200068\\r\\nFile \"/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/auxtel/atcalsys.py\", line 338, in _take_data\\r\\n await asyncio.gather(\\r\\n File \"/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/auxtel/atcalsys.py\", line 430, in take_fiber_spectrum\\r\\n await self.fiberspectrograph.cmd_expose.set_start(\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 416, in set_start\\r\\n return await self.start(timeout=timeout, wait_done=wait_done)\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\\r\\n return await cmd_info.next_ackcmd(timeout=timeout)\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 191, in next_ackcmd\\r\\n raise base.AckError(msg=\"Command failed\", ackcmd=ackcmd)\\r\\nlsst.ts.salobj.base.AckError: msg=\\'Command failed\\', ackcmd=(ackcmd private_seqNum=1157551610, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result=\"Failed: 502, message=\\'Bad Gateway\\', url=URL(\\'http://comcam-mcm-dds.cp.lsst.org/ImageUtilities/rest/imageNameService?n=1&sourceIndex=3&source=FiberSpectrograph\\')\")\\r\\n',\n", + " 'level': 0,\n", + " 'tags': [],\n", + " 'urls': [],\n", + " 'time_lost': 0.0,\n", + " 'date_begin': '2024-06-19T01:59:08.706000',\n", + " 'user_id': 'cmd_user@summit-lsp.lsst.codes',\n", + " 'user_agent': 'LOVE',\n", + " 'is_human': True,\n", + " 'is_valid': False,\n", + " 'date_added': '2024-06-19T01:59:58.701343',\n", + " 'date_invalidated': '2024-06-19T02:12:01.302498',\n", + " 'parent_id': 'af0157e3-d7f0-4068-9a8a-e51a0a77d260',\n", + " 'systems': None,\n", + " 'subsystems': None,\n", + " 'cscs': None,\n", + " 'date_end': '2024-06-19T01:59:08.706000',\n", + " 'components': ['AuxTel'],\n", + " 'primary_software_components': ['None'],\n", + " 'primary_hardware_components': ['None'],\n", + " 'category': 'None',\n", + " 'time_lost_type': 'fault'},\n", + " {'id': '0010e246-8039-4741-b4c6-54047e9464c5',\n", + " 'site_id': 'summit',\n", + " 'message_text': 'Still checking updates in the wavefront estimation pipeline version, offsetting manually the ATHexapod in different axes and deploying different versions of the pipeline.\\r\\n',\n", + " 'level': 0,\n", + " 'tags': [],\n", + " 'urls': [],\n", + " 'time_lost': 0.0,\n", + " 'date_begin': '2024-08-21T04:04:38.053000',\n", + " 'user_id': 'kpena@summit-lsp.lsst.codes',\n", + " 'user_agent': 'LOVE',\n", + " 'is_human': True,\n", + " 'is_valid': True,\n", + " 'date_added': '2024-08-21T04:22:26.232653',\n", + " 'date_invalidated': None,\n", + " 'parent_id': '99de2a12-20eb-45ae-b189-a793b63e6ab5',\n", + " 'systems': None,\n", + " 'subsystems': None,\n", + " 'cscs': None,\n", + " 'date_end': '2024-08-21T04:04:38.053000',\n", + " 'components': ['AuxTel'],\n", + " 'primary_software_components': ['None'],\n", + " 'primary_hardware_components': ['None'],\n", + " 'category': 'None',\n", + " 'time_lost_type': 'fault'}]" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "recs[:4]" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "f6441d81-93a8-4369-93fb-55d3715783cc", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
idsite_idmessage_textleveltagsurlstime_lostdate_beginuser_iduser_agent...parent_idsystemssubsystemscscsdate_endcomponentsprimary_software_componentsprimary_hardware_componentscategorytime_lost_type
00005d0e0-6de7-4857-b658-cfca7147374asummitThe moveShutterMainDoor command Fail\\r\\n\\r\\n\\r...0[][]0.02024-03-20T04:15:26.769000cmorales@summit-lsp.lsst.codesLOVE...NoneNoneNoneNone2024-03-20T04:15:26.769000[AuxTel][None][None]Nonefault
100067b6a-1741-41f3-87af-57261dff30acsummitDavid updated environment. But no need to cycl...0[][]0.02024-04-24T20:53:41.843000ykang@summit-lsp.lsst.codesLOVE...NoneNoneNoneNone2024-04-24T20:53:41.843000[MainTel][None][None]Nonefault
2000a3ba8-b1a9-4b04-9c4a-263da3dac1d1summitRuning BLOCK-T22 AuxTel filter Scans, the auxt...0[][]0.02024-06-19T01:59:08.706000cmd_user@summit-lsp.lsst.codesLOVE...af0157e3-d7f0-4068-9a8a-e51a0a77d260NoneNoneNone2024-06-19T01:59:08.706000[AuxTel][None][None]Nonefault
30010e246-8039-4741-b4c6-54047e9464c5summitStill checking updates in the wavefront estima...0[][]0.02024-08-21T04:04:38.053000kpena@summit-lsp.lsst.codesLOVE...99de2a12-20eb-45ae-b189-a793b63e6ab5NoneNoneNone2024-08-21T04:04:38.053000[AuxTel][None][None]Nonefault
40017b90e-26aa-4a7c-8186-fc06ac9b5aa6summitThe m1m3 ump test failed for actuator 108. \\r\\...0[][]0.02023-11-21T19:51:54.868000bquint@love01.cp.lsst.orgLOVE...NoneNoneNoneNone2023-11-21T19:51:54.868000[MainTel][None][None]Nonefault
..................................................................
99422325b7d-7948-4da8-88ec-2a47ceb088e0summitBlock 109 starting again\\r\\n0[][]0.02023-11-16T02:56:30.009000echristensen@love01.cp.lsst.orgLOVE...NoneNoneNoneNone2023-11-16T02:56:30.009000[MainTel][None][None]Nonefault
9952233da67-c777-4dd6-bc66-a241f93df5aesummitAz encoder checking thru slew test. \\r\\n\\r\\n\\r...0[][]0.02024-04-19T23:27:02.011000ykang@summit-lsp.lsst.codesLOVE...NoneNoneNoneNone2024-04-19T23:27:02.011000[MainTel][None][None]Nonefault
996223c0310-03d5-450d-bf5c-0a04bdea6536summitRestarting [BLOCK-178|https://jira.lsstcorp.or...0[][]0.02024-01-05T20:15:27.453000kfanning@love01.cp.lsst.orgLOVE...NoneNoneNoneNone2024-01-05T20:15:27.453000[MainTel][None][None]Nonefault
9972240e19e-7204-4c50-a3ba-c643d60aa0d2summitLATISS enabled but checkout failed \\r\\n\\r\\n\\r\\...0[][]0.02024-07-15T19:39:38.630000kkelkar@summit-lsp.lsst.codesLOVE...NoneNoneNoneNone2024-07-15T19:39:38.630000[AuxTel][None][None]Nonefault
99822475aa1-f15a-46f1-87b8-b7a1708daaf4summitPowercycled M2 cabinet to bring back the M2 co...0[][]0.02024-02-21T15:37:00kfanning@summit-lsp.lsst.codesLOVE...NoneNoneNoneNone2024-02-21T16:10:28.176000[MainTel][None][None]Nonefault
\n", + "

999 rows × 24 columns

\n", + "
" + ], + "text/plain": [ + " id site_id \\\n", + "0 0005d0e0-6de7-4857-b658-cfca7147374a summit \n", + "1 00067b6a-1741-41f3-87af-57261dff30ac summit \n", + "2 000a3ba8-b1a9-4b04-9c4a-263da3dac1d1 summit \n", + "3 0010e246-8039-4741-b4c6-54047e9464c5 summit \n", + "4 0017b90e-26aa-4a7c-8186-fc06ac9b5aa6 summit \n", + ".. ... ... \n", + "994 22325b7d-7948-4da8-88ec-2a47ceb088e0 summit \n", + "995 2233da67-c777-4dd6-bc66-a241f93df5ae summit \n", + "996 223c0310-03d5-450d-bf5c-0a04bdea6536 summit \n", + "997 2240e19e-7204-4c50-a3ba-c643d60aa0d2 summit \n", + "998 22475aa1-f15a-46f1-87b8-b7a1708daaf4 summit \n", + "\n", + " message_text level tags urls \\\n", + "0 The moveShutterMainDoor command Fail\\r\\n\\r\\n\\r... 0 [] [] \n", + "1 David updated environment. But no need to cycl... 0 [] [] \n", + "2 Runing BLOCK-T22 AuxTel filter Scans, the auxt... 0 [] [] \n", + "3 Still checking updates in the wavefront estima... 0 [] [] \n", + "4 The m1m3 ump test failed for actuator 108. \\r\\... 0 [] [] \n", + ".. ... ... ... ... \n", + "994 Block 109 starting again\\r\\n 0 [] [] \n", + "995 Az encoder checking thru slew test. \\r\\n\\r\\n\\r... 0 [] [] \n", + "996 Restarting [BLOCK-178|https://jira.lsstcorp.or... 0 [] [] \n", + "997 LATISS enabled but checkout failed \\r\\n\\r\\n\\r\\... 0 [] [] \n", + "998 Powercycled M2 cabinet to bring back the M2 co... 0 [] [] \n", + "\n", + " time_lost date_begin user_id \\\n", + "0 0.0 2024-03-20T04:15:26.769000 cmorales@summit-lsp.lsst.codes \n", + "1 0.0 2024-04-24T20:53:41.843000 ykang@summit-lsp.lsst.codes \n", + "2 0.0 2024-06-19T01:59:08.706000 cmd_user@summit-lsp.lsst.codes \n", + "3 0.0 2024-08-21T04:04:38.053000 kpena@summit-lsp.lsst.codes \n", + "4 0.0 2023-11-21T19:51:54.868000 bquint@love01.cp.lsst.org \n", + ".. ... ... ... \n", + "994 0.0 2023-11-16T02:56:30.009000 echristensen@love01.cp.lsst.org \n", + "995 0.0 2024-04-19T23:27:02.011000 ykang@summit-lsp.lsst.codes \n", + "996 0.0 2024-01-05T20:15:27.453000 kfanning@love01.cp.lsst.org \n", + "997 0.0 2024-07-15T19:39:38.630000 kkelkar@summit-lsp.lsst.codes \n", + "998 0.0 2024-02-21T15:37:00 kfanning@summit-lsp.lsst.codes \n", + "\n", + " user_agent ... parent_id systems subsystems \\\n", + "0 LOVE ... None None None \n", + "1 LOVE ... None None None \n", + "2 LOVE ... af0157e3-d7f0-4068-9a8a-e51a0a77d260 None None \n", + "3 LOVE ... 99de2a12-20eb-45ae-b189-a793b63e6ab5 None None \n", + "4 LOVE ... None None None \n", + ".. ... ... ... ... ... \n", + "994 LOVE ... None None None \n", + "995 LOVE ... None None None \n", + "996 LOVE ... None None None \n", + "997 LOVE ... None None None \n", + "998 LOVE ... None None None \n", + "\n", + " cscs date_end components primary_software_components \\\n", + "0 None 2024-03-20T04:15:26.769000 [AuxTel] [None] \n", + "1 None 2024-04-24T20:53:41.843000 [MainTel] [None] \n", + "2 None 2024-06-19T01:59:08.706000 [AuxTel] [None] \n", + "3 None 2024-08-21T04:04:38.053000 [AuxTel] [None] \n", + "4 None 2023-11-21T19:51:54.868000 [MainTel] [None] \n", + ".. ... ... ... ... \n", + "994 None 2023-11-16T02:56:30.009000 [MainTel] [None] \n", + "995 None 2024-04-19T23:27:02.011000 [MainTel] [None] \n", + "996 None 2024-01-05T20:15:27.453000 [MainTel] [None] \n", + "997 None 2024-07-15T19:39:38.630000 [AuxTel] [None] \n", + "998 None 2024-02-21T16:10:28.176000 [MainTel] [None] \n", + "\n", + " primary_hardware_components category time_lost_type \n", + "0 [None] None fault \n", + "1 [None] None fault \n", + "2 [None] None fault \n", + "3 [None] None fault \n", + "4 [None] None fault \n", + ".. ... ... ... \n", + "994 [None] None fault \n", + "995 [None] None fault \n", + "996 [None] None fault \n", + "997 [None] None fault \n", + "998 [None] None fault \n", + "\n", + "[999 rows x 24 columns]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "df = pd.DataFrame(recs)\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d8cefdd9-98ec-4383-a297-6853be7d0c9b", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks_tsqr/logrep_proto_1.yaml b/notebooks_tsqr/logrep_proto_1.yaml new file mode 100644 index 0000000..3c1d085 --- /dev/null +++ b/notebooks_tsqr/logrep_proto_1.yaml @@ -0,0 +1,16 @@ +# For use with a Times Square notebook +title: Logging and Reporting +description: Prototype 1 +authors: + - name: Steve Pothier + slack: Steve Pothier +tags: + - reporting + - prototype +parameters: + record_limit: + type: integer + description: Max number of records to output + default: 99 + minimum: 1 + maximum: 9999 diff --git a/notebooks_tsqr/requirements.txt b/notebooks_tsqr/requirements.txt new file mode 100644 index 0000000..5ab865b --- /dev/null +++ b/notebooks_tsqr/requirements.txt @@ -0,0 +1,6 @@ +requests +# jupyter-lab --ip=0.0.0.0 +jupyterlab # =3.1.17 +pandas +matplotlib +#numpy diff --git a/notebooks_tsqr/scaffolding.org b/notebooks_tsqr/scaffolding.org new file mode 100644 index 0000000..6c1c85a --- /dev/null +++ b/notebooks_tsqr/scaffolding.org @@ -0,0 +1,71 @@ +# Scaffolding for Times Square notebooks (thread-1) +* Parameters +- Parameters available to user via Times Square side panel. + +- A schema for these is given in .yaml in same directory. + +- The defaults given here must all be string values. Convert those to + other types in a subsequent cell + +- Include parameters that affect filtering (query) used in Source API + +- Consider common parameters such as: + + record_limit + + num_recs_in_example + + min_date + + max_date + + + +: # Parameters. Set defaults here. +: # Times Square replaces this cell with the user's parameters + + +* Imports and General Setup +- Setup includes conversion of parameter values to usable values +- Get "environment" (server host). e.g. https://usdf-rsp-dev.slac.stanford.edu +- Included non-user parameters: + + timeout + + env + +* Setup Source +- Choose fields of interest. The records (list of dict) in each field + we need in any way. All other fields could be deleted for our + purposes. + +- Choose fields to use for extracting facets. + +- Create full API URL (including query string) from parameters, etc. + +- Choose fields to display to user in table. ~table_fields~ +- Choose fields to display to user in plot. ~(x,y)~ + +* Get Records +- Usually use python "requests" package for accessing API + But, some APIs might have their own client. + +- Use try/except around requests.get() or requests.post() + User friendly error if exception (cannot connect, etc) + User friend error if get non-200 response status. + +- Get list field names from rec[0] + +- Calc facet over selected field names. + +* Table of (mostly raw) results +- Display pandas table of all records using + : df = pd.DataFrame(recs)[table_fields] + +* Plot of results +- Display minimum useful (?!) plot from this Source + Example: + : plt.plot([r['date'] for r in recs], [r['time_lost'] for r in recs]) + : plt.show() + +* Output raw content analysis (rec, fields, facets) +** Example records +** Fields found +** Facets of selected Fields +* Stakeholder Elicitation +- Source specific questions to stakeholders. + Link to Confluence doc (form?) for L&R stakeholder feeback diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c65b6c6 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,7 @@ +requests +# jupyter-lab --ip=0.0.0.0 +jupyterlab # =3.1.17 +pandas +#matplotlib +#numpy +pre-commit diff --git a/times-square.yaml b/times-square.yaml new file mode 100644 index 0000000..3e6e1d5 --- /dev/null +++ b/times-square.yaml @@ -0,0 +1,4 @@ +enabled: true +root: notebooks_tsqr +description: > + Times Square for project-wide Logging and Reporting. From 3e2e5f5ef852702493584254784ecf5203224e8d Mon Sep 17 00:00:00 2001 From: Steve Pothier Date: Tue, 27 Aug 2024 08:41:34 -0700 Subject: [PATCH 03/10] WIP --- notebooks_tsqr/logrep_proto_1.ipynb | 1133 +++++++++++++++++---------- 1 file changed, 725 insertions(+), 408 deletions(-) diff --git a/notebooks_tsqr/logrep_proto_1.ipynb b/notebooks_tsqr/logrep_proto_1.ipynb index 563c013..d9b2ff2 100644 --- a/notebooks_tsqr/logrep_proto_1.ipynb +++ b/notebooks_tsqr/logrep_proto_1.ipynb @@ -14,11 +14,24 @@ "record_limit = '999'" ] }, + { + "cell_type": "markdown", + "id": "09757010-6091-4769-b940-efd464c5eb68", + "metadata": {}, + "source": [ + "\n", + "## Imports and General Setup" + ] + }, { "cell_type": "code", "execution_count": 2, "id": "c2017a15-e054-4a4e-a522-28deff6d2a7a", - "metadata": {}, + "metadata": { + "jupyter": { + "source_hidden": true + } + }, "outputs": [], "source": [ "# Only use packages available in the Rubin Science Platform\n", @@ -35,12 +48,16 @@ "cell_type": "code", "execution_count": 3, "id": "2a310c90-7c5a-4cc2-a3b2-cc9d8291d245", - "metadata": {}, + "metadata": { + "jupyter": { + "source_hidden": true + } + }, "outputs": [ { "data": { "text/plain": [ - "'https://usdf-rsp-dev.slac.stanford.edu'" + "'https://usdf-rsp-dev.slac.stanford.edu/narrativelog'" ] }, "execution_count": 3, @@ -69,25 +86,31 @@ " tucson = 'https://tucson-teststand.lsst.codes',\n", ")\n", "server = envs[env]\n", - "server" + "service = f'{server}/{log}'\n", + "service" + ] + }, + { + "cell_type": "markdown", + "id": "e926c1a6-cd11-4cc1-9fb5-55a5f4d386c0", + "metadata": { + "jp-MarkdownHeadingCollapsed": true + }, + "source": [ + "\n", + "## Setup Source" ] }, { "cell_type": "code", "execution_count": 4, - "id": "59b7f58f-7a39-42dc-a57d-cc890f5d3375", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Attempt to get logs from url='https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=either&limit=999'\n", - "numr=999 numf=24\n" - ] + "id": "8fdfc281-17db-471a-873e-94e6b41c6f70", + "metadata": { + "jupyter": { + "source_hidden": true } - ], + }, + "outputs": [], "source": [ "recs = None\n", "ok = True\n", @@ -100,15 +123,45 @@ " limit=limit,\n", " )\n", "qstr = urlencode(qparams)\n", - "url = f'{server}/{log}/messages?{qstr}'\n", - "print(f'\\nAttempt to get logs from {url=}')\n", + "url = f'{service}/messages?{qstr}'\n", + "\n", "ignore_fields = set(['tags', 'urls', 'message_text', 'id', 'date_added', \n", " 'obs_id', 'day_obs', 'seq_num', 'parent_id', 'user_id',\n", " 'date_invalidated', 'date_begin', 'date_end',\n", " 'time_lost', # float\n", " #'systems','subsystems','cscs', # values are lists, special handling\n", - " ])\n", + " ])" + ] + }, + { + "cell_type": "markdown", + "id": "1695642e-4d47-4ae9-9311-c77a7312ad22", + "metadata": { + "jp-MarkdownHeadingCollapsed": true + }, + "source": [ + "\n", + "## Get Records" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "8481932e-6b4f-4eeb-98e2-18728c43351e", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Attempt to get logs from url='https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=either&limit=9999'\n", + "Retrieved 7673 records, each with numf=24 fields.\n" + ] + } + ], + "source": [ "try:\n", + " print(f'Attempt to get logs from {url=}')\n", " response = requests.get(url, timeout=timeout)\n", " response.raise_for_status()\n", " recs = response.json()\n", @@ -123,75 +176,183 @@ " print(f'ERROR getting {log} from {env=} using {url=}: {err=}')\n", "numf = len(flds) if ok else 0\n", "numr = len(recs) if ok else 0\n", - "print(f'{numr=} {numf=}')" + "print(f'Retrieved {numr} records, each with {numf=} fields.')" ] }, { - "cell_type": "code", - "execution_count": 5, - "id": "d7fedb0b-02fc-4843-8607-cf593ea3fb91", + "cell_type": "markdown", + "id": "6069eef2-0585-4e67-9527-42ec5dc12cbf", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'category',\n", - " 'components',\n", - " 'cscs',\n", - " 'date_added',\n", - " 'date_begin',\n", - " 'date_end',\n", - " 'date_invalidated',\n", - " 'id',\n", - " 'is_human',\n", - " 'is_valid',\n", - " 'level',\n", - " 'message_text',\n", - " 'parent_id',\n", - " 'primary_hardware_components',\n", - " 'primary_software_components',\n", - " 'site_id',\n", - " 'subsystems',\n", - " 'systems',\n", - " 'tags',\n", - " 'time_lost',\n", - " 'time_lost_type',\n", - " 'urls',\n", - " 'user_agent',\n", - " 'user_id'}" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], "source": [ - "flds" + "\n", + "## Tables of (mostly raw) results" + ] + }, + { + "cell_type": "markdown", + "id": "97d5d456-e2b4-4dd6-b0c7-2d95870d8481", + "metadata": {}, + "source": [ + "### Fields names provided in records from log." ] }, { "cell_type": "code", "execution_count": 6, - "id": "d0432b3b-4270-4702-b79d-ed7a4a2800e3", + "id": "d7fedb0b-02fc-4843-8607-cf593ea3fb91", "metadata": {}, "outputs": [ { "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Field Name
0tags
1primary_software_components
2time_lost
3time_lost_type
4date_invalidated
5id
6site_id
7date_begin
8components
9date_added
10subsystems
11is_human
12user_id
13category
14cscs
15primary_hardware_components
16parent_id
17systems
18date_end
19message_text
20level
21is_valid
22urls
23user_agent
\n", + "
" + ], "text/plain": [ - "{'level': {'0', '100'},\n", - " 'category': {'None', 'SCIENCE'},\n", - " 'subsystems': {'None'},\n", - " 'user_agent': {'LOVE'},\n", - " 'primary_software_components': {'None'},\n", - " 'is_valid': {'False', 'True'},\n", - " 'is_human': {'True'},\n", - " 'systems': {'None'},\n", - " 'primary_hardware_components': {'None'},\n", - " 'cscs': {'None'},\n", - " 'components': {'None'},\n", - " 'site_id': {'summit'},\n", - " 'time_lost_type': {'None', 'fault'}}" + " Field Name\n", + "0 tags\n", + "1 primary_software_components\n", + "2 time_lost\n", + "3 time_lost_type\n", + "4 date_invalidated\n", + "5 id\n", + "6 site_id\n", + "7 date_begin\n", + "8 components\n", + "9 date_added\n", + "10 subsystems\n", + "11 is_human\n", + "12 user_id\n", + "13 category\n", + "14 cscs\n", + "15 primary_hardware_components\n", + "16 parent_id\n", + "17 systems\n", + "18 date_end\n", + "19 message_text\n", + "20 level\n", + "21 is_valid\n", + "22 urls\n", + "23 user_agent" ] }, "execution_count": 6, @@ -200,42 +361,148 @@ } ], "source": [ - "facets" + "pd.DataFrame(flds, columns=['Field Name'])" + ] + }, + { + "cell_type": "markdown", + "id": "51c9a1db-4b38-4f35-bece-c4038ee5b9a5", + "metadata": {}, + "source": [ + "### Facets from log records.\n", + "A *facet* is the set all of values found for a field in the retrieved records. Facets are only calculated for some fields." ] }, { "cell_type": "code", "execution_count": 7, - "id": "0720e2cc-d68c-4830-92c1-74ae5180f057", + "id": "545abf79-808b-4716-b9e2-c5426a86b277", "metadata": {}, "outputs": [ { "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
012
is_humanTrueNoneNone
site_idsummitNoneNone
categorySCIENCENoneENG
primary_software_componentsNoneNoneNone
level1000None
is_validTrueFalseNone
cscsNoneNoneNone
componentsNoneNoneNone
primary_hardware_componentsNoneNoneNone
user_agentLOVEstringNone
subsystemsNoneNoneNone
systemsNoneNoneNone
time_lost_typefaultNoneweather
\n", + "
" + ], "text/plain": [ - "{'id': '0005d0e0-6de7-4857-b658-cfca7147374a',\n", - " 'site_id': 'summit',\n", - " 'message_text': 'The moveShutterMainDoor command Fail\\r\\n\\r\\n\\r\\nwe try the parameter with uppercase now\\r\\n\\r\\n\\r\\n open: False\\r\\n\\r\\n\\r\\n\\r\\n\\r\\nError in run\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 500, in wait_for\\r\\n return fut.result()\\r\\n ^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 239, in _get_next_ackcmd\\r\\n await self._next_ack_event.wait()\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/locks.py\", line 213, in wait\\r\\n await fut\\r\\nasyncio.exceptions.CancelledError\\r\\n\\r\\n\\r\\nThe above exception was the direct cause of the following exception:\\r\\n\\r\\n\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 189, in next_ackcmd\\r\\n ackcmd = await self._wait_task\\r\\n ^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 214, in _basic_next_ackcmd\\r\\n ackcmd = await asyncio.wait_for(\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 502, in wait_for\\r\\n raise exceptions.TimeoutError() from exc\\r\\nTimeoutError\\r\\n\\r\\n\\r\\nDuring handling of the above exception, another exception occurred:\\r\\n\\r\\n\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\\r\\n await self._run_task\\r\\n File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/run_command.py\", line 170, in run\\r\\n await getattr(self.remote, f\"cmd_{self.cmd}\").start(\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\\r\\n return await cmd_info.next_ackcmd(timeout=timeout)\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 205, in next_ackcmd\\r\\n raise base.AckTimeoutError(\\r\\nlsst.ts.salobj.base.AckTimeoutError: msg=\\'Timed out waiting for command acknowledgement\\', ackcmd=(ackcmd private_seqNum=1142844592, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result=\\'No command acknowledgement seen\\')\\r\\n',\n", - " 'level': 0,\n", - " 'tags': [],\n", - " 'urls': [],\n", - " 'time_lost': 0.0,\n", - " 'date_begin': '2024-03-20T04:15:26.769000',\n", - " 'user_id': 'cmorales@summit-lsp.lsst.codes',\n", - " 'user_agent': 'LOVE',\n", - " 'is_human': True,\n", - " 'is_valid': True,\n", - " 'date_added': '2024-03-20T04:16:06.976595',\n", - " 'date_invalidated': None,\n", - " 'parent_id': None,\n", - " 'systems': None,\n", - " 'subsystems': None,\n", - " 'cscs': None,\n", - " 'date_end': '2024-03-20T04:15:26.769000',\n", - " 'components': ['AuxTel'],\n", - " 'primary_software_components': ['None'],\n", - " 'primary_hardware_components': ['None'],\n", - " 'category': 'None',\n", - " 'time_lost_type': 'fault'}" + " 0 1 2\n", + "is_human True None None\n", + "site_id summit None None\n", + "category SCIENCE None ENG\n", + "primary_software_components None None None\n", + "level 100 0 None\n", + "is_valid True False None\n", + "cscs None None None\n", + "components None None None\n", + "primary_hardware_components None None None\n", + "user_agent LOVE string None\n", + "subsystems None None None\n", + "systems None None None\n", + "time_lost_type fault None weather" ] }, "execution_count": 7, @@ -244,15 +511,43 @@ } ], "source": [ - "recs[0]" + "pd.DataFrame.from_dict(facets, orient='index')" + ] + }, + { + "cell_type": "markdown", + "id": "21afc757-8890-4b96-afc9-e5086327df55", + "metadata": {}, + "source": [ + "### Table of selected log record fields.\n", + "Table can be retrieved as CSV file for local use." ] }, { "cell_type": "code", - "execution_count": 8, - "id": "343f9f9c-ffc2-4b06-b18c-3086fe17885f", + "execution_count": 19, + "id": "1c40de6e-91c3-4d9d-972c-efe15d2b4a17", "metadata": {}, "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Table available as CSV file: \n" + ] + }, + { + "data": { + "text/html": [ + "tl.csv
" + ], + "text/plain": [ + "/home/pothiers/sandbox/ts_logging_and_reporting/notebooks_tsqr/tl.csv" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, { "data": { "text/html": [ @@ -310,53 +605,53 @@ " ...\n", " \n", " \n", - " 994\n", - " 2023-11-16T02:57:08.886132\n", + " 7668\n", + " 2024-04-19T19:07:32.755734\n", " 0.0\n", " \n", " \n", - " 995\n", - " 2024-04-19T23:27:40.232791\n", + " 7669\n", + " 2024-03-21T01:04:28.883494\n", " 0.0\n", " \n", " \n", - " 996\n", - " 2024-01-05T20:16:07.172173\n", + " 7670\n", + " 2024-06-12T01:56:12.896718\n", " 0.0\n", " \n", " \n", - " 997\n", - " 2024-07-15T19:40:15.707753\n", + " 7671\n", + " 2024-03-04T03:20:00.151348\n", " 0.0\n", " \n", " \n", - " 998\n", - " 2024-02-21T16:11:05.772351\n", + " 7672\n", + " 2024-01-03T22:42:08.463006\n", " 0.0\n", " \n", " \n", "\n", - "

999 rows × 2 columns

\n", + "

7673 rows × 2 columns

\n", "" ], "text/plain": [ - " date_added time_lost\n", - "0 2024-03-20T04:16:06.976595 0.0\n", - "1 2024-04-24T20:54:19.346646 0.0\n", - "2 2024-06-19T01:59:58.701343 0.0\n", - "3 2024-08-21T04:22:26.232653 0.0\n", - "4 2023-11-21T19:55:36.787105 0.0\n", - ".. ... ...\n", - "994 2023-11-16T02:57:08.886132 0.0\n", - "995 2024-04-19T23:27:40.232791 0.0\n", - "996 2024-01-05T20:16:07.172173 0.0\n", - "997 2024-07-15T19:40:15.707753 0.0\n", - "998 2024-02-21T16:11:05.772351 0.0\n", + " date_added time_lost\n", + "0 2024-03-20T04:16:06.976595 0.0\n", + "1 2024-04-24T20:54:19.346646 0.0\n", + "2 2024-06-19T01:59:58.701343 0.0\n", + "3 2024-08-21T04:22:26.232653 0.0\n", + "4 2023-11-21T19:55:36.787105 0.0\n", + "... ... ...\n", + "7668 2024-04-19T19:07:32.755734 0.0\n", + "7669 2024-03-21T01:04:28.883494 0.0\n", + "7670 2024-06-12T01:56:12.896718 0.0\n", + "7671 2024-03-04T03:20:00.151348 0.0\n", + "7672 2024-01-03T22:42:08.463006 0.0\n", "\n", - "[999 rows x 2 columns]" + "[7673 rows x 2 columns]" ] }, - "execution_count": 8, + "execution_count": 19, "metadata": {}, "output_type": "execute_result" } @@ -364,187 +659,19 @@ "source": [ "cols = ['date_added', 'time_lost']\n", "df = pd.DataFrame(recs)[cols]\n", - "df" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "72f1a445-c5b9-4efa-a5fc-77e1c4cee84e", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "tl.csv
" - ], - "text/plain": [ - "/home/pothiers/sandbox/ts_notebooks/logging/tl.csv" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ + "\n", "# Allow download of CSV version of DataFrame\n", "csvfile = 'tl.csv'\n", "df.to_csv(csvfile)\n", "myfile = FileLink(csvfile)\n", - "display(myfile)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "a74d281f-952d-42d3-bf35-949a17000b49", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqYAAAGdCAYAAADALrFsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAeVklEQVR4nO3df3DdZb3g8U9K27SFJlBpG0qDFnHR+gPcihjwR3E6FIZ15coyOq5KlQHXbd0LdXToiDCg2FmGQdRbKd6FVkdYXPeuoFy2UisWdymgdbsI0l6rSGtr0vKjSVtofjRn/3B7khNSmtPk5Hlyzus1c2ZyTs735IG0ybvP833Ot65QKBQCAAASG5d6AAAAECFMAQDIhDAFACALwhQAgCwIUwAAsiBMAQDIgjAFACALwhQAgCyMTz2AWtHb2xs7d+6MqVOnRl1dXerhAABDUCgUYu/evTFr1qwYN858XqUJ01Gyc+fOaG5uTj0MAOAobN++PWbPnp16GFVPmI6SqVOnRsTf/mA3NDQkHg0AMBQdHR3R3Nxc/D1OZQnTUXJo+b6hoUGYAsAY4zS80eFkCQAAsiBMAQDIgjAFACALwhQAgCwIUwAAsiBMAQDIgjAFACALwhQAgCwIUwAAsiBMAQDIgjAFACALwhQAgCwIU6rGge6D8Y+P/Cm27tp7VMc//qcX4pdbdo3wqBgpu/YeiHuf2BYvd/WkHgoAFSJMqRrf+eUf46YHn4kFtz5S9rGFQiE++t3HYtGqX8fz+zorMDqG66N3PBbX/I/fxVcfeCb1UACoEGFK1fg/21466mMLhb6PX9zfNQKjYaQ9+/z+iIj4+TNtiUcCQKUIUwAAsiBMISIKR34KAFBhwhTib+eYAgBpCVMAALIgTCEs5QNADoQpROmufAAgDWEKAEAWhClERMFiPgAkJ0whLOUDQA6EKQAAWRCmAABkQZhCWMoHgBwIUwAAsiBMIezKB4AcCFMIS/kAkANhCuGSpACQA2EKAEAWhClERMFaPgAkJ0whLOUDQA6EKQAAWRCmEHblA0AOhClEWMsHgAwI04hYvnx5nHXWWTF16tSYMWNGXHzxxbFly5aS5xw4cCAWL14cr3vd6+K4446LSy65JNra2hKNGACg+gjTiFi/fn0sXrw4HnvssVi7dm10d3fH+eefH/v37y8+5+qrr46f/vSn8aMf/SjWr18fO3fujI985CMJR81IcuUnAEhvfOoB5GDNmjUl91evXh0zZsyIjRs3xvvf//5ob2+PO++8M+6555744Ac/GBERq1atire85S3x2GOPxXve854Uw2YEOccUANIzYzqI9vb2iIiYNm1aRERs3Lgxuru7Y8GCBcXnvPnNb45TTjklNmzYkGSMAADVxozpAL29vXHVVVfFueeeG29729siIqK1tTUmTpwYxx9/fMlzZ86cGa2trYO+TmdnZ3R2dhbvd3R0VGzMDJ8JUwBIz4zpAIsXL46nnnoq7r333mG9zvLly6OxsbF4a25uHqERUgmu/AQA6QnTfpYsWRIPPPBAPPzwwzF79uzi401NTdHV1RV79uwpeX5bW1s0NTUN+lrLli2L9vb24m379u2VHDoAwJgnTONvs2VLliyJH//4x/GLX/wi5syZU/L5efPmxYQJE2LdunXFx7Zs2RLbtm2LlpaWQV+zvr4+GhoaSm7ky3wpAKTnHNP42/L9PffcE/fff39MnTq1eN5oY2NjTJ48ORobG+Pyyy+PpUuXxrRp06KhoSE+//nPR0tLix35VaL/Sr5VfQBIQ5hGxO233x4REfPnzy95fNWqVbFo0aKIiPjGN74R48aNi0suuSQ6Oztj4cKF8Z3vfGeUR0qleB9TAEhPmMbQNr5MmjQpVqxYEStWrBiFEZGSSAWANJxjChElJ5laygeANIQphM1PAJADYQoDmDEFgDSEKcSAXfnmTwEgCWEKIUYBIAfCFAawlA8AaQhTCDEKADkQphB25QNADoQpDGD2FADSEKYQpVf/shEKANIQphAD3i5KlwJAEsIUAIAsCFMYwIQpAKQhTCEGLuVLUwBIQZhC2PAEADkQpjCARAWANIQphF35AJADYQphlhQAciBM4VVkKgCkIEwhBlz5SZcCQBLCFMIcKQDkQJjCACIVANIQphB25QNADoQpRIR5UgBIT5jCAC5JCgBpCFOIAUv56YYBADVNmEKIUQDIgTCFsPkJAHIgTGGAgvlTAEhCmEIMiFFdCgBJCFMIy/cAkANhCgNoVABIQ5hC2PwEADkQphA2PAFADoQpDCBSASANYQphKR8AciBMAQDIgjCFAUyYAkAawhRi4FK+NAWAFIQphA1PAJADYQoDSFQASEOYQgzYia9MASAJYQqhRQEgB8IUonTDk/NNASANYQoD2JQPAGkIUwhL+QCQA2EK4ZKkAJADYQoD6FIASEOYQkTIUQBIT5hCuCQpAORAmMIAshQA0hCmEKUxasIUANIQphBiFAByIEzhVVQqAKQgTCEGXJJUlwJAEsIUwhwpAORAmEIMeLuodMMAgJomTGEAS/kAkIYwhYgomCcFgOSEaUQ88sgj8aEPfShmzZoVdXV1cd9995V8vlAoxHXXXRcnnXRSTJ48ORYsWBB/+MMf0gyWyihZyhepAJCCMI2I/fv3xxlnnBErVqwY9PM333xzfOtb34qVK1fG448/Hscee2wsXLgwDhw4MMojZTRYygeANManHkAOLrzwwrjwwgsH/VyhUIjbbrstrr322vjwhz8cERHf//73Y+bMmXHffffFxz72sdEcKhWiRQEgPTOmR/Dss89Ga2trLFiwoPhYY2NjnH322bFhw4aEI2Mk2ZUPAOmZMT2C1tbWiIiYOXNmyeMzZ84sfm4wnZ2d0dnZWbzf0dFRmQEy4grW8gEgCTOmFbJ8+fJobGws3pqbm1MPiddgwxMApCdMj6CpqSkiItra2koeb2trK35uMMuWLYv29vbibfv27RUdJ8NjkhQA0hOmRzBnzpxoamqKdevWFR/r6OiIxx9/PFpaWg57XH19fTQ0NJTcGBtEKgCk4RzTiNi3b19s3bq1eP/ZZ5+NTZs2xbRp0+KUU06Jq666Kr72ta/Fm970ppgzZ0585StfiVmzZsXFF1+cbtCMKC0KAOkJ04j4zW9+E+edd17x/tKlSyMi4rLLLovVq1fHl770pdi/f39ceeWVsWfPnnjve98ba9asiUmTJqUaMiOs/4Yn55sCQBrCNCLmz5//mjux6+rq4sYbb4wbb7xxFEdFKpbyASAN55hSlcp9y6f+zxamAJCGMKUqlR2XYhQAkhOmVKXyu7T/OaYAQArCFAZw5ScASEOYUpXKPsdUiwJAcsKUqlT2Un6/AzQqAKQhTGEgZQoASQhTqlK5S/NaFADSE6ZUpXKv3uTKTwCQnjCFAWyEAoA0hClVyVI+AIw9whTCrnwAyIEwhQEs5QNAGsKUqlR+XKpRAEhNmFKVyt+Vf/THAgAjQ5jCAJbyASANYUpVsisfAMYeYUpVKjc07coHgPSEKVWpUOaUacl5pdbyASAJYQoAQBaEKVXJUj4AjD3ClKo0nM1PVvIBIA1hCgOUe34qADAyhCnVqdwZUzEKAMkJU6rScK7eJFEBIA1hCgOYPAWANIQpVanszU9iFACSE6ZUpbLfLqrfERoV4PCck08lCVMYwA9dgMHt3tsZZ399XSz/n8+kHgpVSphSlcq+JKkWBTii7z7yx9i1tzPuWP+n1EOhSglTqtJwrvwEwOD8rKTShClVo+Syoq78BABjjjAFACALwpSqUbqzvtxzTI/+WCrPhjSA2iBMqRol7WIpv6r4ngDUBmEKAEAWhClVo2TzU9kHD+NYKs73BKA2CFOqRsk5pmUv5R/9sVSec0wBaoMwBQAgC8KUqlG6lH/0V36yKz8/viMAtUGYUjWGs7Pervy8+Z4A1AZhCgBAFoQp1WMYO+vNyOXN6RUAtUGYUjVKd9aXeY7pMI6l8nxLOJLe3kJc809Pxt2PP5d6KDXDz0oqQZjCAH7WwtizbvOuuPfX2+PLP34q9VBqxsFePywZecKUqlGys77czU9+vsKY1v5Kd+oh1JyDfnBSAcKUqjGcH5GFw3xMHvz+g/z09qYeAdVImFI1+p/vVHbIDOdYKs7mJ8hPjzKlAoQpAFXFppzRoUupBGFK1Shdji93V/7RH0vl6QzK0WNTzqhwjimVIEypGiO1+cnP2vz4llCO7oOm8iqlf/NbyqcShCkAVaW7xz9lKqW337/cdSmVIEypGsPZWV+ycWpERsNIcs4gR9Lbbyqvy4xpxfR/71JL+VSCMKV6lOysP/pzTK3l58d35PCe/Mue+I93b4znXtifeihJdfebvrOUXzn9z989eNDfTEbe+NQDAODo/dt/+N8REfHcCy/HP/+n9yUeTTrdPcJ0NPSaMaXCzJhSNYa3lH/0x1J5fv8d2R927Us9hKJ/fORPcevafxnVr9ndb/ZOmFZO/xh1SVIqwYwpVWNYu/IP8zpkwvdkzOg+2Bs3PfhMRERcOm92NE+bMipft/95pV02P1VMyTmmwpQKMGMKwIjp6HfN+v1dPaP2dfvPktr8VDnClEoTplSNwjAW80t35Q9+7Lpn2uLWh7aUnGPF6HDRg7Gj40BfjO47kCZMj7SU710ejp4wpdKEaZlWrFgRb3jDG2LSpElx9tlnxxNPPJF6SPx/I/Um+Yc79vLv/Sa+9YutsfaZtqN/cY6Kjhg7+s+Ytvf7uNJKzjHtOXyYvrCvM1qW/yKuv/+p0RhW1fF2UVSaMC3DD3/4w1i6dGlcf/318dvf/jbOOOOMWLhwYezatSv10Kiwnn4zMNtffDnhSIbu/k074v5NO1IPgxrTcaAvRve8PHph2tUztKX8e3+9PVo7DsT3Njw3GsOqOjY/UWnCtAy33nprXHHFFfHpT3865s6dGytXrowpU6bEXXfdlXpoxPB21h/p2Of3dRU/7n6N9+5b89Rf44afPh2dPQfLHEF52l/pjvs37TjskuXz+zrj7+/dFH9/76bYvbezomMZDbn/+st9F/iTf9kTG597aURf80D3wUH/kdbxSt/y/ejOmPZfyj/8n5jO7r6/m/s7X32qQVdPb1y++tdxzT89ObIDrBKW8qk0u/KHqKurKzZu3BjLli0rPjZu3LhYsGBBbNiwIdm4tr/4ctzzxLZkXz8nu/YeKH5856+ejWnHTRzysZu27Sl+vOGPL8R/XrO55PPP94u7nz3dWjIr1N/tv/xjRET8+fn98eaTGob0tfe83BWTJ4yP+glD/3fif/nVn6L7YCGWP7g5/u5fn/yqz+/c80rx46/98+9j1vGTh/zaQ9FbKMT/3b4n6scfE285qSHq6kb05V/lla6+mNh3oOdV35+Ufv3si/H0zo44783T4/WvOzbZOLp6egf9//L83s740ca/RETEh8+cNeifhUIh4qX9XXH8lAkxbtyRv5mFQsR/fWJbtL/SHQveMiPeOP242PNyd5xw7MTY0tpRfN6d/+vZ2L1vdP5h1D+8//vG7fHbbYOH+KN/fKH48U0PPhONkyeUfL6t40Cs2/y3VbApE8v7e1kLtvZ7W7K7H38uHt6S14rh7BMmx78/+/Wph8Ew1BWcBT4kO3fujJNPPjkeffTRaGlpKT7+pS99KdavXx+PP/54yfM7Ozujs7PvB3JHR0c0NzdHe3t7NDQMLViG4jd/fjH+3cp0YQwAuTjrDSfEj/7DOSP6mh0dHdHY2Djiv78ZnBnTClm+fHnccMMNFf86MxsmxeXvnVPxrzNW7NzzSpzUOPmoZvD2d/ZET2/hVTMoh3T2HIw9L3fHzIZJh32N7oO98df2A3HKEN+7cf2/7C7OQMw/fXq8cfpxQzquUIj4w6698a9mTj3scw6d33f8lMH/e4Zrx0uvxMTx42L61PqKvP5gdu55ZcRnf4frYG8hnt/X+Zp/Liptx0uvxMknHP7/y0v7u6Knt/Ca36u6KO+Uie6DvfHcCy/HaTOOe9Xx9ePHxXGTxscL/U6BGQ0v7OuMY+vHx6QJx7zm81rbD8TrjpsYE44ZfDZ0997OmDRhXEydVJm/O2NdW8eBeN2xE2P8Yf7/pdT8Gn8PGBuE6RCdeOKJccwxx0RbW+mO7La2tmhqanrV85ctWxZLly4t3j80YzrSmqdNia/8m7kj/rqMjpf+26ZimP7dO0+OD5/56mV5AKgV+f1zJ1MTJ06MefPmxbp164qP9fb2xrp160qW9g+pr6+PhoaGkhsAAIdnxrQMS5cujcsuuyze9a53xbvf/e647bbbYv/+/fHpT3869dAYo+qi75yDukrvIAKAzAnTMnz0ox+N3bt3x3XXXRetra1x5plnxpo1a2LmzJmph8YY1b9FZSkAtU6YlmnJkiWxZMmS1MOgSvSPUROmANQ655gCAJAFYQoJlS7lmzIFoLYJU0iodPNTwoEAQAaEKQAAWRCmkJBd+QDQR5hCQiVhqkwBqHHCFACALAhTSKruMB8DQO0RppCQpXwA6CNMAQDIgjCFhCzkA0AfYQoJlS7lS1MAapswhYRKrvyUcBwAkANhCgBAFoQpJGRXPgD0EaaQUMnmJ2EKQI0TpgAAZEGYQkL9d+LX2f4EQI0TppALXQpAjROmAABkQZhCQiW78tMNAwCyIEwhoZI32LctH4AaJ0wBAMiCMIWELOUDQB9hCgl5g30A6CNMAQDIgjCFhEqX8k2ZAlDbhCkkVHLlJ10KQI0TppBQ3WE+BoBaJEwBAMiCMIWUTJkCQJEwhYRKrvykTAGoccIUAIAsCFNIqOTtokyYAlDjhCkk5BRTAOgjTAEAyIIwhYRKl/LNmQJQ24QpJFSyK1+XAlDjhCkAAFkQppBQyVJ+umEAQBaEKSRUsitfmQJQ44QpAABZEKaQUp13MgWAQ4QpJGQpHwD6CFNIyOYnAOgjTAEAyIIwhYRK32DfnCkAtU2YQkKW8gGgjzAFACALwhQSsisfAPoIU0iodClfmQJQ24QpAABZEKaQUP+d+JbyAah1whQAgCwIUwAAsiBMIaGSzU+W8gGoccIUEiq58pNd+QDUOGEKCZkxBYA+whQAgCwI04i46aab4pxzzokpU6bE8ccfP+hztm3bFhdddFFMmTIlZsyYEV/84hejp6dndAdK1XHlJwDoMz71AHLQ1dUVl156abS0tMSdd975qs8fPHgwLrroomhqaopHH300/vrXv8anPvWpmDBhQnz9619PMGKqhSs/AUAfM6YRccMNN8TVV18db3/72wf9/EMPPRS///3v4wc/+EGceeaZceGFF8ZXv/rVWLFiRXR1dY3yaAEAqpMwHYINGzbE29/+9pg5c2bxsYULF0ZHR0c8/fTTgx7T2dkZHR0dJTcYqGRXvglTAGqcMB2C1tbWkiiNiOL91tbWQY9Zvnx5NDY2Fm/Nzc0VHydjT+lSPgDUtqoN02uuuSbq6upe87Z58+aKff1ly5ZFe3t78bZ9+/aKfS0AgGpQtZufvvCFL8SiRYte8zmnnnrqkF6rqakpnnjiiZLH2traip8bTH19fdTX1w/p9SHCUj4AVG2YTp8+PaZPnz4ir9XS0hI33XRT7Nq1K2bMmBEREWvXro2GhoaYO3fuiHwNalNdSY0qUwBqW9WGaTm2bdsWL774Ymzbti0OHjwYmzZtioiI0047LY477rg4//zzY+7cufHJT34ybr755mhtbY1rr702Fi9ebFYUAGCECNOIuO666+J73/te8f473/nOiIh4+OGHY/78+XHMMcfEAw88EJ/73OeipaUljj322LjsssvixhtvTDVkqoQ32AeAPsI0IlavXh2rV69+zee8/vWvjwcffHB0BkTNsCsfAPpU7a58AADGFmEKCZUu5ZszBaC2CVNIqH+MylIAap0whYRKzjFVpgDUOGEKAEAWhCkkVPr2+qZMAahtwhRS6n+OqS4FoMYJUwAAsiBMISGTpADQR5hCQnblA0AfYQoAQBaEKSTUfye+Kz8BUOuEKSRUspSfbhgAkAVhCgBAFoQpJFTyBvumTAGoccIUEipdylemANQ2YQoAQBaEKSRUuis/4UAAIAPCFFKyKx8AioQpJFR32DsAUHuEKQAAWRCmkFD/qz3ZlQ9ArROmkJD3MQWAPsIUAIAsCFNIqM6ufAAoEqaQiTpr+QDUOGEKAEAWhClkwnwpALVOmEImrOQDUOuEKQAAWRCmkAlvsA9ArROmkAtdCkCNE6aQCeeYAlDrhCkAAFkQppAJE6YA1DphCplw5ScAap0wBQAgC8IUMmG+FIBaJ0whE1byAah1whQAgCwIU8iEKz8BUOuEKWTCUj4AtU6YAgCQBWEKAEAWhClkwlI+ALVOmAIAkAVhCpmwKx+AWidMIROW8gGodcIUMqFLAah1whQAgCwIU8hEnbV8AGqcMIVMyFIAap0wBQAgC8IUMmElH4BaJ0whE84xBaDWCVMAALIgTAEAyIIwBQAgC8IUAIAs1HyY/vnPf47LL7885syZE5MnT443vvGNcf3110dXV1fJ85588sl43/veF5MmTYrm5ua4+eabE40YAKA6jU89gNQ2b94cvb29cccdd8Rpp50WTz31VFxxxRWxf//+uOWWWyIioqOjI84///xYsGBBrFy5Mn73u9/FZz7zmTj++OPjyiuvTPxfAABQHWo+TC+44IK44IILivdPPfXU2LJlS9x+++3FML377rujq6sr7rrrrpg4cWK89a1vjU2bNsWtt94qTAEARkjNL+UPpr29PaZNm1a8v2HDhnj/+98fEydOLD62cOHC2LJlS7z00kuDvkZnZ2d0dHSU3AAAODxhOsDWrVvj29/+dnz2s58tPtba2hozZ84sed6h+62trYO+zvLly6OxsbF4a25urtygAQCqQNWG6TXXXBN1dXWvedu8eXPJMTt27IgLLrggLr300rjiiiuG9fWXLVsW7e3txdv27duH9XoAANWuas8x/cIXvhCLFi16zeeceuqpxY937twZ5513Xpxzzjnx3e9+t+R5TU1N0dbWVvLYoftNTU2DvnZ9fX3U19cfxcgBAGpT1Ybp9OnTY/r06UN67o4dO+K8886LefPmxapVq2LcuNKJ5JaWlvjyl78c3d3dMWHChIiIWLt2bZx++ulxwgknjPjYAQBqUdUu5Q/Vjh07Yv78+XHKKafELbfcErt3747W1taSc0c//vGPx8SJE+Pyyy+Pp59+On74wx/GN7/5zVi6dGnCkQMAVJeqnTEdqrVr18bWrVtj69atMXv27JLPFQqFiIhobGyMhx56KBYvXhzz5s2LE088Ma677jpvFQUAMIJqPkwXLVp0xHNRIyLe8Y53xK9+9avKDwgAoEbV/FI+AAB5EKYAAGRBmAIAkAVhCgBAFoQpAABZEKYAAGRBmAIAkAVhCgBAFoQpAABZEKYAAGRBmAIAkAVhCgBAFoQpAABZEKYAAGRBmAIAkAVhCgBAFoQpAABZEKYAAGRBmAIAkAVhCgBAFoQpAABZEKaQ0JwTj009BADIxvjUA4Ba9s5TTohvfuzMOGXalNRDAYDkhCkk9uEzT049BADIgqV8AACyIEwBAMiCMAUAIAvCFACALAhTAACyIEwBAMiCMAUAIAvCFACALAhTAACyIEwBAMiCMAUAIAvCFACALAhTAACyMD71AGpFoVCIiIiOjo7EIwEAhurQ7+1Dv8epLGE6Svbu3RsREc3NzYlHAgCUa+/evdHY2Jh6GFWvruCfAKOit7c3du7cGVOnTo26uroRfe0dO3bE3LlzR/Q1AWAs2r59ezQ0NIzY6xUKhdi7d2/MmjUrxo1zBmSlmTEdJePGjYvZs2dX5LWdHgAAf9PQ0DCiYRoRZkpHkfQHACALwhQAgCxYyq8CDQ0Nce6550ZPT0/xXNZD58IMvB8RQ3rsaI8bydcaK8flMIaxclwOYxgrx+UwhrFyXA5jGCvH5TCGSh03fvz4+MAHPhD19fXB2GXzEwAAWbCUDwBAFoQpAABZEKYAAGRBmAIAkIWyduUvX748brjhhujs7KzUeAAAqAIXXHBB3HnnnTFr1qwhH1PWjOn69eujp6cnjjnmmLIHBwBA7fj5z38el1xySVnHDOvtonbv3h0zZsw42sMBAKhidXV10dnZGRMmTBjS84f1Bvvt7e3DORwAgCo1YcKEOOuss4YcpRHD2PzU29sbn//854/2cAAAqthpp50WP/nJT8o65qjDdPHixfGzn/3saA8HAKCKPffcc/GJT3wiyjlr9KjCdMmSJXHHHXeU9YUAAKgdBw8ejDVr1sRjjz025GPKCtNCoRCf+9znYsWKFaIUAIBXOXRO6aFWLOdtRssK08WLF8fKlSvLOQQAgBrS3d0dERE9PT1x0kknxemnnz7kY8t6u6i6urryRwcAQM1atWpVLFq0aEjPLevtoizfAwBQKUe9Kx8AAEaSMAUAIAvCFACALAhTAACyIEwBAMiCMAUAIAvCFACALAhTAACyIEwBAMiCMAUAIAvCFACALAhTAACy8P8AoPdysIIaqGcAAAAASUVORK5CYII=", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "x = [r['date_added'] for r in recs]\n", - "y = [r['time_lost'] for r in recs]\n", - "plt.plot(x, y) \n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "138e5496-e058-417a-881e-8450e20af0f9", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "0cb5feca-620b-49a0-b25b-835a6d38843a", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[{'id': '0005d0e0-6de7-4857-b658-cfca7147374a',\n", - " 'site_id': 'summit',\n", - " 'message_text': 'The moveShutterMainDoor command Fail\\r\\n\\r\\n\\r\\nwe try the parameter with uppercase now\\r\\n\\r\\n\\r\\n open: False\\r\\n\\r\\n\\r\\n\\r\\n\\r\\nError in run\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 500, in wait_for\\r\\n return fut.result()\\r\\n ^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 239, in _get_next_ackcmd\\r\\n await self._next_ack_event.wait()\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/locks.py\", line 213, in wait\\r\\n await fut\\r\\nasyncio.exceptions.CancelledError\\r\\n\\r\\n\\r\\nThe above exception was the direct cause of the following exception:\\r\\n\\r\\n\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 189, in next_ackcmd\\r\\n ackcmd = await self._wait_task\\r\\n ^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 214, in _basic_next_ackcmd\\r\\n ackcmd = await asyncio.wait_for(\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 502, in wait_for\\r\\n raise exceptions.TimeoutError() from exc\\r\\nTimeoutError\\r\\n\\r\\n\\r\\nDuring handling of the above exception, another exception occurred:\\r\\n\\r\\n\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\\r\\n await self._run_task\\r\\n File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/run_command.py\", line 170, in run\\r\\n await getattr(self.remote, f\"cmd_{self.cmd}\").start(\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\\r\\n return await cmd_info.next_ackcmd(timeout=timeout)\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 205, in next_ackcmd\\r\\n raise base.AckTimeoutError(\\r\\nlsst.ts.salobj.base.AckTimeoutError: msg=\\'Timed out waiting for command acknowledgement\\', ackcmd=(ackcmd private_seqNum=1142844592, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result=\\'No command acknowledgement seen\\')\\r\\n',\n", - " 'level': 0,\n", - " 'tags': [],\n", - " 'urls': [],\n", - " 'time_lost': 0.0,\n", - " 'date_begin': '2024-03-20T04:15:26.769000',\n", - " 'user_id': 'cmorales@summit-lsp.lsst.codes',\n", - " 'user_agent': 'LOVE',\n", - " 'is_human': True,\n", - " 'is_valid': True,\n", - " 'date_added': '2024-03-20T04:16:06.976595',\n", - " 'date_invalidated': None,\n", - " 'parent_id': None,\n", - " 'systems': None,\n", - " 'subsystems': None,\n", - " 'cscs': None,\n", - " 'date_end': '2024-03-20T04:15:26.769000',\n", - " 'components': ['AuxTel'],\n", - " 'primary_software_components': ['None'],\n", - " 'primary_hardware_components': ['None'],\n", - " 'category': 'None',\n", - " 'time_lost_type': 'fault'},\n", - " {'id': '00067b6a-1741-41f3-87af-57261dff30ac',\n", - " 'site_id': 'summit',\n", - " 'message_text': 'David updated environment. But no need to cycle Scheduler 1. Waiting for completion of the current block. \\r\\n',\n", - " 'level': 0,\n", - " 'tags': [],\n", - " 'urls': [],\n", - " 'time_lost': 0.0,\n", - " 'date_begin': '2024-04-24T20:53:41.843000',\n", - " 'user_id': 'ykang@summit-lsp.lsst.codes',\n", - " 'user_agent': 'LOVE',\n", - " 'is_human': True,\n", - " 'is_valid': True,\n", - " 'date_added': '2024-04-24T20:54:19.346646',\n", - " 'date_invalidated': None,\n", - " 'parent_id': None,\n", - " 'systems': None,\n", - " 'subsystems': None,\n", - " 'cscs': None,\n", - " 'date_end': '2024-04-24T20:53:41.843000',\n", - " 'components': ['MainTel'],\n", - " 'primary_software_components': ['None'],\n", - " 'primary_hardware_components': ['None'],\n", - " 'category': 'None',\n", - " 'time_lost_type': 'fault'},\n", - " {'id': '000a3ba8-b1a9-4b04-9c4a-263da3dac1d1',\n", - " 'site_id': 'summit',\n", - " 'message_text': 'Runing BLOCK-T22 AuxTel filter Scans, the auxtel/calibrations/run_calibration_sequence.py went to fault.\\r\\n\\r\\n\\r\\nsalIndex: 200068\\r\\nFile \"/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/auxtel/atcalsys.py\", line 338, in _take_data\\r\\n await asyncio.gather(\\r\\n File \"/net/obs-env/auto_base_packages/ts_observatory_control/python/lsst/ts/observatory/control/auxtel/atcalsys.py\", line 430, in take_fiber_spectrum\\r\\n await self.fiberspectrograph.cmd_expose.set_start(\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 416, in set_start\\r\\n return await self.start(timeout=timeout, wait_done=wait_done)\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\\r\\n return await cmd_info.next_ackcmd(timeout=timeout)\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 191, in next_ackcmd\\r\\n raise base.AckError(msg=\"Command failed\", ackcmd=ackcmd)\\r\\nlsst.ts.salobj.base.AckError: msg=\\'Command failed\\', ackcmd=(ackcmd private_seqNum=1157551610, ack=<SalRetCode.CMD_FAILED: -302>, error=1, result=\"Failed: 502, message=\\'Bad Gateway\\', url=URL(\\'http://comcam-mcm-dds.cp.lsst.org/ImageUtilities/rest/imageNameService?n=1&sourceIndex=3&source=FiberSpectrograph\\')\")\\r\\n',\n", - " 'level': 0,\n", - " 'tags': [],\n", - " 'urls': [],\n", - " 'time_lost': 0.0,\n", - " 'date_begin': '2024-06-19T01:59:08.706000',\n", - " 'user_id': 'cmd_user@summit-lsp.lsst.codes',\n", - " 'user_agent': 'LOVE',\n", - " 'is_human': True,\n", - " 'is_valid': False,\n", - " 'date_added': '2024-06-19T01:59:58.701343',\n", - " 'date_invalidated': '2024-06-19T02:12:01.302498',\n", - " 'parent_id': 'af0157e3-d7f0-4068-9a8a-e51a0a77d260',\n", - " 'systems': None,\n", - " 'subsystems': None,\n", - " 'cscs': None,\n", - " 'date_end': '2024-06-19T01:59:08.706000',\n", - " 'components': ['AuxTel'],\n", - " 'primary_software_components': ['None'],\n", - " 'primary_hardware_components': ['None'],\n", - " 'category': 'None',\n", - " 'time_lost_type': 'fault'},\n", - " {'id': '0010e246-8039-4741-b4c6-54047e9464c5',\n", - " 'site_id': 'summit',\n", - " 'message_text': 'Still checking updates in the wavefront estimation pipeline version, offsetting manually the ATHexapod in different axes and deploying different versions of the pipeline.\\r\\n',\n", - " 'level': 0,\n", - " 'tags': [],\n", - " 'urls': [],\n", - " 'time_lost': 0.0,\n", - " 'date_begin': '2024-08-21T04:04:38.053000',\n", - " 'user_id': 'kpena@summit-lsp.lsst.codes',\n", - " 'user_agent': 'LOVE',\n", - " 'is_human': True,\n", - " 'is_valid': True,\n", - " 'date_added': '2024-08-21T04:22:26.232653',\n", - " 'date_invalidated': None,\n", - " 'parent_id': '99de2a12-20eb-45ae-b189-a793b63e6ab5',\n", - " 'systems': None,\n", - " 'subsystems': None,\n", - " 'cscs': None,\n", - " 'date_end': '2024-08-21T04:04:38.053000',\n", - " 'components': ['AuxTel'],\n", - " 'primary_software_components': ['None'],\n", - " 'primary_hardware_components': ['None'],\n", - " 'category': 'None',\n", - " 'time_lost_type': 'fault'}]" - ] - }, - "execution_count": 11, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "recs[:4]" + "print('Table available as CSV file: ')\n", + "display(myfile)\n", + "df" ] }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "id": "f6441d81-93a8-4369-93fb-55d3715783cc", "metadata": {}, "outputs": [ @@ -738,23 +865,23 @@ " ...\n", " \n", " \n", - " 994\n", - " 22325b7d-7948-4da8-88ec-2a47ceb088e0\n", + " 7668\n", + " ffbd086c-cf20-4c45-aef5-b2783d348192\n", " summit\n", - " Block 109 starting again\\r\\n\n", + " Script Failed salIndex: 101647 with move_rotat...\n", " 0\n", " []\n", " []\n", " 0.0\n", - " 2023-11-16T02:56:30.009000\n", - " echristensen@love01.cp.lsst.org\n", + " 2024-04-19T19:06:54.594000\n", + " ykang@summit-lsp.lsst.codes\n", " LOVE\n", " ...\n", " None\n", " None\n", " None\n", " None\n", - " 2023-11-16T02:56:30.009000\n", + " 2024-04-19T19:06:54.594000\n", " [MainTel]\n", " [None]\n", " [None]\n", @@ -762,23 +889,23 @@ " fault\n", " \n", " \n", - " 995\n", - " 2233da67-c777-4dd6-bc66-a241f93df5ae\n", + " 7669\n", + " ffeaada0-3b08-4de2-b48b-9acf2623fb7e\n", " summit\n", - " Az encoder checking thru slew test. \\r\\n\\r\\n\\r...\n", + " salIndex:&nbsp;101588\\r\\n \\r\\nError in run\\r\\n...\n", " 0\n", " []\n", " []\n", " 0.0\n", - " 2024-04-19T23:27:02.011000\n", - " ykang@summit-lsp.lsst.codes\n", + " 2024-03-21T01:03:51.861000\n", + " pvenegas@love01.cp.lsst.org\n", " LOVE\n", " ...\n", " None\n", " None\n", " None\n", " None\n", - " 2024-04-19T23:27:02.011000\n", + " 2024-03-21T01:03:51.861000\n", " [MainTel]\n", " [None]\n", " [None]\n", @@ -786,71 +913,71 @@ " fault\n", " \n", " \n", - " 996\n", - " 223c0310-03d5-450d-bf5c-0a04bdea6536\n", + " 7670\n", + " ffebf6c9-156c-4309-9662-f5697907cc47\n", " summit\n", - " Restarting [BLOCK-178|https://jira.lsstcorp.or...\n", + " h3. Execution of power_on_atcalsys.py \\r\\n\n", " 0\n", " []\n", " []\n", " 0.0\n", - " 2024-01-05T20:15:27.453000\n", - " kfanning@love01.cp.lsst.org\n", + " 2024-06-12T01:55:35.836000\n", + " pvenegas@summit-lsp.lsst.codes\n", " LOVE\n", " ...\n", " None\n", " None\n", " None\n", " None\n", - " 2024-01-05T20:15:27.453000\n", - " [MainTel]\n", + " 2024-06-12T01:55:35.836000\n", + " [AuxTel]\n", " [None]\n", " [None]\n", " None\n", " fault\n", " \n", " \n", - " 997\n", - " 2240e19e-7204-4c50-a3ba-c643d60aa0d2\n", + " 7671\n", + " fff11764-aae1-4428-aea0-b72fdb3dafa6\n", " summit\n", - " LATISS enabled but checkout failed \\r\\n\\r\\n\\r\\...\n", + " Updated pointing model based on March 2 data. ...\n", " 0\n", " []\n", " []\n", " 0.0\n", - " 2024-07-15T19:39:38.630000\n", - " kkelkar@summit-lsp.lsst.codes\n", + " 2024-03-04T03:19:22.793000\n", + " bstalder@summit-lsp.lsst.codes\n", " LOVE\n", " ...\n", " None\n", " None\n", " None\n", " None\n", - " 2024-07-15T19:39:38.630000\n", - " [AuxTel]\n", + " 2024-03-04T03:19:22.793000\n", + " [MainTel]\n", " [None]\n", " [None]\n", " None\n", " fault\n", " \n", " \n", - " 998\n", - " 22475aa1-f15a-46f1-87b8-b7a1708daaf4\n", + " 7672\n", + " fff14a30-c341-4561-a2ea-0f043d19f526\n", " summit\n", - " Powercycled M2 cabinet to bring back the M2 co...\n", + " M1M3 faulted HP 2 excessive forces\\r\\n\n", " 0\n", " []\n", " []\n", " 0.0\n", - " 2024-02-21T15:37:00\n", - " kfanning@summit-lsp.lsst.codes\n", + " 2024-01-03T19:45:18.279000\n", + " bstalder@love01.cp.lsst.org\n", " LOVE\n", " ...\n", " None\n", " None\n", " None\n", " None\n", - " 2024-02-21T16:10:28.176000\n", + " 2024-01-03T19:45:18.279000\n", " [MainTel]\n", " [None]\n", " [None]\n", @@ -859,92 +986,105 @@ " \n", " \n", "\n", - "

999 rows × 24 columns

\n", + "

7673 rows × 24 columns

\n", "" ], "text/plain": [ - " id site_id \\\n", - "0 0005d0e0-6de7-4857-b658-cfca7147374a summit \n", - "1 00067b6a-1741-41f3-87af-57261dff30ac summit \n", - "2 000a3ba8-b1a9-4b04-9c4a-263da3dac1d1 summit \n", - "3 0010e246-8039-4741-b4c6-54047e9464c5 summit \n", - "4 0017b90e-26aa-4a7c-8186-fc06ac9b5aa6 summit \n", - ".. ... ... \n", - "994 22325b7d-7948-4da8-88ec-2a47ceb088e0 summit \n", - "995 2233da67-c777-4dd6-bc66-a241f93df5ae summit \n", - "996 223c0310-03d5-450d-bf5c-0a04bdea6536 summit \n", - "997 2240e19e-7204-4c50-a3ba-c643d60aa0d2 summit \n", - "998 22475aa1-f15a-46f1-87b8-b7a1708daaf4 summit \n", + " id site_id \\\n", + "0 0005d0e0-6de7-4857-b658-cfca7147374a summit \n", + "1 00067b6a-1741-41f3-87af-57261dff30ac summit \n", + "2 000a3ba8-b1a9-4b04-9c4a-263da3dac1d1 summit \n", + "3 0010e246-8039-4741-b4c6-54047e9464c5 summit \n", + "4 0017b90e-26aa-4a7c-8186-fc06ac9b5aa6 summit \n", + "... ... ... \n", + "7668 ffbd086c-cf20-4c45-aef5-b2783d348192 summit \n", + "7669 ffeaada0-3b08-4de2-b48b-9acf2623fb7e summit \n", + "7670 ffebf6c9-156c-4309-9662-f5697907cc47 summit \n", + "7671 fff11764-aae1-4428-aea0-b72fdb3dafa6 summit \n", + "7672 fff14a30-c341-4561-a2ea-0f043d19f526 summit \n", + "\n", + " message_text level tags urls \\\n", + "0 The moveShutterMainDoor command Fail\\r\\n\\r\\n\\r... 0 [] [] \n", + "1 David updated environment. But no need to cycl... 0 [] [] \n", + "2 Runing BLOCK-T22 AuxTel filter Scans, the auxt... 0 [] [] \n", + "3 Still checking updates in the wavefront estima... 0 [] [] \n", + "4 The m1m3 ump test failed for actuator 108. \\r\\... 0 [] [] \n", + "... ... ... ... ... \n", + "7668 Script Failed salIndex: 101647 with move_rotat... 0 [] [] \n", + "7669 salIndex: 101588\\r\\n \\r\\nError in run\\r\\n... 0 [] [] \n", + "7670 h3. Execution of power_on_atcalsys.py \\r\\n 0 [] [] \n", + "7671 Updated pointing model based on March 2 data. ... 0 [] [] \n", + "7672 M1M3 faulted HP 2 excessive forces\\r\\n 0 [] [] \n", "\n", - " message_text level tags urls \\\n", - "0 The moveShutterMainDoor command Fail\\r\\n\\r\\n\\r... 0 [] [] \n", - "1 David updated environment. But no need to cycl... 0 [] [] \n", - "2 Runing BLOCK-T22 AuxTel filter Scans, the auxt... 0 [] [] \n", - "3 Still checking updates in the wavefront estima... 0 [] [] \n", - "4 The m1m3 ump test failed for actuator 108. \\r\\... 0 [] [] \n", - ".. ... ... ... ... \n", - "994 Block 109 starting again\\r\\n 0 [] [] \n", - "995 Az encoder checking thru slew test. \\r\\n\\r\\n\\r... 0 [] [] \n", - "996 Restarting [BLOCK-178|https://jira.lsstcorp.or... 0 [] [] \n", - "997 LATISS enabled but checkout failed \\r\\n\\r\\n\\r\\... 0 [] [] \n", - "998 Powercycled M2 cabinet to bring back the M2 co... 0 [] [] \n", + " time_lost date_begin user_id \\\n", + "0 0.0 2024-03-20T04:15:26.769000 cmorales@summit-lsp.lsst.codes \n", + "1 0.0 2024-04-24T20:53:41.843000 ykang@summit-lsp.lsst.codes \n", + "2 0.0 2024-06-19T01:59:08.706000 cmd_user@summit-lsp.lsst.codes \n", + "3 0.0 2024-08-21T04:04:38.053000 kpena@summit-lsp.lsst.codes \n", + "4 0.0 2023-11-21T19:51:54.868000 bquint@love01.cp.lsst.org \n", + "... ... ... ... \n", + "7668 0.0 2024-04-19T19:06:54.594000 ykang@summit-lsp.lsst.codes \n", + "7669 0.0 2024-03-21T01:03:51.861000 pvenegas@love01.cp.lsst.org \n", + "7670 0.0 2024-06-12T01:55:35.836000 pvenegas@summit-lsp.lsst.codes \n", + "7671 0.0 2024-03-04T03:19:22.793000 bstalder@summit-lsp.lsst.codes \n", + "7672 0.0 2024-01-03T19:45:18.279000 bstalder@love01.cp.lsst.org \n", "\n", - " time_lost date_begin user_id \\\n", - "0 0.0 2024-03-20T04:15:26.769000 cmorales@summit-lsp.lsst.codes \n", - "1 0.0 2024-04-24T20:53:41.843000 ykang@summit-lsp.lsst.codes \n", - "2 0.0 2024-06-19T01:59:08.706000 cmd_user@summit-lsp.lsst.codes \n", - "3 0.0 2024-08-21T04:04:38.053000 kpena@summit-lsp.lsst.codes \n", - "4 0.0 2023-11-21T19:51:54.868000 bquint@love01.cp.lsst.org \n", - ".. ... ... ... \n", - "994 0.0 2023-11-16T02:56:30.009000 echristensen@love01.cp.lsst.org \n", - "995 0.0 2024-04-19T23:27:02.011000 ykang@summit-lsp.lsst.codes \n", - "996 0.0 2024-01-05T20:15:27.453000 kfanning@love01.cp.lsst.org \n", - "997 0.0 2024-07-15T19:39:38.630000 kkelkar@summit-lsp.lsst.codes \n", - "998 0.0 2024-02-21T15:37:00 kfanning@summit-lsp.lsst.codes \n", + " user_agent ... parent_id systems \\\n", + "0 LOVE ... None None \n", + "1 LOVE ... None None \n", + "2 LOVE ... af0157e3-d7f0-4068-9a8a-e51a0a77d260 None \n", + "3 LOVE ... 99de2a12-20eb-45ae-b189-a793b63e6ab5 None \n", + "4 LOVE ... None None \n", + "... ... ... ... ... \n", + "7668 LOVE ... None None \n", + "7669 LOVE ... None None \n", + "7670 LOVE ... None None \n", + "7671 LOVE ... None None \n", + "7672 LOVE ... None None \n", "\n", - " user_agent ... parent_id systems subsystems \\\n", - "0 LOVE ... None None None \n", - "1 LOVE ... None None None \n", - "2 LOVE ... af0157e3-d7f0-4068-9a8a-e51a0a77d260 None None \n", - "3 LOVE ... 99de2a12-20eb-45ae-b189-a793b63e6ab5 None None \n", - "4 LOVE ... None None None \n", - ".. ... ... ... ... ... \n", - "994 LOVE ... None None None \n", - "995 LOVE ... None None None \n", - "996 LOVE ... None None None \n", - "997 LOVE ... None None None \n", - "998 LOVE ... None None None \n", + " subsystems cscs date_end components \\\n", + "0 None None 2024-03-20T04:15:26.769000 [AuxTel] \n", + "1 None None 2024-04-24T20:53:41.843000 [MainTel] \n", + "2 None None 2024-06-19T01:59:08.706000 [AuxTel] \n", + "3 None None 2024-08-21T04:04:38.053000 [AuxTel] \n", + "4 None None 2023-11-21T19:51:54.868000 [MainTel] \n", + "... ... ... ... ... \n", + "7668 None None 2024-04-19T19:06:54.594000 [MainTel] \n", + "7669 None None 2024-03-21T01:03:51.861000 [MainTel] \n", + "7670 None None 2024-06-12T01:55:35.836000 [AuxTel] \n", + "7671 None None 2024-03-04T03:19:22.793000 [MainTel] \n", + "7672 None None 2024-01-03T19:45:18.279000 [MainTel] \n", "\n", - " cscs date_end components primary_software_components \\\n", - "0 None 2024-03-20T04:15:26.769000 [AuxTel] [None] \n", - "1 None 2024-04-24T20:53:41.843000 [MainTel] [None] \n", - "2 None 2024-06-19T01:59:08.706000 [AuxTel] [None] \n", - "3 None 2024-08-21T04:04:38.053000 [AuxTel] [None] \n", - "4 None 2023-11-21T19:51:54.868000 [MainTel] [None] \n", - ".. ... ... ... ... \n", - "994 None 2023-11-16T02:56:30.009000 [MainTel] [None] \n", - "995 None 2024-04-19T23:27:02.011000 [MainTel] [None] \n", - "996 None 2024-01-05T20:15:27.453000 [MainTel] [None] \n", - "997 None 2024-07-15T19:39:38.630000 [AuxTel] [None] \n", - "998 None 2024-02-21T16:10:28.176000 [MainTel] [None] \n", + " primary_software_components primary_hardware_components category \\\n", + "0 [None] [None] None \n", + "1 [None] [None] None \n", + "2 [None] [None] None \n", + "3 [None] [None] None \n", + "4 [None] [None] None \n", + "... ... ... ... \n", + "7668 [None] [None] None \n", + "7669 [None] [None] None \n", + "7670 [None] [None] None \n", + "7671 [None] [None] None \n", + "7672 [None] [None] None \n", "\n", - " primary_hardware_components category time_lost_type \n", - "0 [None] None fault \n", - "1 [None] None fault \n", - "2 [None] None fault \n", - "3 [None] None fault \n", - "4 [None] None fault \n", - ".. ... ... ... \n", - "994 [None] None fault \n", - "995 [None] None fault \n", - "996 [None] None fault \n", - "997 [None] None fault \n", - "998 [None] None fault \n", + " time_lost_type \n", + "0 fault \n", + "1 fault \n", + "2 fault \n", + "3 fault \n", + "4 fault \n", + "... ... \n", + "7668 fault \n", + "7669 fault \n", + "7670 fault \n", + "7671 fault \n", + "7672 fault \n", "\n", - "[999 rows x 24 columns]" + "[7673 rows x 24 columns]" ] }, - "execution_count": 12, + "execution_count": 13, "metadata": {}, "output_type": "execute_result" } @@ -954,10 +1094,187 @@ "df" ] }, + { + "cell_type": "markdown", + "id": "bd56e1ab-ff5f-4416-9f3b-c690d5cc3adf", + "metadata": {}, + "source": [ + "\n", + "## Plots from log" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "a74d281f-952d-42d3-bf35-949a17000b49", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqYAAAGdCAYAAADALrFsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA7NUlEQVR4nO3de3xU5YH/8e+EkIRbAkFIuCQI3hAVtXghVltFKlLX1ZW63dZd0WWt20W3lXa78mvrraW41mprF9FaCvWCWtqq641LERAh4aYoF4mEW0JCEiBkJhcyuc3vj5BhJpn79ZmZz/v1mhdk5sw5z5xz5pzvPM9znmNxOBwOAQAAAHGWFu8CAAAAABLBFAAAAIYgmAIAAMAIBFMAAAAYgWAKAAAAIxBMAQAAYASCKQAAAIxAMAUAAIAR0uNdgFTR2dmpqqoqDRo0SBaLJd7FAQAAAXA4HGpoaNDIkSOVlkZ9XrQRTGOkqqpKBQUF8S4GAAAIQUVFhUaPHh3vYiQ9gmmMDBo0SFLXjp2dnR3n0gAAgEDYbDYVFBQ4z+OILoJpjHQ332dnZxNMAQBIMHTDiw06S0h65JFHZLFY3B7jx493vt7S0qLZs2dr6NChGjhwoGbMmKGampo4lhgAACD5EExPueCCC3TkyBHn46OPPnK+9sADD+jtt9/WsmXLtG7dOlVVVem2226LY2kBAACSD035p6Snpys/P7/X81arVYsWLdLSpUs1ZcoUSdLixYt1/vnnq6SkRJMnT451UQEAAJISNaan7N27VyNHjtS4ceN0xx13qLy8XJK0bds2tbW1aerUqc5px48fr8LCQhUXF3udn91ul81mc3sAAADAO4KppCuvvFJLlizR8uXLtXDhQh04cEDXXHONGhoaVF1drYyMDA0ePNjtPXl5eaqurvY6z/nz5ysnJ8f5YKgoAAAA32jKlzR9+nTn/ydOnKgrr7xSY8aM0Z/+9Cf169cvpHnOnTtXc+bMcf7dPdwEAAAAPKPG1IPBgwfr3HPPVVlZmfLz89Xa2qr6+nq3aWpqajz2Se2WmZnpHBqKIaIAAAD8I5h60NjYqH379mnEiBGaNGmS+vbtq9WrVztfLy0tVXl5uYqKiuJYSgAAgORCU76kH/7wh7r55ps1ZswYVVVV6eGHH1afPn30rW99Szk5OZo1a5bmzJmj3NxcZWdn6/7771dRURFX5AMAAEQQwVTS4cOH9a1vfUvHjx/XsGHDdPXVV6ukpETDhg2TJD399NNKS0vTjBkzZLfbNW3aND377LNxLjUAAEBysTgcDke8C5EKbDabcnJyZLVa6W8KAECC4PwdW/QxBQAAgBEIpgAS3sZ9x/SnLRXxLgYAIEz0MQWQ8L79wiZJ0vkjsnXR6Jw4lwYAECpqTAEkjcr6k/EuAgAgDARTAAAAGIFgCgAAACMQTAEAAGAEgikAAACMQDAFAACAEQimAJIIN7IDgERGMAUAAIARCKYAAAAwAsEUAAAARiCYAgAAwAgEUwAAABiBYAoAAAAjEEwBAABgBIIpAAAAjEAwBQAAgBEIpgAAADACwRQAAABGIJgCAADACARTAAAAGIFgCgAAACMQTAEkDYcj3iUAAISDYAoAAAAjEEwBAABgBIIpAAAAjEAwBQAAgBEIpgAAADACwRQAAABGIJgCAADACARTAAAAGIFgCgAAACMQTAEAAGAEgikAAACMQDD14PHHH5fFYtH3v/9953MtLS2aPXu2hg4dqoEDB2rGjBmqqamJXyEBAACSDMG0hy1btuj555/XxIkT3Z5/4IEH9Pbbb2vZsmVat26dqqqqdNttt8WplAAAAMmHYOqisbFRd9xxh1544QUNGTLE+bzVatWiRYv01FNPacqUKZo0aZIWL16sjRs3qqSkJI4lBgAASB4EUxezZ8/WTTfdpKlTp7o9v23bNrW1tbk9P378eBUWFqq4uNjjvOx2u2w2m9sDQHQ54l0AAEBY0uNdAFO89tpr+vjjj7Vly5Zer1VXVysjI0ODBw92ez4vL0/V1dUe5zd//nw9+uij0SgqAABAUqLGVFJFRYW+973v6ZVXXlFWVlZE5jl37lxZrVbno6KiIiLzBQAASFYEU3U11dfW1upLX/qS0tPTlZ6ernXr1umZZ55Renq68vLy1Nraqvr6erf31dTUKD8/3+M8MzMzlZ2d7fYAAACAdzTlS7r++uu1Y8cOt+fuvvtujR8/Xv/93/+tgoIC9e3bV6tXr9aMGTMkSaWlpSovL1dRUVE8igwAAJB0CKaSBg0apAsvvNDtuQEDBmjo0KHO52fNmqU5c+YoNzdX2dnZuv/++1VUVKTJkyfHo8gAAABJh2AaoKefflppaWmaMWOG7Ha7pk2bpmeffTbexQIAAEgaBFMv1q5d6/Z3VlaWFixYoAULFsSnQAAAAEmOi58AAABgBIIpAAAAjEAwBQAAgBEIpgAAADACwRQAAABGIJgCAADACARTAAAAGIFgCiBpOBzxLgEAIBwEUwAAABiBYAoAAAAjEEwBAABgBIIpAAAAjEAwBQAAgBEIpgAAADACwRRA0nCI8aIAIJERTAEAAGAEgimApGGRJd5FAACEgWAKAAAAIxBMAQAAYASCKQAAAIxAMAUAAIARCKYAAAAwAsEUQNJgHFMASGwEUwAAABiBYAoAAAAjEEwBAABgBIIpAAAAjEAwBQAAgBEIpgAAADACwRRA0nAwWhQAJDSCKQAAAIxAMAWQNCyWeJcAABAOgikAAACMQDAFAACAEQimAAAAMALBVNLChQs1ceJEZWdnKzs7W0VFRXr//fedr7e0tGj27NkaOnSoBg4cqBkzZqimpiaOJQYAAEg+BFNJo0eP1uOPP65t27Zp69atmjJlim655Rbt2rVLkvTAAw/o7bff1rJly7Ru3TpVVVXptttui3OpAQAAkkt6vAtggptvvtnt73nz5mnhwoUqKSnR6NGjtWjRIi1dulRTpkyRJC1evFjnn3++SkpKNHny5HgUGYAHjGMKAImNGtMeOjo69Nprr6mpqUlFRUXatm2b2traNHXqVOc048ePV2FhoYqLi+NYUgAAgORCjekpO3bsUFFRkVpaWjRw4EC98cYbmjBhgrZv366MjAwNHjzYbfq8vDxVV1d7nZ/dbpfdbnf+bbPZolV0AACApECN6SnnnXeetm/frk2bNum73/2uZs6cqd27d4c8v/nz5ysnJ8f5KCgoiGBpAQAAkg/B9JSMjAydffbZmjRpkubPn6+LL75Yv/nNb5Sfn6/W1lbV19e7TV9TU6P8/Hyv85s7d66sVqvzUVFREeVPAAAAkNgIpl50dnbKbrdr0qRJ6tu3r1avXu18rbS0VOXl5SoqKvL6/szMTOfwU90PAAAAeEcfU3XVbk6fPl2FhYVqaGjQ0qVLtXbtWq1YsUI5OTmaNWuW5syZo9zcXGVnZ+v+++9XUVERV+QDAABEEMFUUm1tre68804dOXJEOTk5mjhxolasWKGvfe1rkqSnn35aaWlpmjFjhux2u6ZNm6Znn302zqUG0BOjRQFAYiOYSlq0aJHP17OysrRgwQItWLAgRiUCAABIPfQxBZA0LPEuAAAgLARTAAAAGIFgCgAAACMQTAEAAGAEgikAAACMQDAFAACAEQimAJIG45gCQGIjmAIAAMAIBFMAAAAYgWAKAAAAIxBMAQAAYASCKQAAAIxAMAUAAIARCKYAkobDwYBRAJDICKYAAAAwAsEUQNKwWCzxLgIAIAwEUwAAABiBYAoAAAAjEEwBAABgBIIpAAAAjEAwTQGN9nYtWFOm3VU2LdlwQIdPNAf1/sMnmrVkwwE1t7ZHqYTx0dHp0Mslh1Ra3RDQ9O98VqWFa/fprx8fDngZ1pNtWrzhgGobWrxOs3xntdaU1uqlkkMqq230OM2GsmN6f8eRXs9vO1SnW/73I+076vl97+04oo1lxwIuryTtqrLq1c3lemt7pZbv7L1Mk0V7uKjahhYtWFOmWtvp7Vlra9HiDQdka2kLaB4rdlXrwy+OenytvaPz1H7gfZ98a3ulNh+oC67gHnx+xKalm8rV2Rn4Otu0/7j+79OqsJcdS8t3HtGGIL8Drjo6HXplk/fjRGX9SS3ZcEBN9sCOj3trGvRSySF1uKz37u2+t8Z9GW9/WqVN+4+HXHZvPj9i03Pr9sne3uH2fEenQy98uF+fHa4Ped4NLV3HvBqb92OeNwePNenF4oO9yoXUkh7vAiD65r37uV7dXK5friiVJD258gvtfHRawO//+m/Wy9bSrv3HmvTYLRdGq5gx96etFfrJmzslSQcfv8nntPuPNuq+pZ84/x4ztL8mjcn1u4wfLvtUq3bX6NXN5Vr5wFd7vV5ja9G/v7zN7TlPZbnj95skSRsenKJRg/s5n5+xsFiSdP2v1vV6X0Vds/7jlY+9ztObm575yO3vL34+XRnp/IaVpHte3KZPK+r13o4jevc/r5EkfeuFEu072qTNB+q08J8n+Xz/sUa77n2pa3sfmP/1XqMIvFxySI+8vVuS5222p9qm77223evrwZj+m/WSpAGZfXTLJaMCes83f1ciSTo/f5DOyRsU1vJj4Yj1pP795eC/A67+su2wfvyG9+PEzb/9SHVNrfqitlG/+IeL/M7va09/KEmySPrnyWMkSa9uqdBPexyLymobdP+rn4RVdm+6t317R6fum3KO8/llWys0773Pw1rm3L/u0DufHdHiDQf14Y+uC+q91z65VpJU39ym/7z+HN8TI2lxtkkBWw+61640BvjLvputpWv6cGodTPTZYWvA0x6xuv/6P3AssFrnVbtrJElf1Hiu0Tze2BpwGSTpWIM94Glrg5jWl04GrXf6tKJekrSryuZ8bt/RJknSB3tq/b6/vtn39v7k1Py9OVx30u8ygrX7iM3/RD1UWYOvDYuHYw3Bfb882VHp+zhR19S1jI/2Bnd8dK2V/NTDdq+sj/463lnpvu33BNh65MuaU9+D8rrgWuZcbTkYfosAEhfBFAAAAEYgmAIAAMAIBFMEjAbd0+J1T/Z4LJWW/MAEu5qMWa8hlCNe+7/JHEGuSBNWYbBlBmKBYJoCuEujZ8Gsl1BXob9lRHPbsN1N5Huj+NtkbNPYi8U697SIeGxq9i+YgGAKAAAAIxBMASAOaEQFgN4IpgAAADACwRQAAABGIJgCAADACARTBI5OcU7xWhXxGKaHIWUCFORqMmXIpVBKYUbJzRLs5jRhHRqyC/ZiarkQGwTTFGCJy8Aj5gtqrYS4CuM5/A9b3Tz+hw/zM5wUGzXmYrHKPW3XeGxrzhUwAcEUAAAARiCYAgAQJpqfgcggmEqaP3++Lr/8cg0aNEjDhw/XrbfeqtLSUrdpWlpaNHv2bA0dOlQDBw7UjBkzVFNTE6cSA0h05BgA6I1gKmndunWaPXu2SkpKtGrVKrW1temGG25QU1OTc5oHHnhAb7/9tpYtW6Z169apqqpKt912WxxLDQAwBf1/gchIj3cBTLB8+XK3v5csWaLhw4dr27Zt+spXviKr1apFixZp6dKlmjJliiRp8eLFOv/881VSUqLJkyfHo9gAAABJhRpTD6xWqyQpNzdXkrRt2za1tbVp6tSpzmnGjx+vwsJCFRcXe5yH3W6XzWZzeyQ6mh5dpNDKoO9cYBhWK7XxPQEig2DaQ2dnp77//e/ry1/+si688EJJUnV1tTIyMjR48GC3afPy8lRdXe1xPvPnz1dOTo7zUVBQEO2iw0D+hv8JFue+5GFKkAlpPFVDyp7ITNj+BhTBI37kpTaCaQ+zZ8/Wzp079dprr4U1n7lz58pqtTofFRUVESph8Oj75Fkw6yVa4/tFc9zASIdihM/vuLZ+X2ebxlosvkeetms8tjWHDJiAPqYu7rvvPr3zzjv68MMPNXr0aOfz+fn5am1tVX19vVutaU1NjfLz8z3OKzMzU5mZmdEuMgAAQNKgxlRdTVn33Xef3njjDX3wwQcaO3as2+uTJk1S3759tXr1audzpaWlKi8vV1FRUayLCyAJ0FwJAL1RY6qu5vulS5fqrbfe0qBBg5z9RnNyctSvXz/l5ORo1qxZmjNnjnJzc5Wdna37779fRUVFXJEPAAAQIQRTSQsXLpQkXXvttW7PL168WHfddZck6emnn1ZaWppmzJghu92uadOm6dlnn41xSQEAAJIXwVSBXZWalZWlBQsWaMGCBTEoEQAAQOqhjykCFtKwMkkq0P6Bkb7INR6bgK0emGC3jSlfp9BGizKk8AmMdeidKd8NxAfBFCkrmOFYojWMSjSHZ2HkF/P4HXoo3PGkkJA87RaxGLqpZwBk94IJCKYAAAAwAsEUSCAMgO0bTYCIF7o6deHGHggXwRRIIKGe+zhpAogFjjUIF8EUAIAwUVMIRAbBFAAAAEYgmCJgNNCclkqtVTTNBYa1lNr4ngCRQTAFoijyrXuhnfw4Z5rHlG0SSjFMKXtCM2IdGlGIXti/UhvBNAXQ98mzYFZLtNZgNLcMm908/ocp9T0FmzT2YvE98jiOafQX23uZ7GAwAMEUSCicOQATUcnXhYoQhItgChjOve9aiE354Sw/jPfCO25JGVv0AY0N1jPCRTAFACBM1BMCkUEwBQAAgBEIpgAAhIkGbCAyCKYIGF2HTgt0Vfi7yjro5cbhlqRs98AEu45NWa+hlMOUsicyVqF39L9ObQTTFEDfJ8+CWS/RutI0mhewRjoUI3z+trf/19mmsRab71F8xovq+QOD/QsmIJgCCYTzBmAmapG7EG4RLoJpCojU8ZLmldMCPglF4BgdcvN9pLZXAm32eO6jgSzZ37b0/3oCbQwkHPYvmIBgCkRThI/zoYdUmIZtEltkrtgg3CJcBNMUEKmGFfosnpYIrVVsr9gKZG3TxzR5JcOmYf+CCQimAACEiYpCIDIIpggYfUxPi2Uf03hjuweGtQQA4SOYAimA2hzzmNIXL5QfHmaUPLGZsv1NxKpJbQTTFEC3Ic+C6U8VvXUYvY3DdjePv36//jYZmzT2YvE98rSMWPQR75n/IrFE+qkiXARTAEmDmhbEC11egMggmAKGc3j5f3Dz4JakpmG1xhbrOzboooBwEUwBJA1aEREviTg8W+KVGKmAYAoAQJgSsSk/8UqMVEAwRcBooTkt0JNQMtRIsNkDw/cD0ULzOFIJwRRIAZzXzGPKNgmlHASl8LEGvWPdpDaCaQqg3134orUK2TapJfxbkkauLAhMLFa5p2XEYlv3+oERgWUyXBTCRTAFACBMVCIDkUEwTQGROmBy4D0t0HURicqDeDebxnv5wTC9qP7KF+7rSE4x2+7sXzAAwRSIokifUAgmSYRtGVOJ9AMrkbGeES6CaQqIVJcfug6dxrpAKOhjCqOxf8EABFNJH374oW6++WaNHDlSFotFb775ptvrDodDDz30kEaMGKF+/fpp6tSp2rt3b3wKi4RCkAAAIHAEU0lNTU26+OKLtWDBAo+vP/HEE3rmmWf03HPPadOmTRowYICmTZumlpaWGJc0vmihOS3R1kU45U2wj5owEnFA9m6JW3JzBPOdTLn1nXIfGK7S410AE0yfPl3Tp0/3+JrD4dCvf/1r/eQnP9Ett9wiSXrxxReVl5enN998U//0T/8Uy6KGJBFvlRcLwdRmRqvmM5pbhtraxOPvu8p3OfZi8T3ytIx4bOlI7F8MF4VwUWPqx4EDB1RdXa2pU6c6n8vJydGVV16p4uJir++z2+2y2WxuDwBAckrESr5ELDOSH8HUj+rqaklSXl6e2/N5eXnO1zyZP3++cnJynI+CgoKolhNmooYrthKtiwUAwB3BNErmzp0rq9XqfFRUVMS7SIiDSPQjdJ1DqEOxhFMOwl50hLteGZYnOIm8tjxt622H6nS0wR6H0vjGfolw0cfUj/z8fElSTU2NRowY4Xy+pqZGl1xyidf3ZWZmKjMzM9rFA+CC7m2Il1juehv3HdO3X9ikNIu0f/5NIc+HrwtMRI2pH2PHjlV+fr5Wr17tfM5ms2nTpk0qKiqKY8mQCqrqT1JjCSSAWH5N1+89JknqDLfW3c/rDS1t+v36/TpiPRnegoAgUGMqqbGxUWVlZc6/Dxw4oO3btys3N1eFhYX6/ve/r5///Oc655xzNHbsWP30pz/VyJEjdeutt8av0IirQM8H4fQxfeHD/Zr33ue655qxIc8jEhJ5WKNk5nBQQ5wq4vUN/MmbO/XW9iot+uiAiudeH6dSINUQTCVt3bpV1113nfPvOXPmSJJmzpypJUuW6Ec/+pGampr0ne98R/X19br66qu1fPlyZWVlxavISAHz3vtckvTC+gNhz4taV/OYsklC6RPI/hS+cFZhrH6PrPviqCTpiDXwMbsjMVwUP4ZTG8FU0rXXXuvz4GyxWPTYY4/psccei2GpIodaFc+Cq82MzkqM5ph/jAhgHm5JmngC/Y6GE9Y9fVfjMR5oz0WyuyEe6GMKRBFBAoCpqPmGiQimKSBSBx+GAXGRAOsiYs1hHmbz0Fs7NedP243bJwwrTi/+yhfM64Z/1KB1djp070tb9csVe+JdFOP03C+i9YPX9O8PUgPBFIiiSB/oQ51dJIvR2t6pF4sP6a8fV+rwCa7WDZVpoT7eNh2o04pdNVqwZl9U5s/qDl4o3QnYrxEugmkKiNSva+6B7CIB1kW0+pi61sS2dXRGZRnJij6m3rWyLwUsUt/tXn1Kk3j/QuIgmAJRlIwHei6qQjSwV8UelZswEcEUAaOJxkWCrYtwtp2vdybWWjBLuOsuUt/HUG9yi/AEs/3iNXxSKD8WIjJcFLtXSiOYpgBqIjwL5vgZrZrPaG6bqJWZHSpq/Dblx6YYcWHqfhV4sUJPU54+u6/1kYjHIyBQBFMgipLxQE9tBpAc+CrDRATTFBCpgw8HsdMSYV1Ebpiw6C8jUgwrTi9hDxfl5f9IbrH6nvVcjKm12EhuBFMgiiJ9Pgn1BBXJcnC7wMgwLdTHW7QvqmO/jQ2uRUC4CKYpIFKHe348n5YI6yI2tR1mnYQSYbv4ksp9TBF7Pfcn9i+YgGAKRBEHeiAwNBsHLlKryv/PSjYKYo9gioCZVTcWX/FaF/E4efdsAqWlzgxsB0RbKMcbbsSCcBFMgQQSch/TKIUYwlHoTOnzGMo2ZLuHj1XoHesmtRFMUwG/YD0KZq1Eaw1Gc9Ow2RNR6nYyNfajBViwcMK6p0X4XGyUvtwcM2ACgimM9edth1Wy/3i8ixGWcJq1EuEkQc0GEJp3PzsS7yL4vYI+AQ5BSEIE01QQqVsXxjCF7Ky06ofLPtU//a4kdgsNQiI0ZcZiHFPTJFBRvQh8IFNTugJETAqmoC0H6wKaLmbjmCbZLoXERDCFkQ6fOOnxeXt7h/YfbYxxaSKnoq5ZjfZ259/BnghCDiMRPOF4K3Nnp0P1za2RW1CyS5AQUFl/UraWtrDm4XA49MsVe/TejvBqCeubW1VtbQmxDGEtOioOHmsK6X0mZ3jGMUW4CKapIEJtwrFsWva2rNufK9aUX63T2tLa2BXGg1DWxYFjTbrmiTW6Yt7fAltG8Itwf38MtpfrOWjWH7foksdWaWelNfoLTlpm9TGtrD+pLz/+gS55dGVY81lbelQL1uzTf7zycVjzueSxVZo8f7WszeEFZVPEO8L17GqUCN2HkPwIpghYLH8Iezs+fna4K/Qs23o4doXxIJR18VHZMUlSc2tHyMuN9t1xPAn0o64pPSpJeqn4UPQKA6dYfB+3HOhqau4Mc1m1Df5rOYPZt8uONoRTnIQTiW4b1dYWdfbYkN21mzW2FnV42MgMF4V4IJgiISVi/7pIHK5D/dyRXF+u8/I030TcNvFgylrytb28ZYxgyx7sDyp/zcFJ01oc4ucINvut3FWtyfNX6/5XP+n1WvG+47ryF6t11+LNoRUmCugOkNoIpikgEX+/xuJXdzCLiER5gp1FuMuMR+0qwuP/lqQJuk0DKLapFW2BrvNQolT3jwJP33Vf6yPY/WDB2n2SpHc99PH948aDkqT1e48l7v6FpEIwhZH8HR4T5Qd1Mh7mE2Xd99TW0aknV5Qm/BBkiSjY74G/fSxBd8G4cV3/PS9ki2QA7ungsSb94r3PVWsL7YI1pCaCaQqI1EHcpCbaeIejUJqaYl0bEant5euzenopntvGV1lfKTmk/11TFtchyPwGLr+BLDord/3eo3pu3b5e6y9SLRfBzsffp4z39z9SAv0c4X5e19X/+Pt7vL4W6f1rxsKN+t2H+3Xf0t5dCABv0uNdAMATf+cxk0KyL66l9PSZgv4Uht2S1OOyXP7//o4jysroo+vOGx67Anhx8HhzvIvgxqRw9S+LuvoXnps3UFPG5zmfDyWW7j/aqLWlR/XtKwuV1bdPwPNxnaYrIHt/V7L0QQz1UwTdLcjl/7urbD1ei+wPZtdtc7ypa/i4rYcCG68VkAimKSFSh51wD2AVdc061mjXpYVD/C/L8DbwUGqSgn1H2MNFRamG1vVk6qvG9GiDXd89NTzQ/l98XWlpseg3bPiO40e8+5j2HD84lNU55VfrJEn1J9s052vnhlSO5Iidiafn/pXgXyckKIIpYuaaJ9ZIklb/4Ks6a9jAsOYVzQqTwyeaZTvZrgkjs8OeVzjHdY8nhQQ4UXTXZltPtrk8h0TQ83sVThDe6nJXo0ACTjA/KhJtf3I4HLpv6ScaM7R/j+djs/w0l3XbazX77GMaPE/bMdG2F+KLYIqARar5fFeVzW8w9XdCjOaB7ur/6QrQxXOnaEROP8/LD6WPaQKESk+C/qinpnf9vJ0Oh/rEOVUn6vr3Jhqhpncf03DmFfp8wu2La5qPy094vCI+0GNqz6mCbn1xeUMQuTRiIrm9OjsdOtnWoQGZxJdkxcVPiLmAQp2/PqYxODHtrYnsrU8j0gwb/zuSum0/n2Nguvy/04AkYdpQOKb0k3bdNIGWKJDN6fr5gh7H1JB1Ewxfx7XWdnM+T++7PZn1vZB874czF2/WBQ+vUGW959tWI/ERTFOAaccdAzKKJP8HZNcwFfIqtHj5f0BvDXMc0zhvd9f1G4ltfrK1Q79fv18HQri/+LZDJ/SHDQfCL0QPu6oie/tVf5ss2tu0d1N+5Oblj9vV4R77Lgf2gyjSIrHOvd6owOfHiNzGdj2W+Kox7VlOj83yEfgyOxwOfVHToLaOzqDfu35v1x303vykMuxywEwEU8RcZE4q0T8xRXoJhv0+CJnfi59O/ev6eT3d7jBYv/7bF/r5u5/ruifXBv3eGQs3hr18T/7utx9FZb7RVNfUql+uOD1k0CGX0Qp6NRmH05Qfofn4nXkCSPOyAmL2MVyb8v38APClxtaiy+f9Tf+zfI//iX14fUuFbnj6Q3335W0hz6Pn7VWRPAimKSBSNZSxnE/voWOiUxaffCwjEQ6J8aqZ7t5ePfuYhmvzQc9DzrS71LrEehihYBfn70dZz1e3V9TrlU2HnJ/Lvdk9tM8696+facGafc6/y2pPd1npvf4ilEwDEMxNNRLh++cq3GAebt9f94Yb71fe+9ufn11TpmONrVq4dp/vCf34/UddrRd/+7w25Hkk2j6AwBFMERH7jzbqwb98pkPH/TezBhRMI9wMHIpINxcG3Szm5+RTbQ38bipN9vaApw3X6RrT0x8gEpUbnmqdmlvbNXn+6vBnHkEvFh/Uyl3VfqcLZL++dcEG/fiNnfpgT+gn8J62Hjzh9nd1lO7K49bHNNgB9n3UxIdcnjgmGa+fPkaFsrgnU/fXYlICd5EYOc6ULmGIPIJpCohUM5qv+dzx+016bUuFc6BuqavJcJuHgZVDvMg7oHk4HA63GrRwhJEZA37PXz8OvZ/UN57z3Tztur289a+sb27Vb1fvVUVd4IPP+zshHDrerJ++uVOHT5ye5/serkgOlqeT2fq9x3SssTXseUfKFzUNeuitXfrOS8E3UdY2tHjdd784dSFez+9gra0l6H56vkJirz6mfnb0Rnu7rM1tHl8LJzh4+lHo1sc0gHlbm9v0yNu7Qi9EGOztHfrZO7u1sayrP6TXPqYxKo+3PqYWi/v+EKt+6d66NgTDhAsqER0EUwTM13HgyKnau3KXgHPdk2s1Y2Gx1pS61/YEckDxd0V3S1uHXi455BZ+JOmB17fr8nl/83qyDIavYoZySPR0LP7fNWXep/czv56DofviOqaoqwf/skO/WvWFbvPRBzPY4//2inq9VHJIdy3ecno5f90R3Ew88HQxWCROcKFqaOm9To812kOa1xc1Dbpi3mr9aethj697+g58fqRBV/xitW57Nrj+s75qqxxyyNrcppeKD6quqdXnPuhwOHThwyt08WMrdbK1w8O8Tgvozk9BNCkH0prxi/c+17ZDJ/xOF6yy2kY1t/pugVj00QEt+uiAvv37TZLCv/K9d99f7/PbduhEr7s7ub/XZb4O39sm3hdQ+kIsTV4E0yAtWLBAZ555prKysnTllVdq8+bN/t9koMYYNO12h6FVu2vcXwioKd9lcg/Tb9x3XD95c6em/3q92/Nvbq/SieY2vfWp/5rIKj/DjcTrF3lre6fXE1+oJfJ2ocDyU03ORxuCCFSuff18FKjVR03etkN1evTtXUF1MfB0koz2zaR8/cB5+K3etXF9Qhwk/u1Pq3xO62n7dV+VvKPS2vs75oOvIjoc0gN/2q6fvrVL9760VXu99D91yOF2QVvPH4g9pw96HFNJh443yd7e4fZct/1H/XcZ2lvbENxCA7D5QJ2mPrVONz3T+6I31/KV97j9rbeP372KPqmoj0j56ppaNWPhRn39mfVe1384o31E6ojoL6gHNBwZNaZJi2AahNdff11z5szRww8/rI8//lgXX3yxpk2bptrayPX/igZPh4Cbnlnv4dno6HlS/dFfPtMXNYGfNHzVjjR4CTbtHf4PWu985rt5ORJXFfsaisWb655cqwkPrZC93XuwC/Yq9w5HV/OizUMtX7fbn9uouqbYNIvPWFisxRsO6jer9/qc7r0dR5y1Xp5qR/3VmFbVnwxpeClJWrqpXBc/tlJ/+MhzN4iVHsKg621X/W2j7hPril3V+u0H3mvOpdN9dF0/bR+XZd3z4laf73cro6+mfMnZn3XLwRP65YpSl/K6T9vh8kSHh5DgXmMaUJ2p838l+47rq79cq398vsTj8h95e1evYLLui6O6dcEG57Glj4dfLburbPr2CyX6uDzwmtQXNx50/v/dz7p+QHjap1yL83n16eNbZ6fDaxDr/gyfegimoRxzXH9gtnc61GhvV0tbh9u8eo3/6WPw/WgJ9gfl/3tjh+a8vt1jd45XNh3SPz5XHJFWMpiBYBqEp556Svfcc4/uvvtuTZgwQc8995z69++vP/zhD/EuWtAOHW/WwVMHV4fDoeU7j+jaX67RhlN9osLR84Th6QR9w9Mf+gxJrieyUH4Yd9d2/m13jf7z1U+cNcTlx5v1P8v3eK0hdC37/31apb01DapratXiDQeDL0QPgdZUBDJw9L/9cYvfaVx1Ohya/uv1mvjIStU3t2rJhgN60iV0SF1B5OlVXwQ132B4al7c6+MHSlltg/7jlY81Y+FGtbZ3aqeHMUPT/Jzhrnr8A4/DS5Ufb/a67OfX7dOsJVv0/97o6n7w2Du7PU7nqdWh0qV7hb++n1f/zxq1dXTqXi/9Uf/+f0/Xyi3Z2Dsc9wyYgdQgHWu0q83Hj7ZAu6/Y2zrdvtfd/3ddp4He+cnhcOiHyz7Vz989vZ5/e6qLi2tg+9Uq95A8c/EWtbR1qKy2UR2dDs38w2Ztr6h3DkHkKYD/2x+3aOO+40F1f2hy6abw6WH3fdC1pru7haisttGt3Jf+bJXP/bynuqZWHW2wa23p0YDf0801jNtOtuniR1dq8vzVbsee8rqetbneN060mvKD6YJjb+/Q0k3l+usnlW7dl7orLH78xk5tPlin+e9/HvFyIj4sDurDA9La2qr+/fvrz3/+s2699Vbn8zNnzlR9fb3eeustt+ntdrvs9tPhx2azqaCgQFarVdnZ4d+DvVtFXbNeLD7oc5oX1nuu8ZGkzPS0XjVzt08are0V9RoztL/GnjHA7f33XDNWUldt0YmmVn31vOHKz850m6ZvH4uGDsh0Xu2bmZ6mO4vGeCxH9/x6OnCsWX/7vKtG6q6rzlTfPhavn+XbVxZqQEafXq8XjRuq4v3H3ZblbV10l8PXunI1KCtdDS3tGnfGAF1//nCv07nO74ozc51DHgW7PFfj8wdpT3XvE90/Ty5Uv759nH/XNbXpLx/37rN42Zgh2hpE37tvXlagyvqT2lPdoItGZasgt79eLD4kSfr7i0cqLztTUmCf5ZKCwbr8zCFu014/frjK65qVZrHosjOHqP+pbbmnusE5mHZP3etv/9EmrXa5Yv1LhYM1acwQSV19Xbcc9P05vzFptIb07+v2nKfPMevqsVpbWqt9ATQhd7vrqjNlb+/Qsq2HNWpIP90wIU+2k+16fWtFwPPoOb8j1pNasavre5GXnakam/sPrG9eVqDsfp5v1bh0U7lbyPJk3LABXpvJ//XLY71eRHfO8IE6eLypV+i9tHCw0tMsOnCsyXmR2r9dPdYt8Bw+cVLv7/Q+ikEo35XpF+b3mmfP7423Y09P3pZ73XnDtKZHePznyYV6uaQ84HKePyJbV5891O9nu7NojDLTT9cjrd5T69xOrp/DerLN2U952gV5zn3Fl/4ZfdR8ar/IzkqXraXrB9dtl47SX10Gse95/PS2/vwdY+ub27Rs2+Fezx883uzWJWV8/iClWSzafcR7f9meZfrmZQW6YFS27iw60+t7QmGz2ZSTkxPx8zc8I5gGqKqqSqNGjdLGjRtVVFTkfP5HP/qR1q1bp02bNrlN/8gjj+jRRx/tNZ9I79hbD9bpG88VR2x+AAAkqsvPHKJl/35VROdJMI0tzz+tEba5c+dqzpw5zr+7a0wjLS87S/d+dZzPaTo7HfrLx5W6+uwz9FHZMdU3t2pg5ulfxpKUnmZReh+L/uHS0RqUla7XNpfrmnOHafSQfjrR1KqVu2v0D5eOUsapX+3V1hZ9sKdW37qiUBaL1Gzv0JufVKqxtV3/cMkopaVZ9OdthzWkf1/942UFkkUqrW7QpxX1arS3yyKLbr9stAZmed8F1+45qtwBGZpYkON8rtraoo/2HtO4YQOctWGzrh6r9FM1qvtqG7WnukF9+6QpLztTI3P66cO9RzXjS6Mli/T8uv3OeXXXeFw8OkeTzxoqSWpoadfSTV01HgW5/XT2sIEqyO2vNz6p1OD+fdXY0q5BWX01/cJ8vb61QleffYZGDenn9TOUH2/W+zurdfuk0codmKFVu2s0anA/TRjZdXCzt3VqycaDGjdsgJrs7fpS4RBZT7bpWKNdV511hpac6t/WXfubnZWu1o5OffmsM7S+7JhaXWq77/3KuF6dxLo/791fPlNltY2qPHFSX7sgT9vL63X4xEm3bgNDB2ToG5NG69DxZi3fVa1hgzI1cnA/TR6Xq5bWDi3bdli3XDJK2f3S9Ul5vRpb2nXNuWc431954qRK9tdp+oX5eqnkkDLS03T+iGy3Zs3bLh2lYdmZOt7Yqjc/qVTfPmn6x8tG6/CJk9pZZdWtl45yK//KXTU6cKxJN16Qrw1lx9TpcGjqhDzl52S57Se1DS0a3D9DN1yQ53y+e93mZ2fp+vOH691TQ1ZdfmaurCfbZG1u07Xjh/XaZmU1jSqtadD144frze1V+vpFI5TdL11r9tSq03F6QPqvnjtM677oqjHrn9FHV47NVcn+Op1s69Cowf30dxePkCS98+kRnZs3UOfmD5IkffjFMX1+qhbonmvGqr65TSt316hf3z66YGS2dlZZ9aXCISqtblBrR6cOnzipSwsH64qxuZKkdaVHNbh/X11cMNhtf85I71qXAzI9f6eONtj1148rdeXYXJXWNKilrUMtbaf3n9FD+ummiSPU0tpx6nPn67PDVu2qsunLZw/VhaNytLHsuHZUWnXXVWfq/z6tUntHp2wt7bqzaIyqrS1aubtGaRYpp19fXXFqfYwe0k+t7Z1ySBqRk+Xc9111r5MROVkakJmuGluLMvqk6fIzczXmjP6SumrHe17k1d3i883LCpSWJr26uUJZfdN0x5Vj1OfUMai73/S9Xx0ne1un3vikUtMvzFdOj5pyb7q/w5J06yUjtflAnS4claOxwwZo+c5q552zJo7OUdFZQ1Wyv04Oh0OfHbYqo0+a+mf20e2TRmtD2XEdbbQ7uxEV5vbX9IvyJUk7K63aUHZcN16Qr2ONdjW1dmjcsAF697MjumJsri4tHNyrXO/tOKLz8gbprOED3Z4v3ndc6WkWXT42VxvLjqtvH4suOzNXv/uwa185Y2CGs/b6O18ZJ4ukNz6p1GVnDlFBbn89v26/Bmam647JhWprd+iNTw7r+vPzNHRghuSQnv9wv/Kzs3TLpSM9rq+Kuma9t6NaV501VGeeMUBLN5VryvjhOievq5wOh/TqpnI12NudxwOpq+vBq5sr1K9vH+X066vzRwxya52ZMCJbHZ0OHTzeJHt7p2YWjVFWRh8drjupd3cc0dnDB+r684dr9JD+AW1XmIsa0wAF25Tfk4m/uG5dsEHbT4WGg4/fFN/CxMiZD74rqatp7LFbLoxzaYDwdO/P/3n9OZrztXPjXBogOZl4/k5mXPwUoIyMDE2aNEmrV5++y0xnZ6dWr17t1rQPAACA0NCUH4Q5c+Zo5syZuuyyy3TFFVfo17/+tZqamnT33XfHu2ghoaocAACYhGAahG9+85s6evSoHnroIVVXV+uSSy7R8uXLlZeX5//NMIrBNzQBgsb+DCBZEEyDdN999+m+++6LdzEAAACSDn1MAQAAYASCKQAAAIxAMAUAAIARCKYAAAAwAsE0lXFvBQAAYBCCKVKSxcIAO0ge7M4AkgXBFAAAAEYgmAIAAMAIBFMAAAAYgWAKAAAAIxBMAQAAYASCKQAAAIxAME1hjGIKJAeLGC8KQHIgmAIAAMAIBFMAAAAYgWAKAAAAIxBMAQAAYASCKQAAAIxAMAUAAIARCKYpzMF4UUBSsDBaFIAkQTBFSuJEDgCAeQimAAAAMALBFAAAAEYgmAIAAMAIBFMAAAAYgWAKAAAAIxBMAQAAYASCaQpziIFMgWTA6GcAkgXBFCnJwqkcAADjEEwBAABgBIIpAAAAjEAwBQAAgBEIpgAAADACwRQAAABGIJimMAejRQFJwcIgEwCSBMEUKYkTOQAA5iGYSpo3b56uuuoq9e/fX4MHD/Y4TXl5uW666Sb1799fw4cP13/913+pvb09tgUFAABIYunxLoAJWltbdfvtt6uoqEiLFi3q9XpHR4duuukm5efna+PGjTpy5IjuvPNO9e3bV7/4xS/iUGIAOM1CEwCAJEGNqaRHH31UDzzwgC666CKPr69cuVK7d+/Wyy+/rEsuuUTTp0/Xz372My1YsECtra0xLi0AuHPQYRxAkiCYBqC4uFgXXXSR8vLynM9NmzZNNptNu3bt8vgeu90um83m9gAAAIB3BNMAVFdXu4VSSc6/q6urPb5n/vz5ysnJcT4KCgqiXk4AAIBElrTB9MEHH5TFYvH52LNnT9SWP3fuXFmtVuejoqIiassCkNroYwogWSTtxU8/+MEPdNddd/mcZty4cQHNKz8/X5s3b3Z7rqamxvmaJ5mZmcrMzAxo/vFCtzQAAGCSpA2mw4YN07BhwyIyr6KiIs2bN0+1tbUaPny4JGnVqlXKzs7WhAkTIrIMxBb1SwAAmCdpg2kwysvLVVdXp/LycnV0dGj79u2SpLPPPlsDBw7UDTfcoAkTJuhf/uVf9MQTT6i6ulo/+clPNHv2bONrRQEAABIFwVTSQw89pD/+8Y/Ovy+99FJJ0po1a3TttdeqT58+euedd/Td735XRUVFGjBggGbOnKnHHnssXkUGAABIOgRTSUuWLNGSJUt8TjNmzBi99957sSkQAABACkraq/IBAACQWAimAAAAMALBNIUxWhQAADAJwRQpifHIAQAwD8EUAAAARiCYAkCCowUAQLIgmAJAguP2wgCSBcEUAAAARiCYAkCCoykfQLIgmKYwB+1/AADAIARTpCQLVUwAABiHYAoAAAAjEEwBIMFZRAsAgORAMAWABOfgBsMAkgTBFAAAAEYgmAIAAMAIBFMASHD0MQWQLAimAAAAMALBFCmJ+iUAAMxDMAWABMf9IgAkC4IpAAAAjEAwBYAE52AYUwBJgmAKAAAAIxBMASDB0ccUQLIgmKYwmv8AAIBJCKZITdQwAQBgHIIpACQ4fmcBSBYEUwAAABiBYAoACY7u4gCSBcEUAAAARiCYAkCCo48pgGRBMAUAAIARCKYpzEHPNAAAYBCCKVKShcZPAACMQzAFgATHLUkBJAuCKQAAAIyQ8sH04MGDmjVrlsaOHat+/frprLPO0sMPP6zW1la36T777DNdc801ysrKUkFBgZ544ok4lRgAACA5pce7APG2Z88edXZ26vnnn9fZZ5+tnTt36p577lFTU5OefPJJSZLNZtMNN9ygqVOn6rnnntOOHTv0r//6rxo8eLC+853vxPkTAAAAJIeUD6Y33nijbrzxRuff48aNU2lpqRYuXOgMpq+88opaW1v1hz/8QRkZGbrgggu0fft2PfXUUwRTAACACEn5pnxPrFarcnNznX8XFxfrK1/5ijIyMpzPTZs2TaWlpTpx4oTHedjtdtlsNreHaRyMFgUAAAxCMO2hrKxMv/3tb3Xvvfc6n6uurlZeXp7bdN1/V1dXe5zP/PnzlZOT43wUFBREr9AIGlcxAwBgnqQNpg8++KAsFovPx549e9zeU1lZqRtvvFG333677rnnnrCWP3fuXFmtVuejoqIirPkBgDeMywsgWSRtH9Mf/OAHuuuuu3xOM27cOOf/q6qqdN111+mqq67S7373O7fp8vPzVVNT4/Zc99/5+fke552ZmanMzMwQSg4AAJCakjaYDhs2TMOGDQto2srKSl133XWaNGmSFi9erLQ094rkoqIi/fjHP1ZbW5v69u0rSVq1apXOO+88DRkyJOJlB4BgcHthAMkiaZvyA1VZWalrr71WhYWFevLJJ3X06FFVV1e79R399re/rYyMDM2aNUu7du3S66+/rt/85jeaM2dOHEsOAACQXJK2xjRQq1atUllZmcrKyjR69Gi31xynLlvPycnRypUrNXv2bE2aNElnnHGGHnroIYaKAmAE+pgCSBYpH0zvuusuv31RJWnixIlav3599AsEAACQolK+KT+V0SsNAACYhGCKlETDJ5IJ4/ICSBYEUwAAABiBYAoAAAAjEEwBAABgBIIpAAAAjEAwBQAAgBEIpims+wYCAAAAJiCYIiUxvA4AAOYhmAIAAMAIBFMAAAAYgWCawr5y7jBJ0uD+feNcktibMCIn3kUAIubcvEHxLgIARER6vAuA+PnRtPEaN2ygrh8/PN5FiZn3v3eNdhy26usX5ce7KEDY3rn/apVWN+iac86Id1EAICIsDi7NjgmbzaacnBxZrVZlZ2fHuzgAACAAnL9ji6Z8AAAAGIFgCgAAACMQTAEAAGAEgikAAACMQDAFAACAEQimAAAAMALBFAAAAEYgmAIAAMAIBFMAAAAYgWAKAAAAIxBMAQAAYASCKQAAAIxAMAUAAIAR0uNdgFThcDgkSTabLc4lAQAAgeo+b3efxxFdBNMYaWhokCQVFBTEuSQAACBYDQ0NysnJiXcxkp7FwU+AmOjs7FRVVZUGDRoki8US0XnPmzdPTzzxRETnCQBAIvrggw80adKkiM3P4XCooaFBI0eOVFoaPSCjjRrTGElLS9Po0aOjMu/MzMyozBcAgEQzcOBAZWdnR3Se1JTGDtEfAAAARiCYAgAAwAgE0yRw2WWXxbsIAADEXVZWlkaMGBHvYiAMXPwEAAAAI1BjCgAAACMQTAEAAGAEgikAAACMQDAFAACAEYIaYD/SdywCAABA8srMzFRlZaWGDh0a0PTUmAIAACAq7Ha7vvGNbwQ8fVDB1OFwuD3mzJkTdAEBAACQOrZt2xbwtGHVmG7dujWctwMAACDJ3XjjjQFPG/IA+3V1dQH3FwAAAEBqamxs1IABAwKaNuQaU0IpAAAA/JkyZUrA04ZUY8rV+QAAAAhUVVWVRowY4Xe6oGpMT548SSgFAABAUAKtBw2qxpRQCgAAgGAUFhbq0KFDAU3LOKYAAACImieffDLgaYO681OIF/ADAAAAflFjCgAAACMQTAEAAGAEgikAAACMQDAFAACAEQimAAAAMALBFAAAAEYgmAIAAMAIBFMAAAAYgWAKAAAAIxBMAQAAYASCKQAAAIxAMAUAAIAR/j8zTFv2a4W9BQAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "x = [r['date_added'] for r in recs]\n", + "y = [r['time_lost'] for r in recs]\n", + "plt.plot(x, y) \n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "6855a63f-7837-4901-8db1-f9d659a28101", + "metadata": {}, + "source": [ + "\n", + "## Raw Content Analysis" + ] + }, + { + "cell_type": "markdown", + "id": "1b9711f5-2f32-4531-bf41-b6083983ed98", + "metadata": {}, + "source": [ + "### Example of one record" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "id": "0720e2cc-d68c-4830-92c1-74ae5180f057", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'id': '0005d0e0-6de7-4857-b658-cfca7147374a',\n", + " 'site_id': 'summit',\n", + " 'message_text': 'The moveShutterMainDoor command Fail\\r\\n\\r\\n\\r\\nwe try the parameter with uppercase now\\r\\n\\r\\n\\r\\n open: False\\r\\n\\r\\n\\r\\n\\r\\n\\r\\nError in run\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 500, in wait_for\\r\\n return fut.result()\\r\\n ^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 239, in _get_next_ackcmd\\r\\n await self._next_ack_event.wait()\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/locks.py\", line 213, in wait\\r\\n await fut\\r\\nasyncio.exceptions.CancelledError\\r\\n\\r\\n\\r\\nThe above exception was the direct cause of the following exception:\\r\\n\\r\\n\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 189, in next_ackcmd\\r\\n ackcmd = await self._wait_task\\r\\n ^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 214, in _basic_next_ackcmd\\r\\n ackcmd = await asyncio.wait_for(\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 502, in wait_for\\r\\n raise exceptions.TimeoutError() from exc\\r\\nTimeoutError\\r\\n\\r\\n\\r\\nDuring handling of the above exception, another exception occurred:\\r\\n\\r\\n\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\\r\\n await self._run_task\\r\\n File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/run_command.py\", line 170, in run\\r\\n await getattr(self.remote, f\"cmd_{self.cmd}\").start(\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\\r\\n return await cmd_info.next_ackcmd(timeout=timeout)\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 205, in next_ackcmd\\r\\n raise base.AckTimeoutError(\\r\\nlsst.ts.salobj.base.AckTimeoutError: msg=\\'Timed out waiting for command acknowledgement\\', ackcmd=(ackcmd private_seqNum=1142844592, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result=\\'No command acknowledgement seen\\')\\r\\n',\n", + " 'level': 0,\n", + " 'tags': [],\n", + " 'urls': [],\n", + " 'time_lost': 0.0,\n", + " 'date_begin': '2024-03-20T04:15:26.769000',\n", + " 'user_id': 'cmorales@summit-lsp.lsst.codes',\n", + " 'user_agent': 'LOVE',\n", + " 'is_human': True,\n", + " 'is_valid': True,\n", + " 'date_added': '2024-03-20T04:16:06.976595',\n", + " 'date_invalidated': None,\n", + " 'parent_id': None,\n", + " 'systems': None,\n", + " 'subsystems': None,\n", + " 'cscs': None,\n", + " 'date_end': '2024-03-20T04:15:26.769000',\n", + " 'components': ['AuxTel'],\n", + " 'primary_software_components': ['None'],\n", + " 'primary_hardware_components': ['None'],\n", + " 'category': 'None',\n", + " 'time_lost_type': 'fault'}" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rec = recs[0]\n", + "rec" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "id": "82392f54-524e-4890-a2f8-ce0579a75e1c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The moveShutterMainDoor command Fail\n", + "\n", + "\n", + "we try the parameter with uppercase now\n", + "\n", + "\n", + " open: False\n", + "\n", + "\n", + "\n", + "\n", + "Error in run\n", + "Traceback (most recent call last):\n", + " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 500, in wait_for\n", + " return fut.result()\n", + " ^^^^^^^^^^^^\n", + " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 239, in _get_next_ackcmd\n", + " await self._next_ack_event.wait()\n", + " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/locks.py\", line 213, in wait\n", + " await fut\n", + "asyncio.exceptions.CancelledError\n", + "\n", + "\n", + "The above exception was the direct cause of the following exception:\n", + "\n", + "\n", + "Traceback (most recent call last):\n", + " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 189, in next_ackcmd\n", + " ackcmd = await self._wait_task\n", + " ^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 214, in _basic_next_ackcmd\n", + " ackcmd = await asyncio.wait_for(\n", + " ^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 502, in wait_for\n", + " raise exceptions.TimeoutError() from exc\n", + "TimeoutError\n", + "\n", + "\n", + "During handling of the above exception, another exception occurred:\n", + "\n", + "\n", + "Traceback (most recent call last):\n", + " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\n", + " await self._run_task\n", + " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/run_command.py\", line 170, in run\n", + " await getattr(self.remote, f\"cmd_{self.cmd}\").start(\n", + " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\n", + " return await cmd_info.next_ackcmd(timeout=timeout)\n", + " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", + " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 205, in next_ackcmd\n", + " raise base.AckTimeoutError(\n", + "lsst.ts.salobj.base.AckTimeoutError: msg='Timed out waiting for command acknowledgement', ackcmd=(ackcmd private_seqNum=1142844592, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result='No command acknowledgement seen')\n", + "\n" + ] + } + ], + "source": [ + "msg = rec[\"message_text\"]\n", + "print(msg)" + ] + }, + { + "cell_type": "markdown", + "id": "3a938226-c21e-4e52-be30-b11148e8e379", + "metadata": {}, + "source": [ + "\n", + "## Stakeholder Elicitation" + ] + }, { "cell_type": "code", "execution_count": null, - "id": "d8cefdd9-98ec-4383-a297-6853be7d0c9b", + "id": "138e5496-e058-417a-881e-8450e20af0f9", "metadata": {}, "outputs": [], "source": [] From d50bb6df0ea44af15473a052eeccce7d52ab3398 Mon Sep 17 00:00:00 2001 From: Steve Pothier Date: Tue, 27 Aug 2024 08:58:56 -0700 Subject: [PATCH 04/10] WIP --- notebooks_tsqr/logrep_proto_1.ipynb | 1059 ++------------------------- 1 file changed, 53 insertions(+), 1006 deletions(-) diff --git a/notebooks_tsqr/logrep_proto_1.ipynb b/notebooks_tsqr/logrep_proto_1.ipynb index d9b2ff2..0bd42a0 100644 --- a/notebooks_tsqr/logrep_proto_1.ipynb +++ b/notebooks_tsqr/logrep_proto_1.ipynb @@ -2,8 +2,8 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, - "id": "e5b65509-9611-4493-8916-77a4c37489f0", + "execution_count": null, + "id": "0", "metadata": {}, "outputs": [], "source": [ @@ -16,7 +16,7 @@ }, { "cell_type": "markdown", - "id": "09757010-6091-4769-b940-efd464c5eb68", + "id": "1", "metadata": {}, "source": [ "\n", @@ -25,8 +25,8 @@ }, { "cell_type": "code", - "execution_count": 2, - "id": "c2017a15-e054-4a4e-a522-28deff6d2a7a", + "execution_count": null, + "id": "2", "metadata": { "jupyter": { "source_hidden": true @@ -46,25 +46,14 @@ }, { "cell_type": "code", - "execution_count": 3, - "id": "2a310c90-7c5a-4cc2-a3b2-cc9d8291d245", + "execution_count": null, + "id": "3", "metadata": { "jupyter": { "source_hidden": true } }, - "outputs": [ - { - "data": { - "text/plain": [ - "'https://usdf-rsp-dev.slac.stanford.edu/narrativelog'" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "log = log_name\n", "limit = int(record_limit)\n", @@ -92,7 +81,7 @@ }, { "cell_type": "markdown", - "id": "e926c1a6-cd11-4cc1-9fb5-55a5f4d386c0", + "id": "4", "metadata": { "jp-MarkdownHeadingCollapsed": true }, @@ -103,8 +92,8 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "8fdfc281-17db-471a-873e-94e6b41c6f70", + "execution_count": null, + "id": "5", "metadata": { "jupyter": { "source_hidden": true @@ -135,10 +124,8 @@ }, { "cell_type": "markdown", - "id": "1695642e-4d47-4ae9-9311-c77a7312ad22", - "metadata": { - "jp-MarkdownHeadingCollapsed": true - }, + "id": "6", + "metadata": {}, "source": [ "\n", "## Get Records" @@ -146,20 +133,17 @@ }, { "cell_type": "code", - "execution_count": 5, - "id": "8481932e-6b4f-4eeb-98e2-18728c43351e", + "execution_count": null, + "id": "7", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Attempt to get logs from url='https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=either&limit=9999'\n", - "Retrieved 7673 records, each with numf=24 fields.\n" - ] - } - ], + "outputs": [], "source": [ + "# TODO Often fails on first request. Find out why!\n", + "try:\n", + " response = requests.get(url, timeout=timeout)\n", + "except:\n", + " pass \n", + " \n", "try:\n", " print(f'Attempt to get logs from {url=}')\n", " response = requests.get(url, timeout=timeout)\n", @@ -181,7 +165,7 @@ }, { "cell_type": "markdown", - "id": "6069eef2-0585-4e67-9527-42ec5dc12cbf", + "id": "8", "metadata": {}, "source": [ "\n", @@ -190,7 +174,7 @@ }, { "cell_type": "markdown", - "id": "97d5d456-e2b4-4dd6-b0c7-2d95870d8481", + "id": "9", "metadata": {}, "source": [ "### Fields names provided in records from log." @@ -198,175 +182,17 @@ }, { "cell_type": "code", - "execution_count": 6, - "id": "d7fedb0b-02fc-4843-8607-cf593ea3fb91", + "execution_count": null, + "id": "10", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
Field Name
0tags
1primary_software_components
2time_lost
3time_lost_type
4date_invalidated
5id
6site_id
7date_begin
8components
9date_added
10subsystems
11is_human
12user_id
13category
14cscs
15primary_hardware_components
16parent_id
17systems
18date_end
19message_text
20level
21is_valid
22urls
23user_agent
\n", - "
" - ], - "text/plain": [ - " Field Name\n", - "0 tags\n", - "1 primary_software_components\n", - "2 time_lost\n", - "3 time_lost_type\n", - "4 date_invalidated\n", - "5 id\n", - "6 site_id\n", - "7 date_begin\n", - "8 components\n", - "9 date_added\n", - "10 subsystems\n", - "11 is_human\n", - "12 user_id\n", - "13 category\n", - "14 cscs\n", - "15 primary_hardware_components\n", - "16 parent_id\n", - "17 systems\n", - "18 date_end\n", - "19 message_text\n", - "20 level\n", - "21 is_valid\n", - "22 urls\n", - "23 user_agent" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pd.DataFrame(flds, columns=['Field Name'])" ] }, { "cell_type": "markdown", - "id": "51c9a1db-4b38-4f35-bece-c4038ee5b9a5", + "id": "11", "metadata": {}, "source": [ "### Facets from log records.\n", @@ -375,148 +201,17 @@ }, { "cell_type": "code", - "execution_count": 7, - "id": "545abf79-808b-4716-b9e2-c5426a86b277", + "execution_count": null, + "id": "12", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
012
is_humanTrueNoneNone
site_idsummitNoneNone
categorySCIENCENoneENG
primary_software_componentsNoneNoneNone
level1000None
is_validTrueFalseNone
cscsNoneNoneNone
componentsNoneNoneNone
primary_hardware_componentsNoneNoneNone
user_agentLOVEstringNone
subsystemsNoneNoneNone
systemsNoneNoneNone
time_lost_typefaultNoneweather
\n", - "
" - ], - "text/plain": [ - " 0 1 2\n", - "is_human True None None\n", - "site_id summit None None\n", - "category SCIENCE None ENG\n", - "primary_software_components None None None\n", - "level 100 0 None\n", - "is_valid True False None\n", - "cscs None None None\n", - "components None None None\n", - "primary_hardware_components None None None\n", - "user_agent LOVE string None\n", - "subsystems None None None\n", - "systems None None None\n", - "time_lost_type fault None weather" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "pd.DataFrame.from_dict(facets, orient='index')" ] }, { "cell_type": "markdown", - "id": "21afc757-8890-4b96-afc9-e5086327df55", + "id": "13", "metadata": {}, "source": [ "### Table of selected log record fields.\n", @@ -525,137 +220,10 @@ }, { "cell_type": "code", - "execution_count": 19, - "id": "1c40de6e-91c3-4d9d-972c-efe15d2b4a17", + "execution_count": null, + "id": "14", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Table available as CSV file: \n" - ] - }, - { - "data": { - "text/html": [ - "tl.csv
" - ], - "text/plain": [ - "/home/pothiers/sandbox/ts_logging_and_reporting/notebooks_tsqr/tl.csv" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
date_addedtime_lost
02024-03-20T04:16:06.9765950.0
12024-04-24T20:54:19.3466460.0
22024-06-19T01:59:58.7013430.0
32024-08-21T04:22:26.2326530.0
42023-11-21T19:55:36.7871050.0
.........
76682024-04-19T19:07:32.7557340.0
76692024-03-21T01:04:28.8834940.0
76702024-06-12T01:56:12.8967180.0
76712024-03-04T03:20:00.1513480.0
76722024-01-03T22:42:08.4630060.0
\n", - "

7673 rows × 2 columns

\n", - "
" - ], - "text/plain": [ - " date_added time_lost\n", - "0 2024-03-20T04:16:06.976595 0.0\n", - "1 2024-04-24T20:54:19.346646 0.0\n", - "2 2024-06-19T01:59:58.701343 0.0\n", - "3 2024-08-21T04:22:26.232653 0.0\n", - "4 2023-11-21T19:55:36.787105 0.0\n", - "... ... ...\n", - "7668 2024-04-19T19:07:32.755734 0.0\n", - "7669 2024-03-21T01:04:28.883494 0.0\n", - "7670 2024-06-12T01:56:12.896718 0.0\n", - "7671 2024-03-04T03:20:00.151348 0.0\n", - "7672 2024-01-03T22:42:08.463006 0.0\n", - "\n", - "[7673 rows x 2 columns]" - ] - }, - "execution_count": 19, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "cols = ['date_added', 'time_lost']\n", "df = pd.DataFrame(recs)[cols]\n", @@ -671,424 +239,10 @@ }, { "cell_type": "code", - "execution_count": 13, - "id": "f6441d81-93a8-4369-93fb-55d3715783cc", + "execution_count": null, + "id": "15", "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
idsite_idmessage_textleveltagsurlstime_lostdate_beginuser_iduser_agent...parent_idsystemssubsystemscscsdate_endcomponentsprimary_software_componentsprimary_hardware_componentscategorytime_lost_type
00005d0e0-6de7-4857-b658-cfca7147374asummitThe moveShutterMainDoor command Fail\\r\\n\\r\\n\\r...0[][]0.02024-03-20T04:15:26.769000cmorales@summit-lsp.lsst.codesLOVE...NoneNoneNoneNone2024-03-20T04:15:26.769000[AuxTel][None][None]Nonefault
100067b6a-1741-41f3-87af-57261dff30acsummitDavid updated environment. But no need to cycl...0[][]0.02024-04-24T20:53:41.843000ykang@summit-lsp.lsst.codesLOVE...NoneNoneNoneNone2024-04-24T20:53:41.843000[MainTel][None][None]Nonefault
2000a3ba8-b1a9-4b04-9c4a-263da3dac1d1summitRuning BLOCK-T22 AuxTel filter Scans, the auxt...0[][]0.02024-06-19T01:59:08.706000cmd_user@summit-lsp.lsst.codesLOVE...af0157e3-d7f0-4068-9a8a-e51a0a77d260NoneNoneNone2024-06-19T01:59:08.706000[AuxTel][None][None]Nonefault
30010e246-8039-4741-b4c6-54047e9464c5summitStill checking updates in the wavefront estima...0[][]0.02024-08-21T04:04:38.053000kpena@summit-lsp.lsst.codesLOVE...99de2a12-20eb-45ae-b189-a793b63e6ab5NoneNoneNone2024-08-21T04:04:38.053000[AuxTel][None][None]Nonefault
40017b90e-26aa-4a7c-8186-fc06ac9b5aa6summitThe m1m3 ump test failed for actuator 108. \\r\\...0[][]0.02023-11-21T19:51:54.868000bquint@love01.cp.lsst.orgLOVE...NoneNoneNoneNone2023-11-21T19:51:54.868000[MainTel][None][None]Nonefault
..................................................................
7668ffbd086c-cf20-4c45-aef5-b2783d348192summitScript Failed salIndex: 101647 with move_rotat...0[][]0.02024-04-19T19:06:54.594000ykang@summit-lsp.lsst.codesLOVE...NoneNoneNoneNone2024-04-19T19:06:54.594000[MainTel][None][None]Nonefault
7669ffeaada0-3b08-4de2-b48b-9acf2623fb7esummitsalIndex:&nbsp;101588\\r\\n \\r\\nError in run\\r\\n...0[][]0.02024-03-21T01:03:51.861000pvenegas@love01.cp.lsst.orgLOVE...NoneNoneNoneNone2024-03-21T01:03:51.861000[MainTel][None][None]Nonefault
7670ffebf6c9-156c-4309-9662-f5697907cc47summith3. Execution of power_on_atcalsys.py \\r\\n0[][]0.02024-06-12T01:55:35.836000pvenegas@summit-lsp.lsst.codesLOVE...NoneNoneNoneNone2024-06-12T01:55:35.836000[AuxTel][None][None]Nonefault
7671fff11764-aae1-4428-aea0-b72fdb3dafa6summitUpdated pointing model based on March 2 data. ...0[][]0.02024-03-04T03:19:22.793000bstalder@summit-lsp.lsst.codesLOVE...NoneNoneNoneNone2024-03-04T03:19:22.793000[MainTel][None][None]Nonefault
7672fff14a30-c341-4561-a2ea-0f043d19f526summitM1M3 faulted HP 2 excessive forces\\r\\n0[][]0.02024-01-03T19:45:18.279000bstalder@love01.cp.lsst.orgLOVE...NoneNoneNoneNone2024-01-03T19:45:18.279000[MainTel][None][None]Nonefault
\n", - "

7673 rows × 24 columns

\n", - "
" - ], - "text/plain": [ - " id site_id \\\n", - "0 0005d0e0-6de7-4857-b658-cfca7147374a summit \n", - "1 00067b6a-1741-41f3-87af-57261dff30ac summit \n", - "2 000a3ba8-b1a9-4b04-9c4a-263da3dac1d1 summit \n", - "3 0010e246-8039-4741-b4c6-54047e9464c5 summit \n", - "4 0017b90e-26aa-4a7c-8186-fc06ac9b5aa6 summit \n", - "... ... ... \n", - "7668 ffbd086c-cf20-4c45-aef5-b2783d348192 summit \n", - "7669 ffeaada0-3b08-4de2-b48b-9acf2623fb7e summit \n", - "7670 ffebf6c9-156c-4309-9662-f5697907cc47 summit \n", - "7671 fff11764-aae1-4428-aea0-b72fdb3dafa6 summit \n", - "7672 fff14a30-c341-4561-a2ea-0f043d19f526 summit \n", - "\n", - " message_text level tags urls \\\n", - "0 The moveShutterMainDoor command Fail\\r\\n\\r\\n\\r... 0 [] [] \n", - "1 David updated environment. But no need to cycl... 0 [] [] \n", - "2 Runing BLOCK-T22 AuxTel filter Scans, the auxt... 0 [] [] \n", - "3 Still checking updates in the wavefront estima... 0 [] [] \n", - "4 The m1m3 ump test failed for actuator 108. \\r\\... 0 [] [] \n", - "... ... ... ... ... \n", - "7668 Script Failed salIndex: 101647 with move_rotat... 0 [] [] \n", - "7669 salIndex: 101588\\r\\n \\r\\nError in run\\r\\n... 0 [] [] \n", - "7670 h3. Execution of power_on_atcalsys.py \\r\\n 0 [] [] \n", - "7671 Updated pointing model based on March 2 data. ... 0 [] [] \n", - "7672 M1M3 faulted HP 2 excessive forces\\r\\n 0 [] [] \n", - "\n", - " time_lost date_begin user_id \\\n", - "0 0.0 2024-03-20T04:15:26.769000 cmorales@summit-lsp.lsst.codes \n", - "1 0.0 2024-04-24T20:53:41.843000 ykang@summit-lsp.lsst.codes \n", - "2 0.0 2024-06-19T01:59:08.706000 cmd_user@summit-lsp.lsst.codes \n", - "3 0.0 2024-08-21T04:04:38.053000 kpena@summit-lsp.lsst.codes \n", - "4 0.0 2023-11-21T19:51:54.868000 bquint@love01.cp.lsst.org \n", - "... ... ... ... \n", - "7668 0.0 2024-04-19T19:06:54.594000 ykang@summit-lsp.lsst.codes \n", - "7669 0.0 2024-03-21T01:03:51.861000 pvenegas@love01.cp.lsst.org \n", - "7670 0.0 2024-06-12T01:55:35.836000 pvenegas@summit-lsp.lsst.codes \n", - "7671 0.0 2024-03-04T03:19:22.793000 bstalder@summit-lsp.lsst.codes \n", - "7672 0.0 2024-01-03T19:45:18.279000 bstalder@love01.cp.lsst.org \n", - "\n", - " user_agent ... parent_id systems \\\n", - "0 LOVE ... None None \n", - "1 LOVE ... None None \n", - "2 LOVE ... af0157e3-d7f0-4068-9a8a-e51a0a77d260 None \n", - "3 LOVE ... 99de2a12-20eb-45ae-b189-a793b63e6ab5 None \n", - "4 LOVE ... None None \n", - "... ... ... ... ... \n", - "7668 LOVE ... None None \n", - "7669 LOVE ... None None \n", - "7670 LOVE ... None None \n", - "7671 LOVE ... None None \n", - "7672 LOVE ... None None \n", - "\n", - " subsystems cscs date_end components \\\n", - "0 None None 2024-03-20T04:15:26.769000 [AuxTel] \n", - "1 None None 2024-04-24T20:53:41.843000 [MainTel] \n", - "2 None None 2024-06-19T01:59:08.706000 [AuxTel] \n", - "3 None None 2024-08-21T04:04:38.053000 [AuxTel] \n", - "4 None None 2023-11-21T19:51:54.868000 [MainTel] \n", - "... ... ... ... ... \n", - "7668 None None 2024-04-19T19:06:54.594000 [MainTel] \n", - "7669 None None 2024-03-21T01:03:51.861000 [MainTel] \n", - "7670 None None 2024-06-12T01:55:35.836000 [AuxTel] \n", - "7671 None None 2024-03-04T03:19:22.793000 [MainTel] \n", - "7672 None None 2024-01-03T19:45:18.279000 [MainTel] \n", - "\n", - " primary_software_components primary_hardware_components category \\\n", - "0 [None] [None] None \n", - "1 [None] [None] None \n", - "2 [None] [None] None \n", - "3 [None] [None] None \n", - "4 [None] [None] None \n", - "... ... ... ... \n", - "7668 [None] [None] None \n", - "7669 [None] [None] None \n", - "7670 [None] [None] None \n", - "7671 [None] [None] None \n", - "7672 [None] [None] None \n", - "\n", - " time_lost_type \n", - "0 fault \n", - "1 fault \n", - "2 fault \n", - "3 fault \n", - "4 fault \n", - "... ... \n", - "7668 fault \n", - "7669 fault \n", - "7670 fault \n", - "7671 fault \n", - "7672 fault \n", - "\n", - "[7673 rows x 24 columns]" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "df = pd.DataFrame(recs)\n", "df" @@ -1096,7 +250,7 @@ }, { "cell_type": "markdown", - "id": "bd56e1ab-ff5f-4416-9f3b-c690d5cc3adf", + "id": "16", "metadata": {}, "source": [ "\n", @@ -1105,21 +259,10 @@ }, { "cell_type": "code", - "execution_count": 11, - "id": "a74d281f-952d-42d3-bf35-949a17000b49", + "execution_count": null, + "id": "17", "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqYAAAGdCAYAAADALrFsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA7NUlEQVR4nO3de3xU5YH/8e+EkIRbAkFIuCQI3hAVtXghVltFKlLX1ZW63dZd0WWt20W3lXa78mvrraW41mprF9FaCvWCWtqq641LERAh4aYoF4mEW0JCEiBkJhcyuc3vj5BhJpn79ZmZz/v1mhdk5sw5z5xz5pzvPM9znmNxOBwOAQAAAHGWFu8CAAAAABLBFAAAAIYgmAIAAMAIBFMAAAAYgWAKAAAAIxBMAQAAYASCKQAAAIxAMAUAAIAR0uNdgFTR2dmpqqoqDRo0SBaLJd7FAQAAAXA4HGpoaNDIkSOVlkZ9XrQRTGOkqqpKBQUF8S4GAAAIQUVFhUaPHh3vYiQ9gmmMDBo0SFLXjp2dnR3n0gAAgEDYbDYVFBQ4z+OILoJpjHQ332dnZxNMAQBIMHTDiw06S0h65JFHZLFY3B7jx493vt7S0qLZs2dr6NChGjhwoGbMmKGampo4lhgAACD5EExPueCCC3TkyBHn46OPPnK+9sADD+jtt9/WsmXLtG7dOlVVVem2226LY2kBAACSD035p6Snpys/P7/X81arVYsWLdLSpUs1ZcoUSdLixYt1/vnnq6SkRJMnT451UQEAAJISNaan7N27VyNHjtS4ceN0xx13qLy8XJK0bds2tbW1aerUqc5px48fr8LCQhUXF3udn91ul81mc3sAAADAO4KppCuvvFJLlizR8uXLtXDhQh04cEDXXHONGhoaVF1drYyMDA0ePNjtPXl5eaqurvY6z/nz5ysnJ8f5YKgoAAAA32jKlzR9+nTn/ydOnKgrr7xSY8aM0Z/+9Cf169cvpHnOnTtXc+bMcf7dPdwEAAAAPKPG1IPBgwfr3HPPVVlZmfLz89Xa2qr6+nq3aWpqajz2Se2WmZnpHBqKIaIAAAD8I5h60NjYqH379mnEiBGaNGmS+vbtq9WrVztfLy0tVXl5uYqKiuJYSgAAgORCU76kH/7wh7r55ps1ZswYVVVV6eGHH1afPn30rW99Szk5OZo1a5bmzJmj3NxcZWdn6/7771dRURFX5AMAAEQQwVTS4cOH9a1vfUvHjx/XsGHDdPXVV6ukpETDhg2TJD399NNKS0vTjBkzZLfbNW3aND377LNxLjUAAEBysTgcDke8C5EKbDabcnJyZLVa6W8KAECC4PwdW/QxBQAAgBEIpgAS3sZ9x/SnLRXxLgYAIEz0MQWQ8L79wiZJ0vkjsnXR6Jw4lwYAECpqTAEkjcr6k/EuAgAgDARTAAAAGIFgCgAAACMQTAEAAGAEgikAAACMQDAFAACAEQimAJIIN7IDgERGMAUAAIARCKYAAAAwAsEUAAAARiCYAgAAwAgEUwAAABiBYAoAAAAjEEwBAABgBIIpAAAAjEAwBQAAgBEIpgAAADACwRQAAABGIJgCAADACARTAAAAGIFgCgAAACMQTAEkDYcj3iUAAISDYAoAAAAjEEwBAABgBIIpAAAAjEAwBQAAgBEIpgAAADACwRQAAABGIJgCAADACARTAAAAGIFgCgAAACMQTAEAAGAEgikAAACMQDD14PHHH5fFYtH3v/9953MtLS2aPXu2hg4dqoEDB2rGjBmqqamJXyEBAACSDMG0hy1btuj555/XxIkT3Z5/4IEH9Pbbb2vZsmVat26dqqqqdNttt8WplAAAAMmHYOqisbFRd9xxh1544QUNGTLE+bzVatWiRYv01FNPacqUKZo0aZIWL16sjRs3qqSkJI4lBgAASB4EUxezZ8/WTTfdpKlTp7o9v23bNrW1tbk9P378eBUWFqq4uNjjvOx2u2w2m9sDQHQ54l0AAEBY0uNdAFO89tpr+vjjj7Vly5Zer1VXVysjI0ODBw92ez4vL0/V1dUe5zd//nw9+uij0SgqAABAUqLGVFJFRYW+973v6ZVXXlFWVlZE5jl37lxZrVbno6KiIiLzBQAASFYEU3U11dfW1upLX/qS0tPTlZ6ernXr1umZZ55Renq68vLy1Nraqvr6erf31dTUKD8/3+M8MzMzlZ2d7fYAAACAdzTlS7r++uu1Y8cOt+fuvvtujR8/Xv/93/+tgoIC9e3bV6tXr9aMGTMkSaWlpSovL1dRUVE8igwAAJB0CKaSBg0apAsvvNDtuQEDBmjo0KHO52fNmqU5c+YoNzdX2dnZuv/++1VUVKTJkyfHo8gAAABJh2AaoKefflppaWmaMWOG7Ha7pk2bpmeffTbexQIAAEgaBFMv1q5d6/Z3VlaWFixYoAULFsSnQAAAAEmOi58AAABgBIIpAAAAjEAwBQAAgBEIpgAAADACwRQAAABGIJgCAADACARTAAAAGIFgCiBpOBzxLgEAIBwEUwAAABiBYAoAAAAjEEwBAABgBIIpAAAAjEAwBQAAgBEIpgAAADACwRRA0nCI8aIAIJERTAEAAGAEgimApGGRJd5FAACEgWAKAAAAIxBMAQAAYASCKQAAAIxAMAUAAIARCKYAAAAwAsEUQNJgHFMASGwEUwAAABiBYAoAAAAjEEwBAABgBIIpAAAAjEAwBQAAgBEIpgAAADACwRRA0nAwWhQAJDSCKQAAAIxAMAWQNCyWeJcAABAOgikAAACMQDAFAACAEQimAAAAMALBVNLChQs1ceJEZWdnKzs7W0VFRXr//fedr7e0tGj27NkaOnSoBg4cqBkzZqimpiaOJQYAAEg+BFNJo0eP1uOPP65t27Zp69atmjJlim655Rbt2rVLkvTAAw/o7bff1rJly7Ru3TpVVVXptttui3OpAQAAkkt6vAtggptvvtnt73nz5mnhwoUqKSnR6NGjtWjRIi1dulRTpkyRJC1evFjnn3++SkpKNHny5HgUGYAHjGMKAImNGtMeOjo69Nprr6mpqUlFRUXatm2b2traNHXqVOc048ePV2FhoYqLi+NYUgAAgORCjekpO3bsUFFRkVpaWjRw4EC98cYbmjBhgrZv366MjAwNHjzYbfq8vDxVV1d7nZ/dbpfdbnf+bbPZolV0AACApECN6SnnnXeetm/frk2bNum73/2uZs6cqd27d4c8v/nz5ysnJ8f5KCgoiGBpAQAAkg/B9JSMjAydffbZmjRpkubPn6+LL75Yv/nNb5Sfn6/W1lbV19e7TV9TU6P8/Hyv85s7d66sVqvzUVFREeVPAAAAkNgIpl50dnbKbrdr0qRJ6tu3r1avXu18rbS0VOXl5SoqKvL6/szMTOfwU90PAAAAeEcfU3XVbk6fPl2FhYVqaGjQ0qVLtXbtWq1YsUI5OTmaNWuW5syZo9zcXGVnZ+v+++9XUVERV+QDAABEEMFUUm1tre68804dOXJEOTk5mjhxolasWKGvfe1rkqSnn35aaWlpmjFjhux2u6ZNm6Znn302zqUG0BOjRQFAYiOYSlq0aJHP17OysrRgwQItWLAgRiUCAABIPfQxBZA0LPEuAAAgLARTAAAAGIFgCgAAACMQTAEAAGAEgikAAACMQDAFAACAEQimAJIG45gCQGIjmAIAAMAIBFMAAAAYgWAKAAAAIxBMAQAAYASCKQAAAIxAMAUAAIARCKYAkobDwYBRAJDICKYAAAAwAsEUQNKwWCzxLgIAIAwEUwAAABiBYAoAAAAjEEwBAABgBIIpAAAAjEAwTQGN9nYtWFOm3VU2LdlwQIdPNAf1/sMnmrVkwwE1t7ZHqYTx0dHp0Mslh1Ra3RDQ9O98VqWFa/fprx8fDngZ1pNtWrzhgGobWrxOs3xntdaU1uqlkkMqq230OM2GsmN6f8eRXs9vO1SnW/73I+076vl97+04oo1lxwIuryTtqrLq1c3lemt7pZbv7L1Mk0V7uKjahhYtWFOmWtvp7Vlra9HiDQdka2kLaB4rdlXrwy+OenytvaPz1H7gfZ98a3ulNh+oC67gHnx+xKalm8rV2Rn4Otu0/7j+79OqsJcdS8t3HtGGIL8Drjo6HXplk/fjRGX9SS3ZcEBN9sCOj3trGvRSySF1uKz37u2+t8Z9GW9/WqVN+4+HXHZvPj9i03Pr9sne3uH2fEenQy98uF+fHa4Ped4NLV3HvBqb92OeNwePNenF4oO9yoXUkh7vAiD65r37uV7dXK5friiVJD258gvtfHRawO//+m/Wy9bSrv3HmvTYLRdGq5gx96etFfrJmzslSQcfv8nntPuPNuq+pZ84/x4ztL8mjcn1u4wfLvtUq3bX6NXN5Vr5wFd7vV5ja9G/v7zN7TlPZbnj95skSRsenKJRg/s5n5+xsFiSdP2v1vV6X0Vds/7jlY+9ztObm575yO3vL34+XRnp/IaVpHte3KZPK+r13o4jevc/r5EkfeuFEu072qTNB+q08J8n+Xz/sUa77n2pa3sfmP/1XqMIvFxySI+8vVuS5222p9qm77223evrwZj+m/WSpAGZfXTLJaMCes83f1ciSTo/f5DOyRsU1vJj4Yj1pP795eC/A67+su2wfvyG9+PEzb/9SHVNrfqitlG/+IeL/M7va09/KEmySPrnyWMkSa9uqdBPexyLymobdP+rn4RVdm+6t317R6fum3KO8/llWys0773Pw1rm3L/u0DufHdHiDQf14Y+uC+q91z65VpJU39ym/7z+HN8TI2lxtkkBWw+61640BvjLvputpWv6cGodTPTZYWvA0x6xuv/6P3AssFrnVbtrJElf1Hiu0Tze2BpwGSTpWIM94Glrg5jWl04GrXf6tKJekrSryuZ8bt/RJknSB3tq/b6/vtn39v7k1Py9OVx30u8ygrX7iM3/RD1UWYOvDYuHYw3Bfb882VHp+zhR19S1jI/2Bnd8dK2V/NTDdq+sj/463lnpvu33BNh65MuaU9+D8rrgWuZcbTkYfosAEhfBFAAAAEYgmAIAAMAIBFMEjAbd0+J1T/Z4LJWW/MAEu5qMWa8hlCNe+7/JHEGuSBNWYbBlBmKBYJoCuEujZ8Gsl1BXob9lRHPbsN1N5Huj+NtkbNPYi8U697SIeGxq9i+YgGAKAAAAIxBMASAOaEQFgN4IpgAAADACwRQAAABGIJgCAADACARTBI5OcU7xWhXxGKaHIWUCFORqMmXIpVBKYUbJzRLs5jRhHRqyC/ZiarkQGwTTFGCJy8Aj5gtqrYS4CuM5/A9b3Tz+hw/zM5wUGzXmYrHKPW3XeGxrzhUwAcEUAAAARiCYAgAQJpqfgcggmEqaP3++Lr/8cg0aNEjDhw/XrbfeqtLSUrdpWlpaNHv2bA0dOlQDBw7UjBkzVFNTE6cSA0h05BgA6I1gKmndunWaPXu2SkpKtGrVKrW1temGG25QU1OTc5oHHnhAb7/9tpYtW6Z169apqqpKt912WxxLDQAwBf1/gchIj3cBTLB8+XK3v5csWaLhw4dr27Zt+spXviKr1apFixZp6dKlmjJliiRp8eLFOv/881VSUqLJkyfHo9gAAABJhRpTD6xWqyQpNzdXkrRt2za1tbVp6tSpzmnGjx+vwsJCFRcXe5yH3W6XzWZzeyQ6mh5dpNDKoO9cYBhWK7XxPQEig2DaQ2dnp77//e/ry1/+si688EJJUnV1tTIyMjR48GC3afPy8lRdXe1xPvPnz1dOTo7zUVBQEO2iw0D+hv8JFue+5GFKkAlpPFVDyp7ITNj+BhTBI37kpTaCaQ+zZ8/Wzp079dprr4U1n7lz58pqtTofFRUVESph8Oj75Fkw6yVa4/tFc9zASIdihM/vuLZ+X2ebxlosvkeetms8tjWHDJiAPqYu7rvvPr3zzjv68MMPNXr0aOfz+fn5am1tVX19vVutaU1NjfLz8z3OKzMzU5mZmdEuMgAAQNKgxlRdTVn33Xef3njjDX3wwQcaO3as2+uTJk1S3759tXr1audzpaWlKi8vV1FRUayLCyAJ0FwJAL1RY6qu5vulS5fqrbfe0qBBg5z9RnNyctSvXz/l5ORo1qxZmjNnjnJzc5Wdna37779fRUVFXJEPAAAQIQRTSQsXLpQkXXvttW7PL168WHfddZck6emnn1ZaWppmzJghu92uadOm6dlnn41xSQEAAJIXwVSBXZWalZWlBQsWaMGCBTEoEQAAQOqhjykCFtKwMkkq0P6Bkb7INR6bgK0emGC3jSlfp9BGizKk8AmMdeidKd8NxAfBFCkrmOFYojWMSjSHZ2HkF/P4HXoo3PGkkJA87RaxGLqpZwBk94IJCKYAAAAwAsEUSCAMgO0bTYCIF7o6deHGHggXwRRIIKGe+zhpAogFjjUIF8EUAIAwUVMIRAbBFAAAAEYgmCJgNNCclkqtVTTNBYa1lNr4ngCRQTAFoijyrXuhnfw4Z5rHlG0SSjFMKXtCM2IdGlGIXti/UhvBNAXQ98mzYFZLtNZgNLcMm908/ocp9T0FmzT2YvE98jiOafQX23uZ7GAwAMEUSCicOQATUcnXhYoQhItgChjOve9aiE354Sw/jPfCO25JGVv0AY0N1jPCRTAFACBM1BMCkUEwBQAAgBEIpgAAhIkGbCAyCKYIGF2HTgt0Vfi7yjro5cbhlqRs98AEu45NWa+hlMOUsicyVqF39L9ObQTTFEDfJ8+CWS/RutI0mhewRjoUI3z+trf/19mmsRab71F8xovq+QOD/QsmIJgCCYTzBmAmapG7EG4RLoJpCojU8ZLmldMCPglF4BgdcvN9pLZXAm32eO6jgSzZ37b0/3oCbQwkHPYvmIBgCkRThI/zoYdUmIZtEltkrtgg3CJcBNMUEKmGFfosnpYIrVVsr9gKZG3TxzR5JcOmYf+CCQimAACEiYpCIDIIpggYfUxPi2Uf03hjuweGtQQA4SOYAimA2hzzmNIXL5QfHmaUPLGZsv1NxKpJbQTTFEC3Ic+C6U8VvXUYvY3DdjePv36//jYZmzT2YvE98rSMWPQR75n/IrFE+qkiXARTAEmDmhbEC11egMggmAKGc3j5f3Dz4JakpmG1xhbrOzboooBwEUwBJA1aEREviTg8W+KVGKmAYAoAQJgSsSk/8UqMVEAwRcBooTkt0JNQMtRIsNkDw/cD0ULzOFIJwRRIAZzXzGPKNgmlHASl8LEGvWPdpDaCaQqg3134orUK2TapJfxbkkauLAhMLFa5p2XEYlv3+oERgWUyXBTCRTAFACBMVCIDkUEwTQGROmBy4D0t0HURicqDeDebxnv5wTC9qP7KF+7rSE4x2+7sXzAAwRSIokifUAgmSYRtGVOJ9AMrkbGeES6CaQqIVJcfug6dxrpAKOhjCqOxf8EABFNJH374oW6++WaNHDlSFotFb775ptvrDodDDz30kEaMGKF+/fpp6tSp2rt3b3wKi4RCkAAAIHAEU0lNTU26+OKLtWDBAo+vP/HEE3rmmWf03HPPadOmTRowYICmTZumlpaWGJc0vmihOS3R1kU45U2wj5owEnFA9m6JW3JzBPOdTLn1nXIfGK7S410AE0yfPl3Tp0/3+JrD4dCvf/1r/eQnP9Ett9wiSXrxxReVl5enN998U//0T/8Uy6KGJBFvlRcLwdRmRqvmM5pbhtraxOPvu8p3OfZi8T3ytIx4bOlI7F8MF4VwUWPqx4EDB1RdXa2pU6c6n8vJydGVV16p4uJir++z2+2y2WxuDwBAckrESr5ELDOSH8HUj+rqaklSXl6e2/N5eXnO1zyZP3++cnJynI+CgoKolhNmooYrthKtiwUAwB3BNErmzp0rq9XqfFRUVMS7SIiDSPQjdJ1DqEOxhFMOwl50hLteGZYnOIm8tjxt622H6nS0wR6H0vjGfolw0cfUj/z8fElSTU2NRowY4Xy+pqZGl1xyidf3ZWZmKjMzM9rFA+CC7m2Il1juehv3HdO3X9ikNIu0f/5NIc+HrwtMRI2pH2PHjlV+fr5Wr17tfM5ms2nTpk0qKiqKY8mQCqrqT1JjCSSAWH5N1+89JknqDLfW3c/rDS1t+v36/TpiPRnegoAgUGMqqbGxUWVlZc6/Dxw4oO3btys3N1eFhYX6/ve/r5///Oc655xzNHbsWP30pz/VyJEjdeutt8av0IirQM8H4fQxfeHD/Zr33ue655qxIc8jEhJ5WKNk5nBQQ5wq4vUN/MmbO/XW9iot+uiAiudeH6dSINUQTCVt3bpV1113nfPvOXPmSJJmzpypJUuW6Ec/+pGampr0ne98R/X19br66qu1fPlyZWVlxavISAHz3vtckvTC+gNhz4taV/OYsklC6RPI/hS+cFZhrH6PrPviqCTpiDXwMbsjMVwUP4ZTG8FU0rXXXuvz4GyxWPTYY4/psccei2GpIodaFc+Cq82MzkqM5ph/jAhgHm5JmngC/Y6GE9Y9fVfjMR5oz0WyuyEe6GMKRBFBAoCpqPmGiQimKSBSBx+GAXGRAOsiYs1hHmbz0Fs7NedP243bJwwrTi/+yhfM64Z/1KB1djp070tb9csVe+JdFOP03C+i9YPX9O8PUgPBFIiiSB/oQ51dJIvR2t6pF4sP6a8fV+rwCa7WDZVpoT7eNh2o04pdNVqwZl9U5s/qDl4o3QnYrxEugmkKiNSva+6B7CIB1kW0+pi61sS2dXRGZRnJij6m3rWyLwUsUt/tXn1Kk3j/QuIgmAJRlIwHei6qQjSwV8UelZswEcEUAaOJxkWCrYtwtp2vdybWWjBLuOsuUt/HUG9yi/AEs/3iNXxSKD8WIjJcFLtXSiOYpgBqIjwL5vgZrZrPaG6bqJWZHSpq/Dblx6YYcWHqfhV4sUJPU54+u6/1kYjHIyBQBFMgipLxQE9tBpAc+CrDRATTFBCpgw8HsdMSYV1Ebpiw6C8jUgwrTi9hDxfl5f9IbrH6nvVcjKm12EhuBFMgiiJ9Pgn1BBXJcnC7wMgwLdTHW7QvqmO/jQ2uRUC4CKYpIFKHe348n5YI6yI2tR1mnYQSYbv4ksp9TBF7Pfcn9i+YgGAKRBEHeiAwNBsHLlKryv/PSjYKYo9gioCZVTcWX/FaF/E4efdsAqWlzgxsB0RbKMcbbsSCcBFMgQQSch/TKIUYwlHoTOnzGMo2ZLuHj1XoHesmtRFMUwG/YD0KZq1Eaw1Gc9Ow2RNR6nYyNfajBViwcMK6p0X4XGyUvtwcM2ACgimM9edth1Wy/3i8ixGWcJq1EuEkQc0GEJp3PzsS7yL4vYI+AQ5BSEIE01QQqVsXxjCF7Ky06ofLPtU//a4kdgsNQiI0ZcZiHFPTJFBRvQh8IFNTugJETAqmoC0H6wKaLmbjmCbZLoXERDCFkQ6fOOnxeXt7h/YfbYxxaSKnoq5ZjfZ259/BnghCDiMRPOF4K3Nnp0P1za2RW1CyS5AQUFl/UraWtrDm4XA49MsVe/TejvBqCeubW1VtbQmxDGEtOioOHmsK6X0mZ3jGMUW4CKapIEJtwrFsWva2rNufK9aUX63T2tLa2BXGg1DWxYFjTbrmiTW6Yt7fAltG8Itwf38MtpfrOWjWH7foksdWaWelNfoLTlpm9TGtrD+pLz/+gS55dGVY81lbelQL1uzTf7zycVjzueSxVZo8f7WszeEFZVPEO8L17GqUCN2HkPwIpghYLH8Iezs+fna4K/Qs23o4doXxIJR18VHZMUlSc2tHyMuN9t1xPAn0o64pPSpJeqn4UPQKA6dYfB+3HOhqau4Mc1m1Df5rOYPZt8uONoRTnIQTiW4b1dYWdfbYkN21mzW2FnV42MgMF4V4IJgiISVi/7pIHK5D/dyRXF+u8/I030TcNvFgylrytb28ZYxgyx7sDyp/zcFJ01oc4ucINvut3FWtyfNX6/5XP+n1WvG+47ryF6t11+LNoRUmCugOkNoIpikgEX+/xuJXdzCLiER5gp1FuMuMR+0qwuP/lqQJuk0DKLapFW2BrvNQolT3jwJP33Vf6yPY/WDB2n2SpHc99PH948aDkqT1e48l7v6FpEIwhZH8HR4T5Qd1Mh7mE2Xd99TW0aknV5Qm/BBkiSjY74G/fSxBd8G4cV3/PS9ki2QA7ungsSb94r3PVWsL7YI1pCaCaQqI1EHcpCbaeIejUJqaYl0bEant5euzenopntvGV1lfKTmk/11TFtchyPwGLr+BLDord/3eo3pu3b5e6y9SLRfBzsffp4z39z9SAv0c4X5e19X/+Pt7vL4W6f1rxsKN+t2H+3Xf0t5dCABv0uNdAMATf+cxk0KyL66l9PSZgv4Uht2S1OOyXP7//o4jysroo+vOGx67Anhx8HhzvIvgxqRw9S+LuvoXnps3UFPG5zmfDyWW7j/aqLWlR/XtKwuV1bdPwPNxnaYrIHt/V7L0QQz1UwTdLcjl/7urbD1ei+wPZtdtc7ypa/i4rYcCG68VkAimKSFSh51wD2AVdc061mjXpYVD/C/L8DbwUGqSgn1H2MNFRamG1vVk6qvG9GiDXd89NTzQ/l98XWlpseg3bPiO40e8+5j2HD84lNU55VfrJEn1J9s052vnhlSO5Iidiafn/pXgXyckKIIpYuaaJ9ZIklb/4Ks6a9jAsOYVzQqTwyeaZTvZrgkjs8OeVzjHdY8nhQQ4UXTXZltPtrk8h0TQ83sVThDe6nJXo0ACTjA/KhJtf3I4HLpv6ScaM7R/j+djs/w0l3XbazX77GMaPE/bMdG2F+KLYIqARar5fFeVzW8w9XdCjOaB7ur/6QrQxXOnaEROP8/LD6WPaQKESk+C/qinpnf9vJ0Oh/rEOVUn6vr3Jhqhpncf03DmFfp8wu2La5qPy094vCI+0GNqz6mCbn1xeUMQuTRiIrm9OjsdOtnWoQGZxJdkxcVPiLmAQp2/PqYxODHtrYnsrU8j0gwb/zuSum0/n2Nguvy/04AkYdpQOKb0k3bdNIGWKJDN6fr5gh7H1JB1Ewxfx7XWdnM+T++7PZn1vZB874czF2/WBQ+vUGW959tWI/ERTFOAaccdAzKKJP8HZNcwFfIqtHj5f0BvDXMc0zhvd9f1G4ltfrK1Q79fv18HQri/+LZDJ/SHDQfCL0QPu6oie/tVf5ss2tu0d1N+5Oblj9vV4R77Lgf2gyjSIrHOvd6owOfHiNzGdj2W+Kox7VlOj83yEfgyOxwOfVHToLaOzqDfu35v1x303vykMuxywEwEU8RcZE4q0T8xRXoJhv0+CJnfi59O/ev6eT3d7jBYv/7bF/r5u5/ruifXBv3eGQs3hr18T/7utx9FZb7RVNfUql+uOD1k0CGX0Qp6NRmH05Qfofn4nXkCSPOyAmL2MVyb8v38APClxtaiy+f9Tf+zfI//iX14fUuFbnj6Q3335W0hz6Pn7VWRPAimKSBSNZSxnE/voWOiUxaffCwjEQ6J8aqZ7t5ePfuYhmvzQc9DzrS71LrEehihYBfn70dZz1e3V9TrlU2HnJ/Lvdk9tM8696+facGafc6/y2pPd1npvf4ilEwDEMxNNRLh++cq3GAebt9f94Yb71fe+9ufn11TpmONrVq4dp/vCf34/UddrRd/+7w25Hkk2j6AwBFMERH7jzbqwb98pkPH/TezBhRMI9wMHIpINxcG3Szm5+RTbQ38bipN9vaApw3X6RrT0x8gEpUbnmqdmlvbNXn+6vBnHkEvFh/Uyl3VfqcLZL++dcEG/fiNnfpgT+gn8J62Hjzh9nd1lO7K49bHNNgB9n3UxIdcnjgmGa+fPkaFsrgnU/fXYlICd5EYOc6ULmGIPIJpCohUM5qv+dzx+016bUuFc6BuqavJcJuHgZVDvMg7oHk4HA63GrRwhJEZA37PXz8OvZ/UN57z3Tztur289a+sb27Vb1fvVUVd4IPP+zshHDrerJ++uVOHT5ye5/serkgOlqeT2fq9x3SssTXseUfKFzUNeuitXfrOS8E3UdY2tHjdd784dSFez+9gra0l6H56vkJirz6mfnb0Rnu7rM1tHl8LJzh4+lHo1sc0gHlbm9v0yNu7Qi9EGOztHfrZO7u1sayrP6TXPqYxKo+3PqYWi/v+EKt+6d66NgTDhAsqER0EUwTM13HgyKnau3KXgHPdk2s1Y2Gx1pS61/YEckDxd0V3S1uHXi455BZ+JOmB17fr8nl/83qyDIavYoZySPR0LP7fNWXep/czv56DofviOqaoqwf/skO/WvWFbvPRBzPY4//2inq9VHJIdy3ecno5f90R3Ew88HQxWCROcKFqaOm9To812kOa1xc1Dbpi3mr9aethj697+g58fqRBV/xitW57Nrj+s75qqxxyyNrcppeKD6quqdXnPuhwOHThwyt08WMrdbK1w8O8Tgvozk9BNCkH0prxi/c+17ZDJ/xOF6yy2kY1t/pugVj00QEt+uiAvv37TZLCv/K9d99f7/PbduhEr7s7ub/XZb4O39sm3hdQ+kIsTV4E0yAtWLBAZ555prKysnTllVdq8+bN/t9koMYYNO12h6FVu2vcXwioKd9lcg/Tb9x3XD95c6em/3q92/Nvbq/SieY2vfWp/5rIKj/DjcTrF3lre6fXE1+oJfJ2ocDyU03ORxuCCFSuff18FKjVR03etkN1evTtXUF1MfB0koz2zaR8/cB5+K3etXF9Qhwk/u1Pq3xO62n7dV+VvKPS2vs75oOvIjoc0gN/2q6fvrVL9760VXu99D91yOF2QVvPH4g9pw96HFNJh443yd7e4fZct/1H/XcZ2lvbENxCA7D5QJ2mPrVONz3T+6I31/KV97j9rbeP372KPqmoj0j56ppaNWPhRn39mfVe1384o31E6ojoL6gHNBwZNaZJi2AahNdff11z5szRww8/rI8//lgXX3yxpk2bptrayPX/igZPh4Cbnlnv4dno6HlS/dFfPtMXNYGfNHzVjjR4CTbtHf4PWu985rt5ORJXFfsaisWb655cqwkPrZC93XuwC/Yq9w5HV/OizUMtX7fbn9uouqbYNIvPWFisxRsO6jer9/qc7r0dR5y1Xp5qR/3VmFbVnwxpeClJWrqpXBc/tlJ/+MhzN4iVHsKg621X/W2j7hPril3V+u0H3mvOpdN9dF0/bR+XZd3z4laf73cro6+mfMnZn3XLwRP65YpSl/K6T9vh8kSHh5DgXmMaUJ2p838l+47rq79cq398vsTj8h95e1evYLLui6O6dcEG57Glj4dfLburbPr2CyX6uDzwmtQXNx50/v/dz7p+QHjap1yL83n16eNbZ6fDaxDr/gyfegimoRxzXH9gtnc61GhvV0tbh9u8eo3/6WPw/WgJ9gfl/3tjh+a8vt1jd45XNh3SPz5XHJFWMpiBYBqEp556Svfcc4/uvvtuTZgwQc8995z69++vP/zhD/EuWtAOHW/WwVMHV4fDoeU7j+jaX67RhlN9osLR84Th6QR9w9Mf+gxJrieyUH4Yd9d2/m13jf7z1U+cNcTlx5v1P8v3eK0hdC37/31apb01DapratXiDQeDL0QPgdZUBDJw9L/9cYvfaVx1Ohya/uv1mvjIStU3t2rJhgN60iV0SF1B5OlVXwQ132B4al7c6+MHSlltg/7jlY81Y+FGtbZ3aqeHMUPT/Jzhrnr8A4/DS5Ufb/a67OfX7dOsJVv0/97o6n7w2Du7PU7nqdWh0qV7hb++n1f/zxq1dXTqXi/9Uf/+f0/Xyi3Z2Dsc9wyYgdQgHWu0q83Hj7ZAu6/Y2zrdvtfd/3ddp4He+cnhcOiHyz7Vz989vZ5/e6qLi2tg+9Uq95A8c/EWtbR1qKy2UR2dDs38w2Ztr6h3DkHkKYD/2x+3aOO+40F1f2hy6abw6WH3fdC1pru7haisttGt3Jf+bJXP/bynuqZWHW2wa23p0YDf0801jNtOtuniR1dq8vzVbsee8rqetbneN060mvKD6YJjb+/Q0k3l+usnlW7dl7orLH78xk5tPlin+e9/HvFyIj4sDurDA9La2qr+/fvrz3/+s2699Vbn8zNnzlR9fb3eeustt+ntdrvs9tPhx2azqaCgQFarVdnZ4d+DvVtFXbNeLD7oc5oX1nuu8ZGkzPS0XjVzt08are0V9RoztL/GnjHA7f33XDNWUldt0YmmVn31vOHKz850m6ZvH4uGDsh0Xu2bmZ6mO4vGeCxH9/x6OnCsWX/7vKtG6q6rzlTfPhavn+XbVxZqQEafXq8XjRuq4v3H3ZblbV10l8PXunI1KCtdDS3tGnfGAF1//nCv07nO74ozc51DHgW7PFfj8wdpT3XvE90/Ty5Uv759nH/XNbXpLx/37rN42Zgh2hpE37tvXlagyvqT2lPdoItGZasgt79eLD4kSfr7i0cqLztTUmCf5ZKCwbr8zCFu014/frjK65qVZrHosjOHqP+pbbmnusE5mHZP3etv/9EmrXa5Yv1LhYM1acwQSV19Xbcc9P05vzFptIb07+v2nKfPMevqsVpbWqt9ATQhd7vrqjNlb+/Qsq2HNWpIP90wIU+2k+16fWtFwPPoOb8j1pNasavre5GXnakam/sPrG9eVqDsfp5v1bh0U7lbyPJk3LABXpvJ//XLY71eRHfO8IE6eLypV+i9tHCw0tMsOnCsyXmR2r9dPdYt8Bw+cVLv7/Q+ikEo35XpF+b3mmfP7423Y09P3pZ73XnDtKZHePznyYV6uaQ84HKePyJbV5891O9nu7NojDLTT9cjrd5T69xOrp/DerLN2U952gV5zn3Fl/4ZfdR8ar/IzkqXraXrB9dtl47SX10Gse95/PS2/vwdY+ub27Rs2+Fezx883uzWJWV8/iClWSzafcR7f9meZfrmZQW6YFS27iw60+t7QmGz2ZSTkxPx8zc8I5gGqKqqSqNGjdLGjRtVVFTkfP5HP/qR1q1bp02bNrlN/8gjj+jRRx/tNZ9I79hbD9bpG88VR2x+AAAkqsvPHKJl/35VROdJMI0tzz+tEba5c+dqzpw5zr+7a0wjLS87S/d+dZzPaTo7HfrLx5W6+uwz9FHZMdU3t2pg5ulfxpKUnmZReh+L/uHS0RqUla7XNpfrmnOHafSQfjrR1KqVu2v0D5eOUsapX+3V1hZ9sKdW37qiUBaL1Gzv0JufVKqxtV3/cMkopaVZ9OdthzWkf1/942UFkkUqrW7QpxX1arS3yyKLbr9stAZmed8F1+45qtwBGZpYkON8rtraoo/2HtO4YQOctWGzrh6r9FM1qvtqG7WnukF9+6QpLztTI3P66cO9RzXjS6Mli/T8uv3OeXXXeFw8OkeTzxoqSWpoadfSTV01HgW5/XT2sIEqyO2vNz6p1OD+fdXY0q5BWX01/cJ8vb61QleffYZGDenn9TOUH2/W+zurdfuk0codmKFVu2s0anA/TRjZdXCzt3VqycaDGjdsgJrs7fpS4RBZT7bpWKNdV511hpac6t/WXfubnZWu1o5OffmsM7S+7JhaXWq77/3KuF6dxLo/791fPlNltY2qPHFSX7sgT9vL63X4xEm3bgNDB2ToG5NG69DxZi3fVa1hgzI1cnA/TR6Xq5bWDi3bdli3XDJK2f3S9Ul5vRpb2nXNuWc431954qRK9tdp+oX5eqnkkDLS03T+iGy3Zs3bLh2lYdmZOt7Yqjc/qVTfPmn6x8tG6/CJk9pZZdWtl45yK//KXTU6cKxJN16Qrw1lx9TpcGjqhDzl52S57Se1DS0a3D9DN1yQ53y+e93mZ2fp+vOH691TQ1ZdfmaurCfbZG1u07Xjh/XaZmU1jSqtadD144frze1V+vpFI5TdL11r9tSq03F6QPqvnjtM677oqjHrn9FHV47NVcn+Op1s69Cowf30dxePkCS98+kRnZs3UOfmD5IkffjFMX1+qhbonmvGqr65TSt316hf3z66YGS2dlZZ9aXCISqtblBrR6cOnzipSwsH64qxuZKkdaVHNbh/X11cMNhtf85I71qXAzI9f6eONtj1148rdeXYXJXWNKilrUMtbaf3n9FD+ummiSPU0tpx6nPn67PDVu2qsunLZw/VhaNytLHsuHZUWnXXVWfq/z6tUntHp2wt7bqzaIyqrS1aubtGaRYpp19fXXFqfYwe0k+t7Z1ySBqRk+Xc9111r5MROVkakJmuGluLMvqk6fIzczXmjP6SumrHe17k1d3i883LCpSWJr26uUJZfdN0x5Vj1OfUMai73/S9Xx0ne1un3vikUtMvzFdOj5pyb7q/w5J06yUjtflAnS4claOxwwZo+c5q552zJo7OUdFZQ1Wyv04Oh0OfHbYqo0+a+mf20e2TRmtD2XEdbbQ7uxEV5vbX9IvyJUk7K63aUHZcN16Qr2ONdjW1dmjcsAF697MjumJsri4tHNyrXO/tOKLz8gbprOED3Z4v3ndc6WkWXT42VxvLjqtvH4suOzNXv/uwa185Y2CGs/b6O18ZJ4ukNz6p1GVnDlFBbn89v26/Bmam647JhWprd+iNTw7r+vPzNHRghuSQnv9wv/Kzs3TLpSM9rq+Kuma9t6NaV501VGeeMUBLN5VryvjhOievq5wOh/TqpnI12NudxwOpq+vBq5sr1K9vH+X066vzRwxya52ZMCJbHZ0OHTzeJHt7p2YWjVFWRh8drjupd3cc0dnDB+r684dr9JD+AW1XmIsa0wAF25Tfk4m/uG5dsEHbT4WGg4/fFN/CxMiZD74rqatp7LFbLoxzaYDwdO/P/3n9OZrztXPjXBogOZl4/k5mXPwUoIyMDE2aNEmrV5++y0xnZ6dWr17t1rQPAACA0NCUH4Q5c+Zo5syZuuyyy3TFFVfo17/+tZqamnT33XfHu2ghoaocAACYhGAahG9+85s6evSoHnroIVVXV+uSSy7R8uXLlZeX5//NMIrBNzQBgsb+DCBZEEyDdN999+m+++6LdzEAAACSDn1MAQAAYASCKQAAAIxAMAUAAIARCKYAAAAwAsE0lXFvBQAAYBCCKVKSxcIAO0ge7M4AkgXBFAAAAEYgmAIAAMAIBFMAAAAYgWAKAAAAIxBMAQAAYASCKQAAAIxAME1hjGIKJAeLGC8KQHIgmAIAAMAIBFMAAAAYgWAKAAAAIxBMAQAAYASCKQAAAIxAMAUAAIARCKYpzMF4UUBSsDBaFIAkQTBFSuJEDgCAeQimAAAAMALBFAAAAEYgmAIAAMAIBFMAAAAYgWAKAAAAIxBMAQAAYASCaQpziIFMgWTA6GcAkgXBFCnJwqkcAADjEEwBAABgBIIpAAAAjEAwBQAAgBEIpgAAADACwRQAAABGIJimMAejRQFJwcIgEwCSBMEUKYkTOQAA5iGYSpo3b56uuuoq9e/fX4MHD/Y4TXl5uW666Sb1799fw4cP13/913+pvb09tgUFAABIYunxLoAJWltbdfvtt6uoqEiLFi3q9XpHR4duuukm5efna+PGjTpy5IjuvPNO9e3bV7/4xS/iUGIAOM1CEwCAJEGNqaRHH31UDzzwgC666CKPr69cuVK7d+/Wyy+/rEsuuUTTp0/Xz372My1YsECtra0xLi0AuHPQYRxAkiCYBqC4uFgXXXSR8vLynM9NmzZNNptNu3bt8vgeu90um83m9gAAAIB3BNMAVFdXu4VSSc6/q6urPb5n/vz5ysnJcT4KCgqiXk4AAIBElrTB9MEHH5TFYvH52LNnT9SWP3fuXFmtVuejoqIiassCkNroYwogWSTtxU8/+MEPdNddd/mcZty4cQHNKz8/X5s3b3Z7rqamxvmaJ5mZmcrMzAxo/vFCtzQAAGCSpA2mw4YN07BhwyIyr6KiIs2bN0+1tbUaPny4JGnVqlXKzs7WhAkTIrIMxBb1SwAAmCdpg2kwysvLVVdXp/LycnV0dGj79u2SpLPPPlsDBw7UDTfcoAkTJuhf/uVf9MQTT6i6ulo/+clPNHv2bONrRQEAABIFwVTSQw89pD/+8Y/Ovy+99FJJ0po1a3TttdeqT58+euedd/Td735XRUVFGjBggGbOnKnHHnssXkUGAABIOgRTSUuWLNGSJUt8TjNmzBi99957sSkQAABACkraq/IBAACQWAimAAAAMALBNIUxWhQAADAJwRQpifHIAQAwD8EUAAAARiCYAkCCowUAQLIgmAJAguP2wgCSBcEUAAAARiCYAkCCoykfQLIgmKYwB+1/AADAIARTpCQLVUwAABiHYAoAAAAjEEwBIMFZRAsAgORAMAWABOfgBsMAkgTBFAAAAEYgmAIAAMAIBFMASHD0MQWQLAimAAAAMALBFCmJ+iUAAMxDMAWABMf9IgAkC4IpAAAAjEAwBYAE52AYUwBJgmAKAAAAIxBMASDB0ccUQLIgmKYwmv8AAIBJCKZITdQwAQBgHIIpACQ4fmcBSBYEUwAAABiBYAoACY7u4gCSBcEUAAAARiCYAkCCo48pgGRBMAUAAIARCKYpzEHPNAAAYBCCKVKShcZPAACMQzAFgATHLUkBJAuCKQAAAIyQ8sH04MGDmjVrlsaOHat+/frprLPO0sMPP6zW1la36T777DNdc801ysrKUkFBgZ544ok4lRgAACA5pce7APG2Z88edXZ26vnnn9fZZ5+tnTt36p577lFTU5OefPJJSZLNZtMNN9ygqVOn6rnnntOOHTv0r//6rxo8eLC+853vxPkTAAAAJIeUD6Y33nijbrzxRuff48aNU2lpqRYuXOgMpq+88opaW1v1hz/8QRkZGbrgggu0fft2PfXUUwRTAACACEn5pnxPrFarcnNznX8XFxfrK1/5ijIyMpzPTZs2TaWlpTpx4oTHedjtdtlsNreHaRyMFgUAAAxCMO2hrKxMv/3tb3Xvvfc6n6uurlZeXp7bdN1/V1dXe5zP/PnzlZOT43wUFBREr9AIGlcxAwBgnqQNpg8++KAsFovPx549e9zeU1lZqRtvvFG333677rnnnrCWP3fuXFmtVuejoqIirPkBgDeMywsgWSRtH9Mf/OAHuuuuu3xOM27cOOf/q6qqdN111+mqq67S7373O7fp8vPzVVNT4/Zc99/5+fke552ZmanMzMwQSg4AAJCakjaYDhs2TMOGDQto2srKSl133XWaNGmSFi9erLQ094rkoqIi/fjHP1ZbW5v69u0rSVq1apXOO+88DRkyJOJlB4BgcHthAMkiaZvyA1VZWalrr71WhYWFevLJJ3X06FFVV1e79R399re/rYyMDM2aNUu7du3S66+/rt/85jeaM2dOHEsOAACQXJK2xjRQq1atUllZmcrKyjR69Gi31xynLlvPycnRypUrNXv2bE2aNElnnHGGHnroIYaKAmAE+pgCSBYpH0zvuusuv31RJWnixIlav3599AsEAACQolK+KT+V0SsNAACYhGCKlETDJ5IJ4/ICSBYEUwAAABiBYAoAAAAjEEwBAABgBIIpAAAAjEAwBQAAgBEIpims+wYCAAAAJiCYIiUxvA4AAOYhmAIAAMAIBFMAAAAYgWCawr5y7jBJ0uD+feNcktibMCIn3kUAIubcvEHxLgIARER6vAuA+PnRtPEaN2ygrh8/PN5FiZn3v3eNdhy26usX5ce7KEDY3rn/apVWN+iac86Id1EAICIsDi7NjgmbzaacnBxZrVZlZ2fHuzgAACAAnL9ji6Z8AAAAGIFgCgAAACMQTAEAAGAEgikAAACMQDAFAACAEQimAAAAMALBFAAAAEYgmAIAAMAIBFMAAAAYgWAKAAAAIxBMAQAAYASCKQAAAIxAMAUAAIAR0uNdgFThcDgkSTabLc4lAQAAgeo+b3efxxFdBNMYaWhokCQVFBTEuSQAACBYDQ0NysnJiXcxkp7FwU+AmOjs7FRVVZUGDRoki8US0XnPmzdPTzzxRETnCQBAIvrggw80adKkiM3P4XCooaFBI0eOVFoaPSCjjRrTGElLS9Po0aOjMu/MzMyozBcAgEQzcOBAZWdnR3Se1JTGDtEfAAAARiCYAgAAwAgE0yRw2WWXxbsIAADEXVZWlkaMGBHvYiAMXPwEAAAAI1BjCgAAACMQTAEAAGAEgikAAACMQDAFAACAEYIaYD/SdywCAABA8srMzFRlZaWGDh0a0PTUmAIAACAq7Ha7vvGNbwQ8fVDB1OFwuD3mzJkTdAEBAACQOrZt2xbwtGHVmG7dujWctwMAACDJ3XjjjQFPG/IA+3V1dQH3FwAAAEBqamxs1IABAwKaNuQaU0IpAAAA/JkyZUrA04ZUY8rV+QAAAAhUVVWVRowY4Xe6oGpMT548SSgFAABAUAKtBw2qxpRQCgAAgGAUFhbq0KFDAU3LOKYAAACImieffDLgaYO681OIF/ADAAAAflFjCgAAACMQTAEAAGAEgikAAACMQDAFAACAEQimAAAAMALBFAAAAEYgmAIAAMAIBFMAAAAYgWAKAAAAIxBMAQAAYASCKQAAAIxAMAUAAIAR/j8zTFv2a4W9BQAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "x = [r['date_added'] for r in recs]\n", "y = [r['time_lost'] for r in recs]\n", @@ -1129,7 +272,7 @@ }, { "cell_type": "markdown", - "id": "6855a63f-7837-4901-8db1-f9d659a28101", + "id": "18", "metadata": {}, "source": [ "\n", @@ -1138,7 +281,7 @@ }, { "cell_type": "markdown", - "id": "1b9711f5-2f32-4531-bf41-b6083983ed98", + "id": "19", "metadata": {}, "source": [ "### Example of one record" @@ -1146,46 +289,10 @@ }, { "cell_type": "code", - "execution_count": 22, - "id": "0720e2cc-d68c-4830-92c1-74ae5180f057", - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "data": { - "text/plain": [ - "{'id': '0005d0e0-6de7-4857-b658-cfca7147374a',\n", - " 'site_id': 'summit',\n", - " 'message_text': 'The moveShutterMainDoor command Fail\\r\\n\\r\\n\\r\\nwe try the parameter with uppercase now\\r\\n\\r\\n\\r\\n open: False\\r\\n\\r\\n\\r\\n\\r\\n\\r\\nError in run\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 500, in wait_for\\r\\n return fut.result()\\r\\n ^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 239, in _get_next_ackcmd\\r\\n await self._next_ack_event.wait()\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/locks.py\", line 213, in wait\\r\\n await fut\\r\\nasyncio.exceptions.CancelledError\\r\\n\\r\\n\\r\\nThe above exception was the direct cause of the following exception:\\r\\n\\r\\n\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 189, in next_ackcmd\\r\\n ackcmd = await self._wait_task\\r\\n ^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 214, in _basic_next_ackcmd\\r\\n ackcmd = await asyncio.wait_for(\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 502, in wait_for\\r\\n raise exceptions.TimeoutError() from exc\\r\\nTimeoutError\\r\\n\\r\\n\\r\\nDuring handling of the above exception, another exception occurred:\\r\\n\\r\\n\\r\\nTraceback (most recent call last):\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\\r\\n await self._run_task\\r\\n File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/run_command.py\", line 170, in run\\r\\n await getattr(self.remote, f\"cmd_{self.cmd}\").start(\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\\r\\n return await cmd_info.next_ackcmd(timeout=timeout)\\r\\n ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\r\\n File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 205, in next_ackcmd\\r\\n raise base.AckTimeoutError(\\r\\nlsst.ts.salobj.base.AckTimeoutError: msg=\\'Timed out waiting for command acknowledgement\\', ackcmd=(ackcmd private_seqNum=1142844592, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result=\\'No command acknowledgement seen\\')\\r\\n',\n", - " 'level': 0,\n", - " 'tags': [],\n", - " 'urls': [],\n", - " 'time_lost': 0.0,\n", - " 'date_begin': '2024-03-20T04:15:26.769000',\n", - " 'user_id': 'cmorales@summit-lsp.lsst.codes',\n", - " 'user_agent': 'LOVE',\n", - " 'is_human': True,\n", - " 'is_valid': True,\n", - " 'date_added': '2024-03-20T04:16:06.976595',\n", - " 'date_invalidated': None,\n", - " 'parent_id': None,\n", - " 'systems': None,\n", - " 'subsystems': None,\n", - " 'cscs': None,\n", - " 'date_end': '2024-03-20T04:15:26.769000',\n", - " 'components': ['AuxTel'],\n", - " 'primary_software_components': ['None'],\n", - " 'primary_hardware_components': ['None'],\n", - " 'category': 'None',\n", - " 'time_lost_type': 'fault'}" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], "source": [ "rec = recs[0]\n", "rec" @@ -1193,70 +300,10 @@ }, { "cell_type": "code", - "execution_count": 26, - "id": "82392f54-524e-4890-a2f8-ce0579a75e1c", + "execution_count": null, + "id": "21", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "The moveShutterMainDoor command Fail\n", - "\n", - "\n", - "we try the parameter with uppercase now\n", - "\n", - "\n", - " open: False\n", - "\n", - "\n", - "\n", - "\n", - "Error in run\n", - "Traceback (most recent call last):\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 500, in wait_for\n", - " return fut.result()\n", - " ^^^^^^^^^^^^\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 239, in _get_next_ackcmd\n", - " await self._next_ack_event.wait()\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/locks.py\", line 213, in wait\n", - " await fut\n", - "asyncio.exceptions.CancelledError\n", - "\n", - "\n", - "The above exception was the direct cause of the following exception:\n", - "\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 189, in next_ackcmd\n", - " ackcmd = await self._wait_task\n", - " ^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 214, in _basic_next_ackcmd\n", - " ackcmd = await asyncio.wait_for(\n", - " ^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", line 502, in wait_for\n", - " raise exceptions.TimeoutError() from exc\n", - "TimeoutError\n", - "\n", - "\n", - "During handling of the above exception, another exception occurred:\n", - "\n", - "\n", - "Traceback (most recent call last):\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", line 603, in do_run\n", - " await self._run_task\n", - " File \"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/run_command.py\", line 170, in run\n", - " await getattr(self.remote, f\"cmd_{self.cmd}\").start(\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 487, in start\n", - " return await cmd_info.next_ackcmd(timeout=timeout)\n", - " ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n", - " File \"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", line 205, in next_ackcmd\n", - " raise base.AckTimeoutError(\n", - "lsst.ts.salobj.base.AckTimeoutError: msg='Timed out waiting for command acknowledgement', ackcmd=(ackcmd private_seqNum=1142844592, ack=<SalRetCode.CMD_NOACK: -301>, error=0, result='No command acknowledgement seen')\n", - "\n" - ] - } - ], + "outputs": [], "source": [ "msg = rec[\"message_text\"]\n", "print(msg)" @@ -1264,7 +311,7 @@ }, { "cell_type": "markdown", - "id": "3a938226-c21e-4e52-be30-b11148e8e379", + "id": "22", "metadata": {}, "source": [ "\n", @@ -1274,7 +321,7 @@ { "cell_type": "code", "execution_count": null, - "id": "138e5496-e058-417a-881e-8450e20af0f9", + "id": "23", "metadata": {}, "outputs": [], "source": [] From 709d215e107ce417aadb11f3c0e5c4a5e10542d6 Mon Sep 17 00:00:00 2001 From: Steve Pothier Date: Tue, 27 Aug 2024 09:07:35 -0700 Subject: [PATCH 05/10] WIP --- notebooks_for_times_square/README.md | 5 - .../logrep_proto_1.ipynb | 236 ------------------ .../logrep_proto_1.yaml | 24 -- notebooks_for_times_square/requirements.txt | 6 - notebooks_for_times_square/scaffolding.org | 71 ------ 5 files changed, 342 deletions(-) delete mode 100644 notebooks_for_times_square/README.md delete mode 100644 notebooks_for_times_square/logrep_proto_1.ipynb delete mode 100644 notebooks_for_times_square/logrep_proto_1.yaml delete mode 100644 notebooks_for_times_square/requirements.txt delete mode 100644 notebooks_for_times_square/scaffolding.org diff --git a/notebooks_for_times_square/README.md b/notebooks_for_times_square/README.md deleted file mode 100644 index 3efda5a..0000000 --- a/notebooks_for_times_square/README.md +++ /dev/null @@ -1,5 +0,0 @@ -Times Square notebooks for (potential) use in project-wide Logging & Reporting - -See [offical -documentation](https://rsp.lsst.io/v/usdfdev/guides/times-square/index.html) -on creating notebooks for use by Times Square. diff --git a/notebooks_for_times_square/logrep_proto_1.ipynb b/notebooks_for_times_square/logrep_proto_1.ipynb deleted file mode 100644 index ff93343..0000000 --- a/notebooks_for_times_square/logrep_proto_1.ipynb +++ /dev/null @@ -1,236 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": null, - "id": "0", - "metadata": {}, - "outputs": [], - "source": [ - "# Parameters. Set defaults here.\n", - "# Times Square replaces this cell with the user's parameters.\n", - "env = 'usdf_dev' # usdf-dev, tucson, slac, summit\n", - "log_name = 'narrativelog'\n", - "record_limit = '999'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1", - "metadata": {}, - "outputs": [], - "source": [ - "# Only use packages available in the Rubin Science Platform\n", - "import requests\n", - "from collections import defaultdict\n", - "import pandas as pd\n", - "from pprint import pp\n", - "from urllib.parse import urlencode\n", - "from IPython.display import FileLink\n", - "from matplotlib import pyplot as plt" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ - "log = log_name\n", - "limit = int(record_limit)\n", - "response_timeout = 3.05 # seconds, how long to wait for connection\n", - "read_timeout = 20 # seconds\n", - "\n", - "timeout = (float(response_timeout), float(read_timeout))\n", - "\n", - "# Env list comes from drop-down menu top of:\n", - "# https://rsp.lsst.io/v/usdfdev/guides/times-square/\n", - "envs = dict(\n", - " #rubin_usdf_dev = '',\n", - " #data_lsst_cloud = '',\n", - " #usdf = '',\n", - " #base_data_facility = '',\n", - " summit = 'https://summit-lsp.lsst.codes',\n", - " usdf_dev = 'https://usdf-rsp-dev.slac.stanford.edu',\n", - " #rubin_idf_int = '',\n", - " tucson = 'https://tucson-teststand.lsst.codes',\n", - ")\n", - "server = envs[env]\n", - "server" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ - "recs = None\n", - "ok = True\n", - "\n", - "# is_human=either&is_valid=either&offset=0&limit=50' \n", - "# site_ids=tucson&message_text=wubba&min_level=0&max_level=999&user_ids=spothier&user_agents=LOVE\n", - "# tags=love&exclude_tags=ignore_message\n", - "qparams = dict(is_human='either',\n", - " is_valid='either',\n", - " limit=limit,\n", - " )\n", - "qstr = urlencode(qparams)\n", - "url = f'{server}/{log}/messages?{qstr}'\n", - "print(f'\\nAttempt to get logs from {url=}')\n", - "ignore_fields = set(['tags', 'urls', 'message_text', 'id', 'date_added', \n", - " 'obs_id', 'day_obs', 'seq_num', 'parent_id', 'user_id',\n", - " 'date_invalidated', 'date_begin', 'date_end',\n", - " 'time_lost', # float\n", - " #'systems','subsystems','cscs', # values are lists, special handling\n", - " ])\n", - "try:\n", - " response = requests.get(url, timeout=timeout)\n", - " response.raise_for_status()\n", - " recs = response.json()\n", - " flds = set(recs[0].keys())\n", - " facflds = flds - ignore_fields\n", - " # facets(field) = set(value-1, value-2, ...)\n", - " facets = {fld: set([str(r[fld])\n", - " for r in recs if not isinstance(r[fld], list)]) \n", - " for fld in facflds}\n", - "except Exception as err:\n", - " ok = False\n", - " print(f'ERROR getting {log} from {env=} using {url=}: {err=}')\n", - "numf = len(flds) if ok else 0\n", - "numr = len(recs) if ok else 0\n", - "print(f'{numr=} {numf=}')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "flds" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5", - "metadata": {}, - "outputs": [], - "source": [ - "facets" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "recs[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [ - "cols = ['date_added', 'time_lost']\n", - "df = pd.DataFrame(recs)[cols]\n", - "df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8", - "metadata": {}, - "outputs": [], - "source": [ - "# Allow download of CSV version of DataFrame\n", - "csvfile = 'tl.csv'\n", - "df.to_csv(csvfile)\n", - "myfile = FileLink(csvfile)\n", - "display(myfile)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9", - "metadata": {}, - "outputs": [], - "source": [ - "x = [r['date_added'] for r in recs]\n", - "y = [r['time_lost'] for r in recs]\n", - "plt.plot(x, y) \n", - "plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "10", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11", - "metadata": {}, - "outputs": [], - "source": [ - "recs[:4]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "12", - "metadata": {}, - "outputs": [], - "source": [ - "df = pd.DataFrame(recs)\n", - "df" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "13", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/notebooks_for_times_square/logrep_proto_1.yaml b/notebooks_for_times_square/logrep_proto_1.yaml deleted file mode 100644 index a7e82ac..0000000 --- a/notebooks_for_times_square/logrep_proto_1.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# For use with a Times Square notebook -title: Logging and Reporting -description: Prototype 1 -authors: - - name: Steve Pothier - slack: Steve Pothier -tags: - - reporting - - prototype -parameters: - env: - type: string - description: "tucson | usdf_dev | summit" - default: "tucson" - log: - type: string - description: "The log name: narrativelog | exposurelog" - default: narrativelog - record_limit: - type: integer - description: Max number of records to output - default: 99 - minimum: 1 - maximum: 9999 diff --git a/notebooks_for_times_square/requirements.txt b/notebooks_for_times_square/requirements.txt deleted file mode 100644 index 5ab865b..0000000 --- a/notebooks_for_times_square/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -requests -# jupyter-lab --ip=0.0.0.0 -jupyterlab # =3.1.17 -pandas -matplotlib -#numpy diff --git a/notebooks_for_times_square/scaffolding.org b/notebooks_for_times_square/scaffolding.org deleted file mode 100644 index 6c1c85a..0000000 --- a/notebooks_for_times_square/scaffolding.org +++ /dev/null @@ -1,71 +0,0 @@ -# Scaffolding for Times Square notebooks (thread-1) -* Parameters -- Parameters available to user via Times Square side panel. - -- A schema for these is given in .yaml in same directory. - -- The defaults given here must all be string values. Convert those to - other types in a subsequent cell - -- Include parameters that affect filtering (query) used in Source API - -- Consider common parameters such as: - + record_limit - + num_recs_in_example - + min_date - + max_date - - - -: # Parameters. Set defaults here. -: # Times Square replaces this cell with the user's parameters - - -* Imports and General Setup -- Setup includes conversion of parameter values to usable values -- Get "environment" (server host). e.g. https://usdf-rsp-dev.slac.stanford.edu -- Included non-user parameters: - + timeout - + env - -* Setup Source -- Choose fields of interest. The records (list of dict) in each field - we need in any way. All other fields could be deleted for our - purposes. - -- Choose fields to use for extracting facets. - -- Create full API URL (including query string) from parameters, etc. - -- Choose fields to display to user in table. ~table_fields~ -- Choose fields to display to user in plot. ~(x,y)~ - -* Get Records -- Usually use python "requests" package for accessing API - But, some APIs might have their own client. - -- Use try/except around requests.get() or requests.post() - User friendly error if exception (cannot connect, etc) - User friend error if get non-200 response status. - -- Get list field names from rec[0] - -- Calc facet over selected field names. - -* Table of (mostly raw) results -- Display pandas table of all records using - : df = pd.DataFrame(recs)[table_fields] - -* Plot of results -- Display minimum useful (?!) plot from this Source - Example: - : plt.plot([r['date'] for r in recs], [r['time_lost'] for r in recs]) - : plt.show() - -* Output raw content analysis (rec, fields, facets) -** Example records -** Fields found -** Facets of selected Fields -* Stakeholder Elicitation -- Source specific questions to stakeholders. - Link to Confluence doc (form?) for L&R stakeholder feeback From 40c4c15aaaf48031b9ef5b2e78483e4afad3bd6b Mon Sep 17 00:00:00 2001 From: Steve Pothier Date: Tue, 27 Aug 2024 09:12:10 -0700 Subject: [PATCH 06/10] WIP --- notebooks_tsqr/logrep_proto_1.ipynb | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/notebooks_tsqr/logrep_proto_1.ipynb b/notebooks_tsqr/logrep_proto_1.ipynb index 0bd42a0..de921b8 100644 --- a/notebooks_tsqr/logrep_proto_1.ipynb +++ b/notebooks_tsqr/logrep_proto_1.ipynb @@ -9,8 +9,6 @@ "source": [ "# Parameters. Set defaults here.\n", "# Times Square replaces this cell with the user's parameters.\n", - "env = 'usdf_dev' # usdf-dev, tucson, slac, summit\n", - "log_name = 'narrativelog'\n", "record_limit = '999'" ] }, @@ -48,13 +46,11 @@ "cell_type": "code", "execution_count": null, "id": "3", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, "outputs": [], "source": [ + "env = 'usdf_dev' # usdf-dev, tucson, slac, summit\n", + "log_name = 'narrativelog'\n", "log = log_name\n", "limit = int(record_limit)\n", "response_timeout = 3.05 # seconds, how long to wait for connection\n", From cbd71b44ccdeec317cb77cd18861115b97c08088 Mon Sep 17 00:00:00 2001 From: Steve Pothier Date: Tue, 27 Aug 2024 09:21:09 -0700 Subject: [PATCH 07/10] WIPO --- notebooks_tsqr/logrep_proto_1.ipynb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/notebooks_tsqr/logrep_proto_1.ipynb b/notebooks_tsqr/logrep_proto_1.ipynb index de921b8..b767518 100644 --- a/notebooks_tsqr/logrep_proto_1.ipynb +++ b/notebooks_tsqr/logrep_proto_1.ipynb @@ -78,9 +78,7 @@ { "cell_type": "markdown", "id": "4", - "metadata": { - "jp-MarkdownHeadingCollapsed": true - }, + "metadata": {}, "source": [ "\n", "## Setup Source" From 0dc737cbd955f5d8f1888a92b8d13aec9a6aa6a6 Mon Sep 17 00:00:00 2001 From: Steve Pothier Date: Tue, 27 Aug 2024 13:54:37 -0700 Subject: [PATCH 08/10] WIP --- notebooks_tsqr/SAVE_exposurelog.ipynb | 353 +++++++++++ notebooks_tsqr/SAVE_exposurelog.yaml | 16 + notebooks_tsqr/exposurelog.ipynb | 353 +++++++++++ notebooks_tsqr/exposurelog.yaml | 16 + notebooks_tsqr/logrep_all_env.ipynb | 804 ++++++++++++++++++++++++++ notebooks_tsqr/logrep_proto_1.ipynb | 6 +- notebooks_tsqr/narrativelog.ipynb | 345 +++++++++++ notebooks_tsqr/narrativelog.yaml | 16 + times-square.yaml | 1 + 9 files changed, 1905 insertions(+), 5 deletions(-) create mode 100644 notebooks_tsqr/SAVE_exposurelog.ipynb create mode 100644 notebooks_tsqr/SAVE_exposurelog.yaml create mode 100644 notebooks_tsqr/exposurelog.ipynb create mode 100644 notebooks_tsqr/exposurelog.yaml create mode 100644 notebooks_tsqr/logrep_all_env.ipynb create mode 100644 notebooks_tsqr/narrativelog.ipynb create mode 100644 notebooks_tsqr/narrativelog.yaml diff --git a/notebooks_tsqr/SAVE_exposurelog.ipynb b/notebooks_tsqr/SAVE_exposurelog.ipynb new file mode 100644 index 0000000..1daba58 --- /dev/null +++ b/notebooks_tsqr/SAVE_exposurelog.ipynb @@ -0,0 +1,353 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# Parameters. Set defaults here.\n", + "# Times Square replaces this cell with the user's parameters.\n", + "record_limit = '999'" + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "\n", + "## Imports and General Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "# Only use packages available in the Rubin Science Platform\n", + "import requests\n", + "from collections import defaultdict\n", + "import pandas as pd\n", + "from pprint import pp\n", + "from urllib.parse import urlencode\n", + "from IPython.display import FileLink\n", + "from matplotlib import pyplot as plt\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'https://tucson-teststand.lsst.codes/exposurelog'" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "limit = int(record_limit)\n", + "\n", + "response_timeout = 3.05 # seconds, how long to wait for connection\n", + "read_timeout = 20 # seconds\n", + "timeout = (float(response_timeout), float(read_timeout))\n", + "\n", + "server = os.environ.get('EXTERNAL_INSTANCE_URL', \n", + " 'https://tucson-teststand.lsst.codes')\n", + "log = 'exposurelog'\n", + "service = f'{server}/{log}'\n", + "service" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "\n", + "## Setup Source" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "recs = None\n", + "ok = True\n", + "\n", + "# is_human=either&is_valid=either&offset=0&limit=50' \n", + "# site_ids=tucson&message_text=wubba&min_level=0&max_level=999&user_ids=spothier&user_agents=LOVE\n", + "# tags=love&exclude_tags=ignore_message\n", + "qparams = dict(is_human='either',\n", + " is_valid='either',\n", + " limit=limit,\n", + " )\n", + "qstr = urlencode(qparams)\n", + "url = f'{service}/messages?{qstr}'\n", + "\n", + "ignore_fields = set(['tags', 'urls', 'message_text', 'id', 'date_added', \n", + " 'obs_id', 'day_obs', 'seq_num', 'parent_id', 'user_id',\n", + " 'date_invalidated', 'date_begin', 'date_end',\n", + " 'time_lost', # float\n", + " #'systems','subsystems','cscs', # values are lists, special handling\n", + " ])" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, + "source": [ + "\n", + "## Get Records" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Attempt to get logs from url='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=either&limit=999'\n", + "Retrieved 0 records, each with numf=0 fields.\n" + ] + } + ], + "source": [ + "try:\n", + " print(f'Attempt to get logs from {url=}')\n", + " response = requests.get(url, timeout=timeout)\n", + " response.raise_for_status()\n", + " recs = response.json()\n", + " flds = set(recs[0].keys())\n", + " facflds = flds - ignore_fields\n", + " # facets(field) = set(value-1, value-2, ...)\n", + " facets = {fld: set([str(r[fld])\n", + " for r in recs if not isinstance(r[fld], list)]) \n", + " for fld in facflds}\n", + "except Exception as err:\n", + " ok = False\n", + " print(f'ERROR getting {log} from {env=} using {url=}: {err=}')\n", + "numf = len(flds) if ok else 0\n", + "numr = len(recs) if ok else 0\n", + "print(f'Retrieved {numr} records, each with {numf=} fields.')" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, + "source": [ + "\n", + "## Tables of (mostly raw) results" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "### Fields names provided in records from log." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "pd.DataFrame(flds, columns=['Field Name'])" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "### Facets from log records.\n", + "A *facet* is the set all of values found for a field in the retrieved records. Facets are only calculated for some fields." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "pd.DataFrame.from_dict(facets, orient='index')" + ] + }, + { + "cell_type": "markdown", + "id": "13", + "metadata": {}, + "source": [ + "### Table of selected log record fields.\n", + "Table can be retrieved as CSV file for local use." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "cols = ['date_added', 'time_lost']\n", + "df = pd.DataFrame(recs)[cols]\n", + "\n", + "# Allow download of CSV version of DataFrame\n", + "csvfile = 'tl.csv'\n", + "df.to_csv(csvfile)\n", + "myfile = FileLink(csvfile)\n", + "print('Table available as CSV file: ')\n", + "display(myfile)\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame(recs)\n", + "df" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "\n", + "## Plots from log" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "x = [r['date_added'] for r in recs]\n", + "y = [r['time_lost'] for r in recs]\n", + "plt.plot(x, y) \n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "\n", + "## Raw Content Analysis" + ] + }, + { + "cell_type": "markdown", + "id": "19", + "metadata": {}, + "source": [ + "### Example of one record" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "rec = recs[0]\n", + "rec" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "msg = rec[\"message_text\"]\n", + "print(msg)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3b745aea-a46a-45e1-bdf2-def878182eb9", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "#EXTERNAL_INSTANCE_URL\n", + "with pd.option_context('display.max_rows', None,):\n", + " print(pd.DataFrame(os.environ))" + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, + "source": [ + "\n", + "## Stakeholder Elicitation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks_tsqr/SAVE_exposurelog.yaml b/notebooks_tsqr/SAVE_exposurelog.yaml new file mode 100644 index 0000000..b6406cb --- /dev/null +++ b/notebooks_tsqr/SAVE_exposurelog.yaml @@ -0,0 +1,16 @@ +# For use with a Times Square notebook +title: Logging and Reporting: Narrative Log +description: Prototype 1 +authors: + - name: Steve Pothier + slack: Steve Pothier +tags: + - reporting + - prototype +parameters: + record_limit: + type: integer + description: Max number of records to output + default: 99 + minimum: 1 + maximum: 9999 diff --git a/notebooks_tsqr/exposurelog.ipynb b/notebooks_tsqr/exposurelog.ipynb new file mode 100644 index 0000000..1daba58 --- /dev/null +++ b/notebooks_tsqr/exposurelog.ipynb @@ -0,0 +1,353 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# Parameters. Set defaults here.\n", + "# Times Square replaces this cell with the user's parameters.\n", + "record_limit = '999'" + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "\n", + "## Imports and General Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "# Only use packages available in the Rubin Science Platform\n", + "import requests\n", + "from collections import defaultdict\n", + "import pandas as pd\n", + "from pprint import pp\n", + "from urllib.parse import urlencode\n", + "from IPython.display import FileLink\n", + "from matplotlib import pyplot as plt\n", + "import os" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'https://tucson-teststand.lsst.codes/exposurelog'" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "\n", + "limit = int(record_limit)\n", + "\n", + "response_timeout = 3.05 # seconds, how long to wait for connection\n", + "read_timeout = 20 # seconds\n", + "timeout = (float(response_timeout), float(read_timeout))\n", + "\n", + "server = os.environ.get('EXTERNAL_INSTANCE_URL', \n", + " 'https://tucson-teststand.lsst.codes')\n", + "log = 'exposurelog'\n", + "service = f'{server}/{log}'\n", + "service" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "\n", + "## Setup Source" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "recs = None\n", + "ok = True\n", + "\n", + "# is_human=either&is_valid=either&offset=0&limit=50' \n", + "# site_ids=tucson&message_text=wubba&min_level=0&max_level=999&user_ids=spothier&user_agents=LOVE\n", + "# tags=love&exclude_tags=ignore_message\n", + "qparams = dict(is_human='either',\n", + " is_valid='either',\n", + " limit=limit,\n", + " )\n", + "qstr = urlencode(qparams)\n", + "url = f'{service}/messages?{qstr}'\n", + "\n", + "ignore_fields = set(['tags', 'urls', 'message_text', 'id', 'date_added', \n", + " 'obs_id', 'day_obs', 'seq_num', 'parent_id', 'user_id',\n", + " 'date_invalidated', 'date_begin', 'date_end',\n", + " 'time_lost', # float\n", + " #'systems','subsystems','cscs', # values are lists, special handling\n", + " ])" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, + "source": [ + "\n", + "## Get Records" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "7", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Attempt to get logs from url='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=either&limit=999'\n", + "Retrieved 0 records, each with numf=0 fields.\n" + ] + } + ], + "source": [ + "try:\n", + " print(f'Attempt to get logs from {url=}')\n", + " response = requests.get(url, timeout=timeout)\n", + " response.raise_for_status()\n", + " recs = response.json()\n", + " flds = set(recs[0].keys())\n", + " facflds = flds - ignore_fields\n", + " # facets(field) = set(value-1, value-2, ...)\n", + " facets = {fld: set([str(r[fld])\n", + " for r in recs if not isinstance(r[fld], list)]) \n", + " for fld in facflds}\n", + "except Exception as err:\n", + " ok = False\n", + " print(f'ERROR getting {log} from {env=} using {url=}: {err=}')\n", + "numf = len(flds) if ok else 0\n", + "numr = len(recs) if ok else 0\n", + "print(f'Retrieved {numr} records, each with {numf=} fields.')" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, + "source": [ + "\n", + "## Tables of (mostly raw) results" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "### Fields names provided in records from log." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "pd.DataFrame(flds, columns=['Field Name'])" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "### Facets from log records.\n", + "A *facet* is the set all of values found for a field in the retrieved records. Facets are only calculated for some fields." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "pd.DataFrame.from_dict(facets, orient='index')" + ] + }, + { + "cell_type": "markdown", + "id": "13", + "metadata": {}, + "source": [ + "### Table of selected log record fields.\n", + "Table can be retrieved as CSV file for local use." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "cols = ['date_added', 'time_lost']\n", + "df = pd.DataFrame(recs)[cols]\n", + "\n", + "# Allow download of CSV version of DataFrame\n", + "csvfile = 'tl.csv'\n", + "df.to_csv(csvfile)\n", + "myfile = FileLink(csvfile)\n", + "print('Table available as CSV file: ')\n", + "display(myfile)\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame(recs)\n", + "df" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "\n", + "## Plots from log" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "x = [r['date_added'] for r in recs]\n", + "y = [r['time_lost'] for r in recs]\n", + "plt.plot(x, y) \n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "\n", + "## Raw Content Analysis" + ] + }, + { + "cell_type": "markdown", + "id": "19", + "metadata": {}, + "source": [ + "### Example of one record" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "rec = recs[0]\n", + "rec" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "msg = rec[\"message_text\"]\n", + "print(msg)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3b745aea-a46a-45e1-bdf2-def878182eb9", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "#EXTERNAL_INSTANCE_URL\n", + "with pd.option_context('display.max_rows', None,):\n", + " print(pd.DataFrame(os.environ))" + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, + "source": [ + "\n", + "## Stakeholder Elicitation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks_tsqr/exposurelog.yaml b/notebooks_tsqr/exposurelog.yaml new file mode 100644 index 0000000..b6406cb --- /dev/null +++ b/notebooks_tsqr/exposurelog.yaml @@ -0,0 +1,16 @@ +# For use with a Times Square notebook +title: Logging and Reporting: Narrative Log +description: Prototype 1 +authors: + - name: Steve Pothier + slack: Steve Pothier +tags: + - reporting + - prototype +parameters: + record_limit: + type: integer + description: Max number of records to output + default: 99 + minimum: 1 + maximum: 9999 diff --git a/notebooks_tsqr/logrep_all_env.ipynb b/notebooks_tsqr/logrep_all_env.ipynb new file mode 100644 index 0000000..3653ea6 --- /dev/null +++ b/notebooks_tsqr/logrep_all_env.ipynb @@ -0,0 +1,804 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "27d224c4-b9b2-43d6-bef4-76a0e17bf2b1", + "metadata": {}, + "source": [ + "# Logging and Reporting" + ] + }, + { + "cell_type": "markdown", + "id": "4e56da66-e40a-4ef2-93ba-fc288a2c60bd", + "metadata": {}, + "source": [ + "## Table of contents\n", + "* [Parameters](#params)\n", + "* [Imports and setup](#imports)\n", + "* [Try every server](#every-server)\n", + "* [Report](#report)" + ] + }, + { + "cell_type": "markdown", + "id": "9bab8290-360c-4784-ba92-f733701dac59", + "metadata": {}, + "source": [ + "\n", + "## Parameters\n", + "The first code cell must contain parameters with string values for compatibility with Times Square.\n", + "\n", + "See: https://rsp.lsst.io/v/usdfdev/guides/times-square/index.html" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "b4278d59-484d-4adc-8f0b-fd2100d9dcdc", + "metadata": {}, + "outputs": [], + "source": [ + "#Parameters\n", + "env = 'tucson' # usdf-dev, tucson, slac, summit\n", + "record_limit = '9999'\n", + "response_timeout = '3.05' # seconds, how long to wait for connection\n", + "read_timeout = '20' # seconds" + ] + }, + { + "cell_type": "markdown", + "id": "1ba64706-2d9a-46b7-9b94-915dc669b7db", + "metadata": {}, + "source": [ + "\n", + "## Imports and General Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "4368be95-2c7d-4111-8056-93237b8fa331", + "metadata": {}, + "outputs": [], + "source": [ + "import requests\n", + "from collections import defaultdict\n", + "import pandas as pd\n", + "from pprint import pp" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "9e701445-ff88-4b42-93e1-4d14452c8a78", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'summit': 'https://summit-lsp.lsst.codes',\n", + " 'usdf_dev': 'https://usdf-rsp-dev.slac.stanford.edu',\n", + " 'tucson': 'https://tucson-teststand.lsst.codes'}" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "limit = int(record_limit)\n", + "timeout = (float(response_timeout), float(read_timeout))\n", + "\n", + "# Env list comes from drop-down menu top of:\n", + "# https://rsp.lsst.io/v/usdfdev/guides/times-square/\n", + "envs = dict(\n", + " #rubin_usdf_dev = '',\n", + " #data_lsst_cloud = '',\n", + " #usdf = '',\n", + " #base_data_facility = '',\n", + " summit = 'https://summit-lsp.lsst.codes',\n", + " usdf_dev = 'https://usdf-rsp-dev.slac.stanford.edu',\n", + " #rubin_idf_int = '',\n", + " tucson = 'https://tucson-teststand.lsst.codes',\n", + ")\n", + "envs" + ] + }, + { + "cell_type": "markdown", + "id": "d83d76bc-3169-45eb-bf7a-32ecf8d75563", + "metadata": {}, + "source": [ + "\n", + "## Try to access every Server, every Log in our list\n", + "We call the combination of a specific Server and specific Log a \"service\".\n", + "This is a First Look. As such, we don't try to get a useful list of records. \n", + "Instead, we save a few pieces of data from each service. A more tailored web-service call should be done to get useful records. For each service, we save:\n", + "1. The number of records retrieved\n", + "1. The list of fields found in a record (we assume all records from a service have the same fields)\n", + "1. An example of 1-2 records.\n", + "1. The [Facets](https://en.wikipedia.org/wiki/Faceted_search) of the service for all service fields that are not explictly excluded." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "d2ee9226-ce4d-43ff-936a-4127c19466ce", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Attempt to get logs from url='https://summit-lsp.lsst.codes/exposurelog/messages?is_human=either&is_valid=either&limit=9999'\n", + "ERROR getting exposurelog from env='summit' using url='https://summit-lsp.lsst.codes/exposurelog/messages?is_human=either&is_valid=either&limit=9999': err=ConnectTimeout(MaxRetryError(\"HTTPSConnectionPool(host='summit-lsp.lsst.codes', port=443): Max retries exceeded with url: /exposurelog/messages?is_human=either&is_valid=either&limit=9999 (Caused by ConnectTimeoutError(, 'Connection to summit-lsp.lsst.codes timed out. (connect timeout=3.05)'))\"))\n", + "\n", + "\n", + "Attempt to get logs from url='https://summit-lsp.lsst.codes/narrativelog/messages?is_human=either&is_valid=either&limit=9999'\n", + "ERROR getting narrativelog from env='summit' using url='https://summit-lsp.lsst.codes/narrativelog/messages?is_human=either&is_valid=either&limit=9999': err=ConnectTimeout(MaxRetryError(\"HTTPSConnectionPool(host='summit-lsp.lsst.codes', port=443): Max retries exceeded with url: /narrativelog/messages?is_human=either&is_valid=either&limit=9999 (Caused by ConnectTimeoutError(, 'Connection to summit-lsp.lsst.codes timed out. (connect timeout=3.05)'))\"))\n", + "\n", + "Attempt to get logs from url='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=either&limit=9999'\n", + "\n", + "\n", + "Attempt to get logs from url='https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=either&limit=9999'\n", + "\n", + "Attempt to get logs from url='https://tucson-teststand.lsst.codes/exposurelog/messages?is_human=either&is_valid=either&limit=9999'\n", + "\n", + "\n", + "Attempt to get logs from url='https://tucson-teststand.lsst.codes/narrativelog/messages?is_human=either&is_valid=either&limit=9999'\n" + ] + } + ], + "source": [ + "verbose=False\n", + "fields = defaultdict(set) # fields[(env,log)] = {field1, field2, ...}\n", + "examples = defaultdict(list) # examples[(env,log)] = [rec1, rec2]\n", + "results = defaultdict(dict) # results[(env,log)] = dict(server,url, ok, numfields, numrecs)\n", + "facets = defaultdict(dict) # facets[(env,log)] = dict(field) = set(value-1, value-2, ...)\n", + "\n", + "# Dumb! Using same ignore set for all LOGS.\n", + "ignore_fields = set(['tags', 'urls', 'message_text', 'id', 'date_added', \n", + " 'obs_id', 'day_obs', 'seq_num', 'parent_id', 'user_id',\n", + " 'date_invalidated', 'date_begin', 'date_end',\n", + " 'time_lost', # float\n", + " #'systems','subsystems','cscs', # values are lists, special handling\n", + " ])\n", + "for env,server in envs.items():\n", + " ok = True\n", + " try:\n", + " recs = None\n", + " log = 'exposurelog'\n", + " #!url = f'{server}/{log}/messages?is_human=either&is_valid=either&offset=0&{limit=}'\n", + " url = f'{server}/{log}/messages?is_human=either&is_valid=either&{limit=}'\n", + " print(f'\\nAttempt to get logs from {url=}')\n", + " response = requests.get(url, timeout=timeout)\n", + " response.raise_for_status()\n", + " recs = response.json()\n", + " flds = set(recs[0].keys())\n", + " if verbose:\n", + " print(f'Number of {log} records: {len(recs):,}')\n", + " print(f'Got {log} fields: {flds}')\n", + " print(f'Example record: {recs[0]}') \n", + " fields[(env,log)] = flds\n", + " examples[(env,log)] = recs[:2] \n", + "\n", + " facflds = flds - ignore_fields\n", + " # Fails when r[fld] is a LIST instead of singleton\n", + " # I think when that happens occasionaly, its a BUG in the data! It happens.\n", + " facets[(env,log)] = {fld: set([str(r[fld])\n", + " for r in recs if not isinstance(r[fld], list)]) \n", + " for fld in facflds}\n", + " except Exception as err:\n", + " ok = False\n", + " print(f'ERROR getting {log} from {env=} using {url=}: {err=}')\n", + " numf = len(flds) if ok else 0\n", + " numr = len(recs) if ok else 0\n", + " results[(env,log)] = dict(ok=ok, server=server, url=url,numfields=numf, numrecs=numr)\n", + "\n", + " print()\n", + " try:\n", + " recs = None\n", + " log = 'narrativelog'\n", + " #! url = f'{server}/{log}/messages?is_human=either&is_valid=true&offset=0&{limit=}'\n", + " url = f'{server}/{log}/messages?is_human=either&is_valid=either&{limit=}'\n", + " print(f'\\nAttempt to get logs from {url=}')\n", + " response = requests.get(url, timeout=timeout)\n", + " response.raise_for_status()\n", + " recs = response.json()\n", + " flds = set(recs[0].keys())\n", + " if verbose:\n", + " print(f'Number of {log} records: {len(recs):,}')\n", + " print(f'Got {log} fields: {flds}')\n", + " print(f'Example record: {recs[0]}')\n", + " fields[(env,log)] = flds \n", + " examples[(env,log)] = recs[:2] \n", + "\n", + " facflds = flds - ignore_fields\n", + " # Fails when r[fld] is a LIST instead of singleton\n", + " # I think when that happens occasionaly, its a BUG in the data! It happens.\n", + " # Look for BAD facet values like: {'None', None}\n", + " facets[(env,log)] = {fld: set([r[fld] \n", + " for r in recs if not isinstance(r[fld], list)]) \n", + " for fld in facflds}\n", + " except Exception as err:\n", + " ok = False\n", + " print(f'ERROR getting {log} from {env=} using {url=}: {err=}')\n", + " numf = len(flds) if ok else 0\n", + " numr = len(recs) if ok else 0\n", + " results[(env,log)] = dict(ok=ok, server=server, url=url,numfields=numf, numrecs=numr)" + ] + }, + { + "cell_type": "markdown", + "id": "c59da227-15e7-4f10-9509-405a9d1c230d", + "metadata": {}, + "source": [ + "\n", + "## Report\n", + "This is a silly report that may be useful for developers. Not so much for astronomers." + ] + }, + { + "cell_type": "markdown", + "id": "402a093d-4443-468f-85a4-b82ee6d3ab06", + "metadata": {}, + "source": [ + "\n", + "### Success/Failure table" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "147c066a-bc3a-4c0e-8499-eaa2f1ec9f4c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Got results from 4 of 6 env/logs\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
okservernumfieldsnumrecs
summitexposurelogFalsehttps://summit-lsp.lsst.codes00
narrativelogFalsehttps://summit-lsp.lsst.codes00
usdf_devexposurelogTruehttps://usdf-rsp-dev.slac.stanford.edu187895
narrativelogTruehttps://usdf-rsp-dev.slac.stanford.edu247597
tucsonexposurelogTruehttps://tucson-teststand.lsst.codes1820
narrativelogTruehttps://tucson-teststand.lsst.codes24110
\n", + "
" + ], + "text/plain": [ + " ok server \\\n", + "summit exposurelog False https://summit-lsp.lsst.codes \n", + " narrativelog False https://summit-lsp.lsst.codes \n", + "usdf_dev exposurelog True https://usdf-rsp-dev.slac.stanford.edu \n", + " narrativelog True https://usdf-rsp-dev.slac.stanford.edu \n", + "tucson exposurelog True https://tucson-teststand.lsst.codes \n", + " narrativelog True https://tucson-teststand.lsst.codes \n", + "\n", + " numfields numrecs \n", + "summit exposurelog 0 0 \n", + " narrativelog 0 0 \n", + "usdf_dev exposurelog 18 7895 \n", + " narrativelog 24 7597 \n", + "tucson exposurelog 18 20 \n", + " narrativelog 24 110 " + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "show_columns = ['ok', 'server', 'numfields', 'numrecs']\n", + "df = pd.DataFrame(data=dict(results)).T.loc[:,show_columns]\n", + "print(f'Got results from {df[\"ok\"].values.sum()} of {len(df)} env/logs')\n", + "df" + ] + }, + { + "cell_type": "markdown", + "id": "f6ce4545-74d6-4551-b125-44c7c4021c19", + "metadata": {}, + "source": [ + "\n", + "### Field Names" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "0abedbf3-5976-4713-b5ab-f953396dbcff", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Field names for each Environment/Log source:\n", + "\n", + "usdf_dev/exposurelog: parent_id, instrument, obs_id, user_id, site_id, user_agent, is_human, date_added, message_text, level, tags, day_obs, exposure_flag, is_valid, seq_num, urls, date_invalidated, id\n", + "\n", + "usdf_dev/narrativelog: systems, date_end, message_text, primary_hardware_components, user_agent, site_id, is_human, cscs, category, is_valid, date_invalidated, time_lost_type, tags, urls, time_lost, components, date_begin, parent_id, subsystems, user_id, date_added, level, primary_software_components, id\n", + "\n", + "tucson/exposurelog: parent_id, instrument, obs_id, user_id, site_id, user_agent, is_human, date_added, message_text, level, tags, day_obs, exposure_flag, is_valid, seq_num, urls, date_invalidated, id\n", + "\n", + "tucson/narrativelog: systems, date_end, message_text, primary_hardware_components, user_agent, site_id, is_human, cscs, category, is_valid, date_invalidated, time_lost_type, tags, urls, time_lost, components, date_begin, parent_id, subsystems, user_id, date_added, level, primary_software_components, id\n" + ] + } + ], + "source": [ + "print('Field names for each Environment/Log source:')\n", + "for (env,log),flds in fields.items():\n", + " field_names = ', '.join(flds)\n", + " print(f'\\n{env}/{log}: {field_names}')\n", + "#!dict(fields)" + ] + }, + { + "cell_type": "markdown", + "id": "557803ba-3f59-432a-9d39-efca609a1f97", + "metadata": {}, + "source": [ + "\n", + "### Facets" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "4d98ac00-6694-47e9-a53e-5c5dfef96948", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "usdf_dev/exposurelog:\n", + " instrument: \t{'LSSTComCam', 'LATISS'}\n", + " user_agent: \t{'notebook:nublado', 'LOVE'}\n", + " is_human: \t{'True'}\n", + " site_id: \t{'summit'}\n", + " level: \t{'20', '10'}\n", + " exposure_flag: \t{'junk', 'none', 'questionable'}\n", + " is_valid: \t{'False', 'True'}\n", + "usdf_dev/narrativelog:\n", + " subsystems: \t{None}\n", + " systems: \t{None}\n", + " user_agent: \t{'LOVE', 'string'}\n", + " is_human: \t{True}\n", + " site_id: \t{'summit'}\n", + " cscs: \t{None}\n", + " time_lost_type: \t{'weather', 'fault', None}\n", + " level: \t{0, 100}\n", + " primary_software_components: \t{None}\n", + " primary_hardware_components: \t{None}\n", + " category: \t{'None', 'SCIENCE', 'ENG', None}\n", + " is_valid: \t{False, True}\n", + " components: \t{None}\n", + "tucson/exposurelog:\n", + " instrument: \t{'LSSTComCam', 'LATISS'}\n", + " user_agent: \t{'LOVE'}\n", + " is_human: \t{'True'}\n", + " site_id: \t{'tucson'}\n", + " level: \t{'10'}\n", + " exposure_flag: \t{'junk', 'none', 'questionable'}\n", + " is_valid: \t{'False', 'True'}\n", + "tucson/narrativelog:\n", + " subsystems: \t{None}\n", + " systems: \t{None}\n", + " user_agent: \t{'LOVE', 'string'}\n", + " is_human: \t{False, True}\n", + " site_id: \t{'tucson'}\n", + " cscs: \t{None}\n", + " time_lost_type: \t{'fault', None}\n", + " level: \t{0, 100}\n", + " primary_software_components: \t{None}\n", + " primary_hardware_components: \t{None}\n", + " category: \t{'None', None}\n", + " is_valid: \t{False, True}\n", + " components: \t{None}\n" + ] + } + ], + "source": [ + "dict(facets)\n", + "for (env,log),flds in facets.items():\n", + " print(f'{env}/{log}:')\n", + " for fld,vals in flds.items():\n", + " print(f' {fld}: \\t{vals}')" + ] + }, + { + "cell_type": "markdown", + "id": "fb8c376d-13aa-4740-bd01-9c565b331bab", + "metadata": {}, + "source": [ + "\n", + "### Example Records" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "a4a4725b-ff92-44f2-b6eb-13c52c14b5da", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "env='usdf_dev', log='exposurelog': \n", + " Example records: \n", + "[{'id': '0005ebfa-1832-4e53-bc7a-a69254deef88',\n", + " 'site_id': 'summit',\n", + " 'obs_id': 'AT_O_20230118_000113',\n", + " 'instrument': 'LATISS',\n", + " 'day_obs': 20230118,\n", + " 'seq_num': 113,\n", + " 'message_text': ' ',\n", + " 'level': 20,\n", + " 'tags': [],\n", + " 'urls': [],\n", + " 'user_id': 'dsanmartim',\n", + " 'user_agent': 'notebook:nublado',\n", + " 'is_human': True,\n", + " 'is_valid': False,\n", + " 'exposure_flag': 'junk',\n", + " 'date_added': '2023-01-19T07:26:28.887152',\n", + " 'date_invalidated': '2023-01-19T07:32:32.435376',\n", + " 'parent_id': '18345e26-3c8c-4472-864f-68257ced64f4'},\n", + " {'id': '0007f3e6-ac70-4cf3-8edd-70d4a311aa4e',\n", + " 'site_id': 'summit',\n", + " 'obs_id': 'AT_O_20230118_000215',\n", + " 'instrument': 'LATISS',\n", + " 'day_obs': 20230118,\n", + " 'seq_num': 215,\n", + " 'message_text': '',\n", + " 'level': 20,\n", + " 'tags': [],\n", + " 'urls': [],\n", + " 'user_id': 'dsanmartim',\n", + " 'user_agent': 'notebook:nublado',\n", + " 'is_human': True,\n", + " 'is_valid': False,\n", + " 'exposure_flag': 'none',\n", + " 'date_added': '2023-01-19T07:00:11.295541',\n", + " 'date_invalidated': '2023-01-19T07:32:03.334049',\n", + " 'parent_id': 'cde04ee9-01b3-4428-8e4a-09c5bb25a0e3'}]\n", + "\n", + "env='usdf_dev', log='narrativelog': \n", + " Example records: \n", + "[{'id': '0005d0e0-6de7-4857-b658-cfca7147374a',\n", + " 'site_id': 'summit',\n", + " 'message_text': 'The moveShutterMainDoor command Fail\\r\\n'\n", + " '\\r\\n'\n", + " '\\r\\n'\n", + " 'we try the parameter with uppercase now\\r\\n'\n", + " '\\r\\n'\n", + " '\\r\\n'\n", + " ' open: False\\r\\n'\n", + " '\\r\\n'\n", + " '\\r\\n'\n", + " '\\r\\n'\n", + " '\\r\\n'\n", + " 'Error in run\\r\\n'\n", + " 'Traceback (most recent call last):\\r\\n'\n", + " ' File '\n", + " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", '\n", + " 'line 500, in wait_for\\r\\n'\n", + " ' return fut.result()\\r\\n'\n", + " ' ^^^^^^^^^^^^\\r\\n'\n", + " ' File '\n", + " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", '\n", + " 'line 239, in _get_next_ackcmd\\r\\n'\n", + " ' await self._next_ack_event.wait()\\r\\n'\n", + " ' File '\n", + " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/locks.py\", '\n", + " 'line 213, in wait\\r\\n'\n", + " ' await fut\\r\\n'\n", + " 'asyncio.exceptions.CancelledError\\r\\n'\n", + " '\\r\\n'\n", + " '\\r\\n'\n", + " 'The above exception was the direct cause of the following '\n", + " 'exception:\\r\\n'\n", + " '\\r\\n'\n", + " '\\r\\n'\n", + " 'Traceback (most recent call last):\\r\\n'\n", + " ' File '\n", + " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", '\n", + " 'line 189, in next_ackcmd\\r\\n'\n", + " ' ackcmd = await self._wait_task\\r\\n'\n", + " ' ^^^^^^^^^^^^^^^^^^^^^\\r\\n'\n", + " ' File '\n", + " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", '\n", + " 'line 214, in _basic_next_ackcmd\\r\\n'\n", + " ' ackcmd = await asyncio.wait_for(\\r\\n'\n", + " ' ^^^^^^^^^^^^^^^^^^^^^^^\\r\\n'\n", + " ' File '\n", + " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", '\n", + " 'line 502, in wait_for\\r\\n'\n", + " ' raise exceptions.TimeoutError() from exc\\r\\n'\n", + " 'TimeoutError\\r\\n'\n", + " '\\r\\n'\n", + " '\\r\\n'\n", + " 'During handling of the above exception, another exception '\n", + " 'occurred:\\r\\n'\n", + " '\\r\\n'\n", + " '\\r\\n'\n", + " 'Traceback (most recent call last):\\r\\n'\n", + " ' File '\n", + " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", '\n", + " 'line 603, in do_run\\r\\n'\n", + " ' await self._run_task\\r\\n'\n", + " ' File '\n", + " '\"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/run_command.py\", '\n", + " 'line 170, in run\\r\\n'\n", + " ' await getattr(self.remote, f\"cmd_{self.cmd}\").start(\\r\\n'\n", + " ' File '\n", + " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", '\n", + " 'line 487, in start\\r\\n'\n", + " ' return await cmd_info.next_ackcmd(timeout=timeout)\\r\\n'\n", + " ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\r\\n'\n", + " ' File '\n", + " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", '\n", + " 'line 205, in next_ackcmd\\r\\n'\n", + " ' raise base.AckTimeoutError(\\r\\n'\n", + " \"lsst.ts.salobj.base.AckTimeoutError: msg='Timed out waiting \"\n", + " \"for command acknowledgement', ackcmd=(ackcmd \"\n", + " 'private_seqNum=1142844592, ack=<SalRetCode.CMD_NOACK: '\n", + " \"-301>, error=0, result='No command acknowledgement \"\n", + " \"seen')\\r\\n\",\n", + " 'level': 0,\n", + " 'tags': [],\n", + " 'urls': [],\n", + " 'time_lost': 0.0,\n", + " 'date_begin': '2024-03-20T04:15:26.769000',\n", + " 'user_id': 'cmorales@summit-lsp.lsst.codes',\n", + " 'user_agent': 'LOVE',\n", + " 'is_human': True,\n", + " 'is_valid': True,\n", + " 'date_added': '2024-03-20T04:16:06.976595',\n", + " 'date_invalidated': None,\n", + " 'parent_id': None,\n", + " 'systems': None,\n", + " 'subsystems': None,\n", + " 'cscs': None,\n", + " 'date_end': '2024-03-20T04:15:26.769000',\n", + " 'components': ['AuxTel'],\n", + " 'primary_software_components': ['None'],\n", + " 'primary_hardware_components': ['None'],\n", + " 'category': 'None',\n", + " 'time_lost_type': 'fault'},\n", + " {'id': '00067b6a-1741-41f3-87af-57261dff30ac',\n", + " 'site_id': 'summit',\n", + " 'message_text': 'David updated environment. But no need to cycle Scheduler '\n", + " '1. Waiting for completion of the current block. \\r\\n',\n", + " 'level': 0,\n", + " 'tags': [],\n", + " 'urls': [],\n", + " 'time_lost': 0.0,\n", + " 'date_begin': '2024-04-24T20:53:41.843000',\n", + " 'user_id': 'ykang@summit-lsp.lsst.codes',\n", + " 'user_agent': 'LOVE',\n", + " 'is_human': True,\n", + " 'is_valid': True,\n", + " 'date_added': '2024-04-24T20:54:19.346646',\n", + " 'date_invalidated': None,\n", + " 'parent_id': None,\n", + " 'systems': None,\n", + " 'subsystems': None,\n", + " 'cscs': None,\n", + " 'date_end': '2024-04-24T20:53:41.843000',\n", + " 'components': ['MainTel'],\n", + " 'primary_software_components': ['None'],\n", + " 'primary_hardware_components': ['None'],\n", + " 'category': 'None',\n", + " 'time_lost_type': 'fault'}]\n", + "\n", + "env='tucson', log='exposurelog': \n", + " Example records: \n", + "[{'id': '0076ce05-752f-46df-b8a5-90ffd51d612e',\n", + " 'site_id': 'tucson',\n", + " 'obs_id': 'AT_O_20221129_000001',\n", + " 'instrument': 'LSSTComCam',\n", + " 'day_obs': 20221129,\n", + " 'seq_num': 1,\n", + " 'message_text': 'this was a test image for LOVE LVV November test cycle',\n", + " 'level': 10,\n", + " 'tags': ['undefined'],\n", + " 'urls': [],\n", + " 'user_id': 'admin@love1.tu.lsst.org',\n", + " 'user_agent': 'LOVE',\n", + " 'is_human': True,\n", + " 'is_valid': True,\n", + " 'exposure_flag': 'junk',\n", + " 'date_added': '2022-11-30T02:13:46.847038',\n", + " 'date_invalidated': None,\n", + " 'parent_id': None},\n", + " {'id': '1e61fbbe-cb87-45f2-88be-cb6647b0cd43',\n", + " 'site_id': 'tucson',\n", + " 'obs_id': 'AT_C_20230613_000001',\n", + " 'instrument': 'LATISS',\n", + " 'day_obs': 20230613,\n", + " 'seq_num': 1,\n", + " 'message_text': 'Exposure log creation test',\n", + " 'level': 10,\n", + " 'tags': ['undefined'],\n", + " 'urls': [],\n", + " 'user_id': 'admin@love1.tu.lsst.org',\n", + " 'user_agent': 'LOVE',\n", + " 'is_human': True,\n", + " 'is_valid': False,\n", + " 'exposure_flag': 'none',\n", + " 'date_added': '2023-07-18T22:13:41.228510',\n", + " 'date_invalidated': '2023-07-18T22:15:14.006037',\n", + " 'parent_id': None}]\n", + "\n", + "env='tucson', log='narrativelog': \n", + " Example records: \n", + "[{'id': '01806d0e-c21a-4e2e-8a53-1db4e8f88c0b',\n", + " 'site_id': 'tucson',\n", + " 'message_text': 'Wubba. Using do_makeLogEntry; vr=0.1',\n", + " 'level': 0,\n", + " 'tags': ['watcher', 'alarm'],\n", + " 'urls': [],\n", + " 'time_lost': 0.0,\n", + " 'date_begin': '2024-08-15T12:44:06.456000',\n", + " 'user_id': 'admin@love1.tu.lsst.org',\n", + " 'user_agent': 'LOVE',\n", + " 'is_human': False,\n", + " 'is_valid': True,\n", + " 'date_added': '2024-08-15T17:54:06.567418',\n", + " 'date_invalidated': None,\n", + " 'parent_id': None,\n", + " 'systems': ['ObservatorySoftwareTools'],\n", + " 'subsystems': ['LOVE'],\n", + " 'cscs': [],\n", + " 'date_end': '2024-08-15T12:44:06.456000',\n", + " 'components': None,\n", + " 'primary_software_components': None,\n", + " 'primary_hardware_components': None,\n", + " 'category': None,\n", + " 'time_lost_type': None},\n", + " {'id': '05b046b7-1266-47a9-b1de-93b6d44af242',\n", + " 'site_id': 'tucson',\n", + " 'message_text': '',\n", + " 'level': 0,\n", + " 'tags': [],\n", + " 'urls': [],\n", + " 'time_lost': 0.0,\n", + " 'date_begin': '2023-01-19T22:50:03.167000',\n", + " 'user_id': 'admin@love1.tu.lsst.org',\n", + " 'user_agent': 'LOVE',\n", + " 'is_human': True,\n", + " 'is_valid': True,\n", + " 'date_added': '2023-01-19T22:57:39.561293',\n", + " 'date_invalidated': None,\n", + " 'parent_id': None,\n", + " 'systems': [],\n", + " 'subsystems': [],\n", + " 'cscs': [],\n", + " 'date_end': '2023-01-19T22:50:03.167000',\n", + " 'components': None,\n", + " 'primary_software_components': None,\n", + " 'primary_hardware_components': None,\n", + " 'category': None,\n", + " 'time_lost_type': None}]\n" + ] + } + ], + "source": [ + "for (env,log),recs in examples.items():\n", + " print(f'\\n{env=}, {log=}: ')\n", + " print(' Example records: ')\n", + " pp(recs)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c59d4f72-4bdb-4613-b634-26c2bb315221", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks_tsqr/logrep_proto_1.ipynb b/notebooks_tsqr/logrep_proto_1.ipynb index b767518..03549c9 100644 --- a/notebooks_tsqr/logrep_proto_1.ipynb +++ b/notebooks_tsqr/logrep_proto_1.ipynb @@ -25,11 +25,7 @@ "cell_type": "code", "execution_count": null, "id": "2", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, "outputs": [], "source": [ "# Only use packages available in the Rubin Science Platform\n", diff --git a/notebooks_tsqr/narrativelog.ipynb b/notebooks_tsqr/narrativelog.ipynb new file mode 100644 index 0000000..b767518 --- /dev/null +++ b/notebooks_tsqr/narrativelog.ipynb @@ -0,0 +1,345 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "# Parameters. Set defaults here.\n", + "# Times Square replaces this cell with the user's parameters.\n", + "record_limit = '999'" + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "\n", + "## Imports and General Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": { + "jupyter": { + "source_hidden": true + } + }, + "outputs": [], + "source": [ + "# Only use packages available in the Rubin Science Platform\n", + "import requests\n", + "from collections import defaultdict\n", + "import pandas as pd\n", + "from pprint import pp\n", + "from urllib.parse import urlencode\n", + "from IPython.display import FileLink\n", + "from matplotlib import pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "env = 'usdf_dev' # usdf-dev, tucson, slac, summit\n", + "log_name = 'narrativelog'\n", + "log = log_name\n", + "limit = int(record_limit)\n", + "response_timeout = 3.05 # seconds, how long to wait for connection\n", + "read_timeout = 20 # seconds\n", + "\n", + "timeout = (float(response_timeout), float(read_timeout))\n", + "\n", + "# Env list comes from drop-down menu top of:\n", + "# https://rsp.lsst.io/v/usdfdev/guides/times-square/\n", + "envs = dict(\n", + " #rubin_usdf_dev = '',\n", + " #data_lsst_cloud = '',\n", + " #usdf = '',\n", + " #base_data_facility = '',\n", + " summit = 'https://summit-lsp.lsst.codes',\n", + " usdf_dev = 'https://usdf-rsp-dev.slac.stanford.edu',\n", + " #rubin_idf_int = '',\n", + " tucson = 'https://tucson-teststand.lsst.codes',\n", + ")\n", + "server = envs[env]\n", + "service = f'{server}/{log}'\n", + "service" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "\n", + "## Setup Source" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": { + "jupyter": { + "source_hidden": true + } + }, + "outputs": [], + "source": [ + "recs = None\n", + "ok = True\n", + "\n", + "# is_human=either&is_valid=either&offset=0&limit=50' \n", + "# site_ids=tucson&message_text=wubba&min_level=0&max_level=999&user_ids=spothier&user_agents=LOVE\n", + "# tags=love&exclude_tags=ignore_message\n", + "qparams = dict(is_human='either',\n", + " is_valid='either',\n", + " limit=limit,\n", + " )\n", + "qstr = urlencode(qparams)\n", + "url = f'{service}/messages?{qstr}'\n", + "\n", + "ignore_fields = set(['tags', 'urls', 'message_text', 'id', 'date_added', \n", + " 'obs_id', 'day_obs', 'seq_num', 'parent_id', 'user_id',\n", + " 'date_invalidated', 'date_begin', 'date_end',\n", + " 'time_lost', # float\n", + " #'systems','subsystems','cscs', # values are lists, special handling\n", + " ])" + ] + }, + { + "cell_type": "markdown", + "id": "6", + "metadata": {}, + "source": [ + "\n", + "## Get Records" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "# TODO Often fails on first request. Find out why!\n", + "try:\n", + " response = requests.get(url, timeout=timeout)\n", + "except:\n", + " pass \n", + " \n", + "try:\n", + " print(f'Attempt to get logs from {url=}')\n", + " response = requests.get(url, timeout=timeout)\n", + " response.raise_for_status()\n", + " recs = response.json()\n", + " flds = set(recs[0].keys())\n", + " facflds = flds - ignore_fields\n", + " # facets(field) = set(value-1, value-2, ...)\n", + " facets = {fld: set([str(r[fld])\n", + " for r in recs if not isinstance(r[fld], list)]) \n", + " for fld in facflds}\n", + "except Exception as err:\n", + " ok = False\n", + " print(f'ERROR getting {log} from {env=} using {url=}: {err=}')\n", + "numf = len(flds) if ok else 0\n", + "numr = len(recs) if ok else 0\n", + "print(f'Retrieved {numr} records, each with {numf=} fields.')" + ] + }, + { + "cell_type": "markdown", + "id": "8", + "metadata": {}, + "source": [ + "\n", + "## Tables of (mostly raw) results" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "### Fields names provided in records from log." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "pd.DataFrame(flds, columns=['Field Name'])" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "### Facets from log records.\n", + "A *facet* is the set all of values found for a field in the retrieved records. Facets are only calculated for some fields." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "pd.DataFrame.from_dict(facets, orient='index')" + ] + }, + { + "cell_type": "markdown", + "id": "13", + "metadata": {}, + "source": [ + "### Table of selected log record fields.\n", + "Table can be retrieved as CSV file for local use." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "14", + "metadata": {}, + "outputs": [], + "source": [ + "cols = ['date_added', 'time_lost']\n", + "df = pd.DataFrame(recs)[cols]\n", + "\n", + "# Allow download of CSV version of DataFrame\n", + "csvfile = 'tl.csv'\n", + "df.to_csv(csvfile)\n", + "myfile = FileLink(csvfile)\n", + "print('Table available as CSV file: ')\n", + "display(myfile)\n", + "df" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "df = pd.DataFrame(recs)\n", + "df" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "\n", + "## Plots from log" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "x = [r['date_added'] for r in recs]\n", + "y = [r['time_lost'] for r in recs]\n", + "plt.plot(x, y) \n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "\n", + "## Raw Content Analysis" + ] + }, + { + "cell_type": "markdown", + "id": "19", + "metadata": {}, + "source": [ + "### Example of one record" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "20", + "metadata": {}, + "outputs": [], + "source": [ + "rec = recs[0]\n", + "rec" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "msg = rec[\"message_text\"]\n", + "print(msg)" + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, + "source": [ + "\n", + "## Stakeholder Elicitation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/notebooks_tsqr/narrativelog.yaml b/notebooks_tsqr/narrativelog.yaml new file mode 100644 index 0000000..b6406cb --- /dev/null +++ b/notebooks_tsqr/narrativelog.yaml @@ -0,0 +1,16 @@ +# For use with a Times Square notebook +title: Logging and Reporting: Narrative Log +description: Prototype 1 +authors: + - name: Steve Pothier + slack: Steve Pothier +tags: + - reporting + - prototype +parameters: + record_limit: + type: integer + description: Max number of records to output + default: 99 + minimum: 1 + maximum: 9999 diff --git a/times-square.yaml b/times-square.yaml index 3e6e1d5..e4692c5 100644 --- a/times-square.yaml +++ b/times-square.yaml @@ -1,4 +1,5 @@ enabled: true root: notebooks_tsqr +ignore: SAVE* description: > Times Square for project-wide Logging and Reporting. From 52c288ed0c498493509649d7597c0dc54a3c3647 Mon Sep 17 00:00:00 2001 From: Steve Pothier Date: Tue, 27 Aug 2024 14:27:17 -0700 Subject: [PATCH 09/10] WIP --- notebooks_tsqr/SAVE_exposurelog.ipynb | 40 +- notebooks_tsqr/SAVE_exposurelog.yaml | 2 +- notebooks_tsqr/exposurelog.ipynb | 40 +- notebooks_tsqr/exposurelog.yaml | 2 +- notebooks_tsqr/logrep_all_env.ipynb | 543 ++------------------------ notebooks_tsqr/narrativelog.yaml | 2 +- 6 files changed, 56 insertions(+), 573 deletions(-) diff --git a/notebooks_tsqr/SAVE_exposurelog.ipynb b/notebooks_tsqr/SAVE_exposurelog.ipynb index 1daba58..a29966d 100644 --- a/notebooks_tsqr/SAVE_exposurelog.ipynb +++ b/notebooks_tsqr/SAVE_exposurelog.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "0", "metadata": {}, "outputs": [], @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "2", "metadata": {}, "outputs": [], @@ -41,21 +41,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "3", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'https://tucson-teststand.lsst.codes/exposurelog'" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "\n", "limit = int(record_limit)\n", @@ -82,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "5", "metadata": {}, "outputs": [], @@ -119,19 +108,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "7", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Attempt to get logs from url='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=either&limit=999'\n", - "Retrieved 0 records, each with numf=0 fields.\n" - ] - } - ], + "outputs": [], "source": [ "try:\n", " print(f'Attempt to get logs from {url=}')\n", @@ -301,7 +281,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3b745aea-a46a-45e1-bdf2-def878182eb9", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -313,7 +293,7 @@ }, { "cell_type": "markdown", - "id": "22", + "id": "23", "metadata": {}, "source": [ "\n", @@ -323,7 +303,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "24", "metadata": {}, "outputs": [], "source": [] diff --git a/notebooks_tsqr/SAVE_exposurelog.yaml b/notebooks_tsqr/SAVE_exposurelog.yaml index b6406cb..a26b1dc 100644 --- a/notebooks_tsqr/SAVE_exposurelog.yaml +++ b/notebooks_tsqr/SAVE_exposurelog.yaml @@ -1,5 +1,5 @@ # For use with a Times Square notebook -title: Logging and Reporting: Narrative Log +title: Logging and Reporting for Narrative Log description: Prototype 1 authors: - name: Steve Pothier diff --git a/notebooks_tsqr/exposurelog.ipynb b/notebooks_tsqr/exposurelog.ipynb index 1daba58..a29966d 100644 --- a/notebooks_tsqr/exposurelog.ipynb +++ b/notebooks_tsqr/exposurelog.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "0", "metadata": {}, "outputs": [], @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "id": "2", "metadata": {}, "outputs": [], @@ -41,21 +41,10 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "id": "3", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'https://tucson-teststand.lsst.codes/exposurelog'" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "\n", "limit = int(record_limit)\n", @@ -82,7 +71,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "id": "5", "metadata": {}, "outputs": [], @@ -119,19 +108,10 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "id": "7", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Attempt to get logs from url='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=either&limit=999'\n", - "Retrieved 0 records, each with numf=0 fields.\n" - ] - } - ], + "outputs": [], "source": [ "try:\n", " print(f'Attempt to get logs from {url=}')\n", @@ -301,7 +281,7 @@ { "cell_type": "code", "execution_count": null, - "id": "3b745aea-a46a-45e1-bdf2-def878182eb9", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -313,7 +293,7 @@ }, { "cell_type": "markdown", - "id": "22", + "id": "23", "metadata": {}, "source": [ "\n", @@ -323,7 +303,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "24", "metadata": {}, "outputs": [], "source": [] diff --git a/notebooks_tsqr/exposurelog.yaml b/notebooks_tsqr/exposurelog.yaml index b6406cb..a26b1dc 100644 --- a/notebooks_tsqr/exposurelog.yaml +++ b/notebooks_tsqr/exposurelog.yaml @@ -1,5 +1,5 @@ # For use with a Times Square notebook -title: Logging and Reporting: Narrative Log +title: Logging and Reporting for Narrative Log description: Prototype 1 authors: - name: Steve Pothier diff --git a/notebooks_tsqr/logrep_all_env.ipynb b/notebooks_tsqr/logrep_all_env.ipynb index 3653ea6..4036bdf 100644 --- a/notebooks_tsqr/logrep_all_env.ipynb +++ b/notebooks_tsqr/logrep_all_env.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "27d224c4-b9b2-43d6-bef4-76a0e17bf2b1", + "id": "0", "metadata": {}, "source": [ "# Logging and Reporting" @@ -10,7 +10,7 @@ }, { "cell_type": "markdown", - "id": "4e56da66-e40a-4ef2-93ba-fc288a2c60bd", + "id": "1", "metadata": {}, "source": [ "## Table of contents\n", @@ -22,7 +22,7 @@ }, { "cell_type": "markdown", - "id": "9bab8290-360c-4784-ba92-f733701dac59", + "id": "2", "metadata": {}, "source": [ "\n", @@ -34,8 +34,8 @@ }, { "cell_type": "code", - "execution_count": 1, - "id": "b4278d59-484d-4adc-8f0b-fd2100d9dcdc", + "execution_count": null, + "id": "3", "metadata": {}, "outputs": [], "source": [ @@ -48,7 +48,7 @@ }, { "cell_type": "markdown", - "id": "1ba64706-2d9a-46b7-9b94-915dc669b7db", + "id": "4", "metadata": {}, "source": [ "\n", @@ -57,8 +57,8 @@ }, { "cell_type": "code", - "execution_count": 2, - "id": "4368be95-2c7d-4111-8056-93237b8fa331", + "execution_count": null, + "id": "5", "metadata": {}, "outputs": [], "source": [ @@ -70,23 +70,10 @@ }, { "cell_type": "code", - "execution_count": 3, - "id": "9e701445-ff88-4b42-93e1-4d14452c8a78", + "execution_count": null, + "id": "6", "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'summit': 'https://summit-lsp.lsst.codes',\n", - " 'usdf_dev': 'https://usdf-rsp-dev.slac.stanford.edu',\n", - " 'tucson': 'https://tucson-teststand.lsst.codes'}" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "limit = int(record_limit)\n", "timeout = (float(response_timeout), float(read_timeout))\n", @@ -108,7 +95,7 @@ }, { "cell_type": "markdown", - "id": "d83d76bc-3169-45eb-bf7a-32ecf8d75563", + "id": "7", "metadata": {}, "source": [ "\n", @@ -124,34 +111,10 @@ }, { "cell_type": "code", - "execution_count": 4, - "id": "d2ee9226-ce4d-43ff-936a-4127c19466ce", + "execution_count": null, + "id": "8", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Attempt to get logs from url='https://summit-lsp.lsst.codes/exposurelog/messages?is_human=either&is_valid=either&limit=9999'\n", - "ERROR getting exposurelog from env='summit' using url='https://summit-lsp.lsst.codes/exposurelog/messages?is_human=either&is_valid=either&limit=9999': err=ConnectTimeout(MaxRetryError(\"HTTPSConnectionPool(host='summit-lsp.lsst.codes', port=443): Max retries exceeded with url: /exposurelog/messages?is_human=either&is_valid=either&limit=9999 (Caused by ConnectTimeoutError(, 'Connection to summit-lsp.lsst.codes timed out. (connect timeout=3.05)'))\"))\n", - "\n", - "\n", - "Attempt to get logs from url='https://summit-lsp.lsst.codes/narrativelog/messages?is_human=either&is_valid=either&limit=9999'\n", - "ERROR getting narrativelog from env='summit' using url='https://summit-lsp.lsst.codes/narrativelog/messages?is_human=either&is_valid=either&limit=9999': err=ConnectTimeout(MaxRetryError(\"HTTPSConnectionPool(host='summit-lsp.lsst.codes', port=443): Max retries exceeded with url: /narrativelog/messages?is_human=either&is_valid=either&limit=9999 (Caused by ConnectTimeoutError(, 'Connection to summit-lsp.lsst.codes timed out. (connect timeout=3.05)'))\"))\n", - "\n", - "Attempt to get logs from url='https://usdf-rsp-dev.slac.stanford.edu/exposurelog/messages?is_human=either&is_valid=either&limit=9999'\n", - "\n", - "\n", - "Attempt to get logs from url='https://usdf-rsp-dev.slac.stanford.edu/narrativelog/messages?is_human=either&is_valid=either&limit=9999'\n", - "\n", - "Attempt to get logs from url='https://tucson-teststand.lsst.codes/exposurelog/messages?is_human=either&is_valid=either&limit=9999'\n", - "\n", - "\n", - "Attempt to get logs from url='https://tucson-teststand.lsst.codes/narrativelog/messages?is_human=either&is_valid=either&limit=9999'\n" - ] - } - ], + "outputs": [], "source": [ "verbose=False\n", "fields = defaultdict(set) # fields[(env,log)] = {field1, field2, ...}\n", @@ -233,7 +196,7 @@ }, { "cell_type": "markdown", - "id": "c59da227-15e7-4f10-9509-405a9d1c230d", + "id": "9", "metadata": {}, "source": [ "\n", @@ -243,7 +206,7 @@ }, { "cell_type": "markdown", - "id": "402a093d-4443-468f-85a4-b82ee6d3ab06", + "id": "10", "metadata": {}, "source": [ "\n", @@ -252,118 +215,10 @@ }, { "cell_type": "code", - "execution_count": 5, - "id": "147c066a-bc3a-4c0e-8499-eaa2f1ec9f4c", + "execution_count": null, + "id": "11", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Got results from 4 of 6 env/logs\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
okservernumfieldsnumrecs
summitexposurelogFalsehttps://summit-lsp.lsst.codes00
narrativelogFalsehttps://summit-lsp.lsst.codes00
usdf_devexposurelogTruehttps://usdf-rsp-dev.slac.stanford.edu187895
narrativelogTruehttps://usdf-rsp-dev.slac.stanford.edu247597
tucsonexposurelogTruehttps://tucson-teststand.lsst.codes1820
narrativelogTruehttps://tucson-teststand.lsst.codes24110
\n", - "
" - ], - "text/plain": [ - " ok server \\\n", - "summit exposurelog False https://summit-lsp.lsst.codes \n", - " narrativelog False https://summit-lsp.lsst.codes \n", - "usdf_dev exposurelog True https://usdf-rsp-dev.slac.stanford.edu \n", - " narrativelog True https://usdf-rsp-dev.slac.stanford.edu \n", - "tucson exposurelog True https://tucson-teststand.lsst.codes \n", - " narrativelog True https://tucson-teststand.lsst.codes \n", - "\n", - " numfields numrecs \n", - "summit exposurelog 0 0 \n", - " narrativelog 0 0 \n", - "usdf_dev exposurelog 18 7895 \n", - " narrativelog 24 7597 \n", - "tucson exposurelog 18 20 \n", - " narrativelog 24 110 " - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "show_columns = ['ok', 'server', 'numfields', 'numrecs']\n", "df = pd.DataFrame(data=dict(results)).T.loc[:,show_columns]\n", @@ -373,7 +228,7 @@ }, { "cell_type": "markdown", - "id": "f6ce4545-74d6-4551-b125-44c7c4021c19", + "id": "12", "metadata": {}, "source": [ "\n", @@ -382,26 +237,10 @@ }, { "cell_type": "code", - "execution_count": 6, - "id": "0abedbf3-5976-4713-b5ab-f953396dbcff", + "execution_count": null, + "id": "13", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Field names for each Environment/Log source:\n", - "\n", - "usdf_dev/exposurelog: parent_id, instrument, obs_id, user_id, site_id, user_agent, is_human, date_added, message_text, level, tags, day_obs, exposure_flag, is_valid, seq_num, urls, date_invalidated, id\n", - "\n", - "usdf_dev/narrativelog: systems, date_end, message_text, primary_hardware_components, user_agent, site_id, is_human, cscs, category, is_valid, date_invalidated, time_lost_type, tags, urls, time_lost, components, date_begin, parent_id, subsystems, user_id, date_added, level, primary_software_components, id\n", - "\n", - "tucson/exposurelog: parent_id, instrument, obs_id, user_id, site_id, user_agent, is_human, date_added, message_text, level, tags, day_obs, exposure_flag, is_valid, seq_num, urls, date_invalidated, id\n", - "\n", - "tucson/narrativelog: systems, date_end, message_text, primary_hardware_components, user_agent, site_id, is_human, cscs, category, is_valid, date_invalidated, time_lost_type, tags, urls, time_lost, components, date_begin, parent_id, subsystems, user_id, date_added, level, primary_software_components, id\n" - ] - } - ], + "outputs": [], "source": [ "print('Field names for each Environment/Log source:')\n", "for (env,log),flds in fields.items():\n", @@ -412,7 +251,7 @@ }, { "cell_type": "markdown", - "id": "557803ba-3f59-432a-9d39-efca609a1f97", + "id": "14", "metadata": {}, "source": [ "\n", @@ -421,61 +260,10 @@ }, { "cell_type": "code", - "execution_count": 7, - "id": "4d98ac00-6694-47e9-a53e-5c5dfef96948", + "execution_count": null, + "id": "15", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "usdf_dev/exposurelog:\n", - " instrument: \t{'LSSTComCam', 'LATISS'}\n", - " user_agent: \t{'notebook:nublado', 'LOVE'}\n", - " is_human: \t{'True'}\n", - " site_id: \t{'summit'}\n", - " level: \t{'20', '10'}\n", - " exposure_flag: \t{'junk', 'none', 'questionable'}\n", - " is_valid: \t{'False', 'True'}\n", - "usdf_dev/narrativelog:\n", - " subsystems: \t{None}\n", - " systems: \t{None}\n", - " user_agent: \t{'LOVE', 'string'}\n", - " is_human: \t{True}\n", - " site_id: \t{'summit'}\n", - " cscs: \t{None}\n", - " time_lost_type: \t{'weather', 'fault', None}\n", - " level: \t{0, 100}\n", - " primary_software_components: \t{None}\n", - " primary_hardware_components: \t{None}\n", - " category: \t{'None', 'SCIENCE', 'ENG', None}\n", - " is_valid: \t{False, True}\n", - " components: \t{None}\n", - "tucson/exposurelog:\n", - " instrument: \t{'LSSTComCam', 'LATISS'}\n", - " user_agent: \t{'LOVE'}\n", - " is_human: \t{'True'}\n", - " site_id: \t{'tucson'}\n", - " level: \t{'10'}\n", - " exposure_flag: \t{'junk', 'none', 'questionable'}\n", - " is_valid: \t{'False', 'True'}\n", - "tucson/narrativelog:\n", - " subsystems: \t{None}\n", - " systems: \t{None}\n", - " user_agent: \t{'LOVE', 'string'}\n", - " is_human: \t{False, True}\n", - " site_id: \t{'tucson'}\n", - " cscs: \t{None}\n", - " time_lost_type: \t{'fault', None}\n", - " level: \t{0, 100}\n", - " primary_software_components: \t{None}\n", - " primary_hardware_components: \t{None}\n", - " category: \t{'None', None}\n", - " is_valid: \t{False, True}\n", - " components: \t{None}\n" - ] - } - ], + "outputs": [], "source": [ "dict(facets)\n", "for (env,log),flds in facets.items():\n", @@ -486,7 +274,7 @@ }, { "cell_type": "markdown", - "id": "fb8c376d-13aa-4740-bd01-9c565b331bab", + "id": "16", "metadata": {}, "source": [ "\n", @@ -495,275 +283,10 @@ }, { "cell_type": "code", - "execution_count": 8, - "id": "a4a4725b-ff92-44f2-b6eb-13c52c14b5da", + "execution_count": null, + "id": "17", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "env='usdf_dev', log='exposurelog': \n", - " Example records: \n", - "[{'id': '0005ebfa-1832-4e53-bc7a-a69254deef88',\n", - " 'site_id': 'summit',\n", - " 'obs_id': 'AT_O_20230118_000113',\n", - " 'instrument': 'LATISS',\n", - " 'day_obs': 20230118,\n", - " 'seq_num': 113,\n", - " 'message_text': ' ',\n", - " 'level': 20,\n", - " 'tags': [],\n", - " 'urls': [],\n", - " 'user_id': 'dsanmartim',\n", - " 'user_agent': 'notebook:nublado',\n", - " 'is_human': True,\n", - " 'is_valid': False,\n", - " 'exposure_flag': 'junk',\n", - " 'date_added': '2023-01-19T07:26:28.887152',\n", - " 'date_invalidated': '2023-01-19T07:32:32.435376',\n", - " 'parent_id': '18345e26-3c8c-4472-864f-68257ced64f4'},\n", - " {'id': '0007f3e6-ac70-4cf3-8edd-70d4a311aa4e',\n", - " 'site_id': 'summit',\n", - " 'obs_id': 'AT_O_20230118_000215',\n", - " 'instrument': 'LATISS',\n", - " 'day_obs': 20230118,\n", - " 'seq_num': 215,\n", - " 'message_text': '',\n", - " 'level': 20,\n", - " 'tags': [],\n", - " 'urls': [],\n", - " 'user_id': 'dsanmartim',\n", - " 'user_agent': 'notebook:nublado',\n", - " 'is_human': True,\n", - " 'is_valid': False,\n", - " 'exposure_flag': 'none',\n", - " 'date_added': '2023-01-19T07:00:11.295541',\n", - " 'date_invalidated': '2023-01-19T07:32:03.334049',\n", - " 'parent_id': 'cde04ee9-01b3-4428-8e4a-09c5bb25a0e3'}]\n", - "\n", - "env='usdf_dev', log='narrativelog': \n", - " Example records: \n", - "[{'id': '0005d0e0-6de7-4857-b658-cfca7147374a',\n", - " 'site_id': 'summit',\n", - " 'message_text': 'The moveShutterMainDoor command Fail\\r\\n'\n", - " '\\r\\n'\n", - " '\\r\\n'\n", - " 'we try the parameter with uppercase now\\r\\n'\n", - " '\\r\\n'\n", - " '\\r\\n'\n", - " ' open: False\\r\\n'\n", - " '\\r\\n'\n", - " '\\r\\n'\n", - " '\\r\\n'\n", - " '\\r\\n'\n", - " 'Error in run\\r\\n'\n", - " 'Traceback (most recent call last):\\r\\n'\n", - " ' File '\n", - " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", '\n", - " 'line 500, in wait_for\\r\\n'\n", - " ' return fut.result()\\r\\n'\n", - " ' ^^^^^^^^^^^^\\r\\n'\n", - " ' File '\n", - " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", '\n", - " 'line 239, in _get_next_ackcmd\\r\\n'\n", - " ' await self._next_ack_event.wait()\\r\\n'\n", - " ' File '\n", - " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/locks.py\", '\n", - " 'line 213, in wait\\r\\n'\n", - " ' await fut\\r\\n'\n", - " 'asyncio.exceptions.CancelledError\\r\\n'\n", - " '\\r\\n'\n", - " '\\r\\n'\n", - " 'The above exception was the direct cause of the following '\n", - " 'exception:\\r\\n'\n", - " '\\r\\n'\n", - " '\\r\\n'\n", - " 'Traceback (most recent call last):\\r\\n'\n", - " ' File '\n", - " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", '\n", - " 'line 189, in next_ackcmd\\r\\n'\n", - " ' ackcmd = await self._wait_task\\r\\n'\n", - " ' ^^^^^^^^^^^^^^^^^^^^^\\r\\n'\n", - " ' File '\n", - " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", '\n", - " 'line 214, in _basic_next_ackcmd\\r\\n'\n", - " ' ackcmd = await asyncio.wait_for(\\r\\n'\n", - " ' ^^^^^^^^^^^^^^^^^^^^^^^\\r\\n'\n", - " ' File '\n", - " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/asyncio/tasks.py\", '\n", - " 'line 502, in wait_for\\r\\n'\n", - " ' raise exceptions.TimeoutError() from exc\\r\\n'\n", - " 'TimeoutError\\r\\n'\n", - " '\\r\\n'\n", - " '\\r\\n'\n", - " 'During handling of the above exception, another exception '\n", - " 'occurred:\\r\\n'\n", - " '\\r\\n'\n", - " '\\r\\n'\n", - " 'Traceback (most recent call last):\\r\\n'\n", - " ' File '\n", - " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/base_script.py\", '\n", - " 'line 603, in do_run\\r\\n'\n", - " ' await self._run_task\\r\\n'\n", - " ' File '\n", - " '\"/net/obs-env/auto_base_packages/ts_standardscripts/python/lsst/ts/standardscripts/run_command.py\", '\n", - " 'line 170, in run\\r\\n'\n", - " ' await getattr(self.remote, f\"cmd_{self.cmd}\").start(\\r\\n'\n", - " ' File '\n", - " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", '\n", - " 'line 487, in start\\r\\n'\n", - " ' return await cmd_info.next_ackcmd(timeout=timeout)\\r\\n'\n", - " ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\\r\\n'\n", - " ' File '\n", - " '\"/opt/lsst/software/stack/conda/envs/lsst-scipipe-8.0.0/lib/python3.11/site-packages/lsst/ts/salobj/topics/remote_command.py\", '\n", - " 'line 205, in next_ackcmd\\r\\n'\n", - " ' raise base.AckTimeoutError(\\r\\n'\n", - " \"lsst.ts.salobj.base.AckTimeoutError: msg='Timed out waiting \"\n", - " \"for command acknowledgement', ackcmd=(ackcmd \"\n", - " 'private_seqNum=1142844592, ack=<SalRetCode.CMD_NOACK: '\n", - " \"-301>, error=0, result='No command acknowledgement \"\n", - " \"seen')\\r\\n\",\n", - " 'level': 0,\n", - " 'tags': [],\n", - " 'urls': [],\n", - " 'time_lost': 0.0,\n", - " 'date_begin': '2024-03-20T04:15:26.769000',\n", - " 'user_id': 'cmorales@summit-lsp.lsst.codes',\n", - " 'user_agent': 'LOVE',\n", - " 'is_human': True,\n", - " 'is_valid': True,\n", - " 'date_added': '2024-03-20T04:16:06.976595',\n", - " 'date_invalidated': None,\n", - " 'parent_id': None,\n", - " 'systems': None,\n", - " 'subsystems': None,\n", - " 'cscs': None,\n", - " 'date_end': '2024-03-20T04:15:26.769000',\n", - " 'components': ['AuxTel'],\n", - " 'primary_software_components': ['None'],\n", - " 'primary_hardware_components': ['None'],\n", - " 'category': 'None',\n", - " 'time_lost_type': 'fault'},\n", - " {'id': '00067b6a-1741-41f3-87af-57261dff30ac',\n", - " 'site_id': 'summit',\n", - " 'message_text': 'David updated environment. But no need to cycle Scheduler '\n", - " '1. Waiting for completion of the current block. \\r\\n',\n", - " 'level': 0,\n", - " 'tags': [],\n", - " 'urls': [],\n", - " 'time_lost': 0.0,\n", - " 'date_begin': '2024-04-24T20:53:41.843000',\n", - " 'user_id': 'ykang@summit-lsp.lsst.codes',\n", - " 'user_agent': 'LOVE',\n", - " 'is_human': True,\n", - " 'is_valid': True,\n", - " 'date_added': '2024-04-24T20:54:19.346646',\n", - " 'date_invalidated': None,\n", - " 'parent_id': None,\n", - " 'systems': None,\n", - " 'subsystems': None,\n", - " 'cscs': None,\n", - " 'date_end': '2024-04-24T20:53:41.843000',\n", - " 'components': ['MainTel'],\n", - " 'primary_software_components': ['None'],\n", - " 'primary_hardware_components': ['None'],\n", - " 'category': 'None',\n", - " 'time_lost_type': 'fault'}]\n", - "\n", - "env='tucson', log='exposurelog': \n", - " Example records: \n", - "[{'id': '0076ce05-752f-46df-b8a5-90ffd51d612e',\n", - " 'site_id': 'tucson',\n", - " 'obs_id': 'AT_O_20221129_000001',\n", - " 'instrument': 'LSSTComCam',\n", - " 'day_obs': 20221129,\n", - " 'seq_num': 1,\n", - " 'message_text': 'this was a test image for LOVE LVV November test cycle',\n", - " 'level': 10,\n", - " 'tags': ['undefined'],\n", - " 'urls': [],\n", - " 'user_id': 'admin@love1.tu.lsst.org',\n", - " 'user_agent': 'LOVE',\n", - " 'is_human': True,\n", - " 'is_valid': True,\n", - " 'exposure_flag': 'junk',\n", - " 'date_added': '2022-11-30T02:13:46.847038',\n", - " 'date_invalidated': None,\n", - " 'parent_id': None},\n", - " {'id': '1e61fbbe-cb87-45f2-88be-cb6647b0cd43',\n", - " 'site_id': 'tucson',\n", - " 'obs_id': 'AT_C_20230613_000001',\n", - " 'instrument': 'LATISS',\n", - " 'day_obs': 20230613,\n", - " 'seq_num': 1,\n", - " 'message_text': 'Exposure log creation test',\n", - " 'level': 10,\n", - " 'tags': ['undefined'],\n", - " 'urls': [],\n", - " 'user_id': 'admin@love1.tu.lsst.org',\n", - " 'user_agent': 'LOVE',\n", - " 'is_human': True,\n", - " 'is_valid': False,\n", - " 'exposure_flag': 'none',\n", - " 'date_added': '2023-07-18T22:13:41.228510',\n", - " 'date_invalidated': '2023-07-18T22:15:14.006037',\n", - " 'parent_id': None}]\n", - "\n", - "env='tucson', log='narrativelog': \n", - " Example records: \n", - "[{'id': '01806d0e-c21a-4e2e-8a53-1db4e8f88c0b',\n", - " 'site_id': 'tucson',\n", - " 'message_text': 'Wubba. Using do_makeLogEntry; vr=0.1',\n", - " 'level': 0,\n", - " 'tags': ['watcher', 'alarm'],\n", - " 'urls': [],\n", - " 'time_lost': 0.0,\n", - " 'date_begin': '2024-08-15T12:44:06.456000',\n", - " 'user_id': 'admin@love1.tu.lsst.org',\n", - " 'user_agent': 'LOVE',\n", - " 'is_human': False,\n", - " 'is_valid': True,\n", - " 'date_added': '2024-08-15T17:54:06.567418',\n", - " 'date_invalidated': None,\n", - " 'parent_id': None,\n", - " 'systems': ['ObservatorySoftwareTools'],\n", - " 'subsystems': ['LOVE'],\n", - " 'cscs': [],\n", - " 'date_end': '2024-08-15T12:44:06.456000',\n", - " 'components': None,\n", - " 'primary_software_components': None,\n", - " 'primary_hardware_components': None,\n", - " 'category': None,\n", - " 'time_lost_type': None},\n", - " {'id': '05b046b7-1266-47a9-b1de-93b6d44af242',\n", - " 'site_id': 'tucson',\n", - " 'message_text': '',\n", - " 'level': 0,\n", - " 'tags': [],\n", - " 'urls': [],\n", - " 'time_lost': 0.0,\n", - " 'date_begin': '2023-01-19T22:50:03.167000',\n", - " 'user_id': 'admin@love1.tu.lsst.org',\n", - " 'user_agent': 'LOVE',\n", - " 'is_human': True,\n", - " 'is_valid': True,\n", - " 'date_added': '2023-01-19T22:57:39.561293',\n", - " 'date_invalidated': None,\n", - " 'parent_id': None,\n", - " 'systems': [],\n", - " 'subsystems': [],\n", - " 'cscs': [],\n", - " 'date_end': '2023-01-19T22:50:03.167000',\n", - " 'components': None,\n", - " 'primary_software_components': None,\n", - " 'primary_hardware_components': None,\n", - " 'category': None,\n", - " 'time_lost_type': None}]\n" - ] - } - ], + "outputs": [], "source": [ "for (env,log),recs in examples.items():\n", " print(f'\\n{env=}, {log=}: ')\n", @@ -774,7 +297,7 @@ { "cell_type": "code", "execution_count": null, - "id": "c59d4f72-4bdb-4613-b634-26c2bb315221", + "id": "18", "metadata": {}, "outputs": [], "source": [] diff --git a/notebooks_tsqr/narrativelog.yaml b/notebooks_tsqr/narrativelog.yaml index b6406cb..a26b1dc 100644 --- a/notebooks_tsqr/narrativelog.yaml +++ b/notebooks_tsqr/narrativelog.yaml @@ -1,5 +1,5 @@ # For use with a Times Square notebook -title: Logging and Reporting: Narrative Log +title: Logging and Reporting for Narrative Log description: Prototype 1 authors: - name: Steve Pothier From bcc4669fe21509ce104ab20a53d5164f60ccdf87 Mon Sep 17 00:00:00 2001 From: Steve Pothier Date: Tue, 27 Aug 2024 15:20:18 -0700 Subject: [PATCH 10/10] ignore as list --- times-square.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/times-square.yaml b/times-square.yaml index e4692c5..2e9f430 100644 --- a/times-square.yaml +++ b/times-square.yaml @@ -1,5 +1,6 @@ enabled: true root: notebooks_tsqr -ignore: SAVE* +ignore: + - SAVE* description: > Times Square for project-wide Logging and Reporting.