Skip to content

Commit

Permalink
Tickets/dm 45892 (#13)
Browse files Browse the repository at this point in the history
* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* WIPO

* WIP

* WIP

* ignore as list

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP

* WIP
pothiers authored Aug 30, 2024
1 parent b9a1fd9 commit 91df4e1
Showing 7 changed files with 729 additions and 26 deletions.
5 changes: 4 additions & 1 deletion notebooks_tsqr/TEMPLATE_logrep.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
# For use with a Times Square notebook
title: TEMPLATE for LR
description: Prototype 1
description: >
Copy and rename this ipynb and yaml sidecar into a new
pair of files (<log_source>.ipynb, <log_source>.yaml).
The TEMPLATE_* files will eventually be hidden in Times Square.
authors:
- name: Steve Pothier
slack: Steve Pothier
356 changes: 356 additions & 0 deletions notebooks_tsqr/efd.ipynb
Original file line number Diff line number Diff line change
@@ -0,0 +1,356 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": null,
"id": "0",
"metadata": {},
"outputs": [],
"source": [
"# Parameters. Set defaults here.\n",
"# Times Square replaces this cell with the user's parameters.\n",
"record_limit = '999'"
]
},
{
"cell_type": "markdown",
"id": "1",
"metadata": {},
"source": [
"<a class=\"anchor\" id=\"imports\"></a>\n",
"## Imports and General Setup"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2",
"metadata": {},
"outputs": [],
"source": [
"# Only use packages available in the Rubin Science Platform\n",
"import requests\n",
"from collections import defaultdict\n",
"import pandas as pd\n",
"from pprint import pp, pformat\n",
"from urllib.parse import urlencode\n",
"from IPython.display import FileLink, display_markdown\n",
"from matplotlib import pyplot as plt\n",
"import os"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3",
"metadata": {},
"outputs": [],
"source": [
"env = 'usdf_dev' # usdf-dev, tucson, slac, summit\n",
"log_name = 'narrativelog'\n",
"log = log_name\n",
"limit = int(record_limit)\n",
"response_timeout = 3.05 # seconds, how long to wait for connection\n",
"read_timeout = 20 # seconds\n",
"\n",
"timeout = (float(response_timeout), float(read_timeout))\n",
"\n",
"# RUNNING_INSIDE_JUPYTERLAB is True when running under Times Square\n",
"server = os.environ.get('EXTERNAL_INSTANCE_URL', \n",
" 'https://tucson-teststand.lsst.codes')\n",
"service = f'{server}/{log}'\n",
"service"
]
},
{
"cell_type": "markdown",
"id": "4",
"metadata": {},
"source": [
"<a class=\"anchor\" id=\"setup_source\"></a>\n",
"## Setup Source"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5",
"metadata": {},
"outputs": [],
"source": [
"md = f'### Will retrieve from {service}'\n",
"display_markdown(md, raw=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "6",
"metadata": {},
"outputs": [],
"source": [
"recs = None\n",
"ok = True\n",
"\n",
"# is_human=either&is_valid=either&offset=0&limit=50' \n",
"# site_ids=tucson&message_text=wubba&min_level=0&max_level=999&user_ids=spothier&user_agents=LOVE\n",
"# tags=love&exclude_tags=ignore_message\n",
"qparams = dict(is_human='either',\n",
" is_valid='either',\n",
" limit=limit,\n",
" )\n",
"qstr = urlencode(qparams)\n",
"url = f'{service}/messages?{qstr}'\n",
"\n",
"ignore_fields = set(['tags', 'urls', 'message_text', 'id', 'date_added', \n",
" 'obs_id', 'day_obs', 'seq_num', 'parent_id', 'user_id',\n",
" 'date_invalidated', 'date_begin', 'date_end',\n",
" 'time_lost', # float\n",
" #'systems','subsystems','cscs', # values are lists, special handling\n",
" ])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7",
"metadata": {},
"outputs": [],
"source": [
"display_markdown(f'## Get (up to {limit}) Records', raw=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8",
"metadata": {},
"outputs": [],
"source": [
"# TODO Often fails on first request. Find out why!\n",
"try:\n",
" response = requests.get(url, timeout=timeout)\n",
"except:\n",
" pass \n",
" \n",
"try:\n",
" print(f'Attempt to get logs from {url=}')\n",
" response = requests.get(url, timeout=timeout)\n",
" response.raise_for_status()\n",
" recs = response.json()\n",
" flds = set(recs[0].keys())\n",
" facflds = flds - ignore_fields\n",
" # facets(field) = set(value-1, value-2, ...)\n",
" facets = {fld: set([str(r[fld])\n",
" for r in recs if not isinstance(r[fld], list)]) \n",
" for fld in facflds}\n",
"except Exception as err:\n",
" ok = False\n",
" print(f'ERROR getting {log} from {env=} using {url=}: {err=}')\n",
"numf = len(flds) if ok else 0\n",
"numr = len(recs) if ok else 0\n",
"print(f'Retrieved {numr} records, each with {numf} fields.')"
]
},
{
"cell_type": "markdown",
"id": "9",
"metadata": {},
"source": [
"<a class=\"anchor\" id=\"table\"></a>\n",
"## Tables of (mostly raw) results"
]
},
{
"cell_type": "markdown",
"id": "10",
"metadata": {},
"source": [
"### Fields names provided in records from log."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "11",
"metadata": {},
"outputs": [],
"source": [
"pd.DataFrame(flds, columns=['Field Name'])"
]
},
{
"cell_type": "markdown",
"id": "12",
"metadata": {},
"source": [
"### Facets from log records.\n",
"A *facet* is the set all of values found for a field in the retrieved records. Facets are only calculated for some fields."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "13",
"metadata": {
"jupyter": {
"source_hidden": true
}
},
"outputs": [],
"source": [
"display(pd.DataFrame.from_dict(facets, orient='index'))\n",
"display(facets)"
]
},
{
"cell_type": "markdown",
"id": "14",
"metadata": {},
"source": [
"### Table of selected log record fields.\n",
"Table can be retrieved as CSV file for local use."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "15",
"metadata": {
"jupyter": {
"source_hidden": true
}
},
"outputs": [],
"source": [
"cols = ['date_added', 'time_lost']\n",
"df = pd.DataFrame(recs)[cols]\n",
"\n",
"# Allow download of CSV version of DataFrame\n",
"csvfile = 'tl.csv'\n",
"df.to_csv(csvfile)\n",
"myfile = FileLink(csvfile)\n",
"print('Table available as CSV file: ')\n",
"display(myfile)\n",
"df"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "16",
"metadata": {},
"outputs": [],
"source": [
"df = pd.DataFrame(recs)\n",
"df"
]
},
{
"cell_type": "markdown",
"id": "17",
"metadata": {},
"source": [
"<a class=\"anchor\" id=\"plot\"></a>\n",
"## Plots from log"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "18",
"metadata": {},
"outputs": [],
"source": [
"x = [r['date_added'] for r in recs]\n",
"y = [r['time_lost'] for r in recs]\n",
"plt.plot(x, y) \n",
"plt.show()"
]
},
{
"cell_type": "markdown",
"id": "19",
"metadata": {},
"source": [
"<a class=\"anchor\" id=\"raw_analysis\"></a>\n",
"## Raw Content Analysis"
]
},
{
"cell_type": "markdown",
"id": "20",
"metadata": {},
"source": [
"### Example of one record"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "21",
"metadata": {},
"outputs": [],
"source": [
"rec = recs[-1]\n",
"\n",
"msg = rec[\"message_text\"]\n",
"md = f'Message text from log:\\n> {msg}'\n",
"display_markdown(md, raw=True)\n",
"\n",
"display(rec)"
]
},
{
"cell_type": "markdown",
"id": "22",
"metadata": {},
"source": [
"<a class=\"anchor\" id=\"elicitation\"></a>\n",
"## Stakeholder Elicitation"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "23",
"metadata": {},
"outputs": [],
"source": [
"#EXTERNAL_INSTANCE_URL\n",
"ed = dict(os.environ.items())\n",
"with pd.option_context('display.max_rows', None,):\n",
" print(pd.DataFrame(ed.values(), index=ed.keys()))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "24",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.10.12"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
19 changes: 19 additions & 0 deletions notebooks_tsqr/efd.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# For use with a Times Square notebook
title: TEMPLATE for LR
description: >
Copy and rename this ipynb and yaml sidecar into a new
pair of files (<log_source>.ipynb, <log_source>.yaml).
The TEMPLATE_* files will eventually be hidden in Times Square.
authors:
- name: Steve Pothier
slack: Steve Pothier
tags:
- reporting
- prototype
parameters:
record_limit:
type: integer
description: Max number of records to output
default: 99
minimum: 1
maximum: 9999
Loading

0 comments on commit 91df4e1

Please sign in to comment.