diff --git a/notebooks_tsqr/NightLog.ipynb b/notebooks_tsqr/NightLog.ipynb index 336da77..fdead63 100644 --- a/notebooks_tsqr/NightLog.ipynb +++ b/notebooks_tsqr/NightLog.ipynb @@ -179,18 +179,33 @@ "id": "9", "metadata": {}, "source": [ - "# Almanac" + "# DDV" ] }, { "cell_type": "code", "execution_count": null, "id": "10", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "metadata": {}, + "outputs": [], + "source": [ + "DDV = f\"{server}/rubintv-dev/ddv/index.html\"\n", + "display_markdown(f'Access DDV part of RubinTV: {DDV}', raw=True)\n" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "# Almanac" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, "outputs": [], "source": [ "# Display various almanac values (for moon, sun)\n", @@ -199,7 +214,7 @@ }, { "cell_type": "markdown", - "id": "11", + "id": "13", "metadata": {}, "source": [ "# Night Report" @@ -208,12 +223,8 @@ { "cell_type": "code", "execution_count": null, - "id": "12", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "id": "14", + "metadata": {}, "outputs": [], "source": [ "# Get data from Night Report log. Display nightly Jira BLOCKS.\n", @@ -254,7 +265,7 @@ }, { "cell_type": "markdown", - "id": "13", + "id": "15", "metadata": {}, "source": [ "# Exposure Log" @@ -263,12 +274,8 @@ { "cell_type": "code", "execution_count": null, - "id": "14", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "id": "16", + "metadata": {}, "outputs": [], "source": [ "# Get data from Exposure log. Display time log.\n", @@ -298,7 +305,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -308,12 +315,8 @@ { "cell_type": "code", "execution_count": null, - "id": "16", - "metadata": { - "jupyter": { - "source_hidden": true - } - }, + "id": "18", + "metadata": {}, "outputs": [], "source": [ "# Display Observation gaps\n", @@ -335,7 +338,7 @@ }, { "cell_type": "markdown", - "id": "17", + "id": "19", "metadata": {}, "source": [ "# Narrative Log\n" @@ -344,7 +347,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "20", "metadata": { "jupyter": { "source_hidden": true @@ -389,7 +392,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "21", "metadata": {}, "outputs": [], "source": [ @@ -403,7 +406,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "22", "metadata": {}, "outputs": [], "source": [ @@ -413,7 +416,7 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "23", "metadata": {}, "outputs": [], "source": [] @@ -421,7 +424,7 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "24", "metadata": {}, "outputs": [], "source": [] @@ -443,7 +446,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.5" + "version": "3.11.9" } }, "nbformat": 4, diff --git a/notebooks_tsqr/README.md b/notebooks_tsqr/README.md index 3efda5a..98fb692 100644 --- a/notebooks_tsqr/README.md +++ b/notebooks_tsqr/README.md @@ -1,5 +1,58 @@ -Times Square notebooks for (potential) use in project-wide Logging & Reporting +# Logging & Reporting Times Square Notebooks -See [offical -documentation](https://rsp.lsst.io/v/usdfdev/guides/times-square/index.html) -on creating notebooks for use by Times Square. +Notebooks found here `./notebooks/` are meant to be run in Times Square towards use in project-wide Nightly Logging & Reporting +Times-Square: + +See [official Times-Square documentation](https://rsp.lsst.io/v/usdfdev/guides/times-square/index.html) on creating notebooks for use by Times Square. + +## Development Guidelines + +Rapid Prototyping is enabled with the branch `prototype` +Times-Square for this repository displays the `prototype` branch. + +- Create a branch for your Jira Ticket in the format `tickets/dm-####` off of `prototype` +- Communicate often with team mate when you want to push changes to `prototype` +- Rebase your branch off `prototype` before merging your branch into `prototype` + +Example of flow: + +1. `git checkout prototype; git pull` +2. `git checkout -b tickets/dm-23456` +3. `git commit -m "work happened"; git push` +4. `git checkout prototype; git pull` +5. `git checkout tickets/dm-23456` +6. `git rebase prototype` +7. `git checkout prototype; git merge tickets/dm-23456; git push` + +  + +Once Per Sprint (2 week), the developers on this repository (Steve Pothier & Valerie Becker) gather to discuss updates made to `prototype`, outstanding pull requests, and tickets that have been completed. + +Once they are in agreement, they merge `prototype` into the `develop` branch and close the related Jira Tickets. Squash commit should be used here with a descriptive title and description in the PR. + + +## NightLog.ipynb + +NightLog.ipynb is our main Logging And Reporting notebook. This notebook is meant to display completed* views of logging information. +Each separate notebook should be used to mature a logging/reporting product, and then expect to be integrated into this 'main' notebook. + +\*_Completed to an alpha\beta level -- quick improvements will continue to happen during Fall-Winter 2024_ + +## Dashboard + +Dashboard.ipynb is intended for local development purposes and debugging. Run this notebook not from RSP to evaluate your connection to an array of data sources. +_RSP is not intended to have access to all of the data sources queried here._ + +## Kernel + +Times Square developers/maintainers have indicated that the LSST Kernel should be used in notebooks displayed there. +[RSP Stack info](https://developer.lsst.io/stack/conda.html#rubin-science-platform-notebooks) + +## Backend Code + +We are working our way into a non-Times-Square dependent project. Towards that effort, we are incrementally abstracting common code out of the notebooks. This code is kept in `./python/lsst/ts/logging_and_reporting/` + +`almanac.py` ... +`reports.py` ... +`source_adapters.py` .... +`utils.py` is used for ... \ No newline at end of file diff --git a/notebooks_tsqr/consdb/access_consdb.ipynb b/notebooks_tsqr/consdb/access_consdb.ipynb index 59dd2f8..dcc59d1 100644 --- a/notebooks_tsqr/consdb/access_consdb.ipynb +++ b/notebooks_tsqr/consdb/access_consdb.ipynb @@ -34,7 +34,7 @@ "import pandas as pd\n", "from IPython.display import display, Markdown, display_markdown\n", "\n", - "# URL = \"https://usdf-rsp.slac.stanford.edu/consdb/\" \n", + "# URL = \"https://usdf-rsp.slac.stanford.edu/consdb/\"\n", "# Need to add that part about the headers to client flow through\n", "URL = \"http://consdb-pq.consdb:8080/consdb\" # Don't use this one\n", "\n", diff --git a/notebooks_tsqr/consdb/assorted_plots.ipynb b/notebooks_tsqr/consdb/assorted_plots.ipynb index 525ae90..f886feb 100644 --- a/notebooks_tsqr/consdb/assorted_plots.ipynb +++ b/notebooks_tsqr/consdb/assorted_plots.ipynb @@ -9,7 +9,7 @@ "source": [ "# Parameters\n", "day_obs = '2024-06-26'\n", - "instrument = 'latiss'" + "instruments = 'latiss, lsstcomcamsim, lsstcomcam'" ] }, { @@ -23,8 +23,7 @@ "import pandas as pd\n", "import matplotlib.pylab as plt\n", "from IPython.display import display, Markdown, display_markdown\n", - "\n", - "from lsst.summit.utils import ConsDbClient" + "%matplotlib inline" ] }, { @@ -34,21 +33,12 @@ "metadata": {}, "outputs": [], "source": [ + "### 'Get Consdb access'\n", + "from lsst.summit.utils import ConsDbClient\n", "\n", "URL = \"http://consdb-pq.consdb:8080/consdb\" # Don't use this one\n", "os.environ[\"no_proxy\"] += \",.consdb\"\n", - "%matplotlib inline\n", - "day_obs_int = int(day_obs.replace('-', ''))\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], - "source": [ + "\n", "client = ConsDbClient(URL)\n", "display_markdown('### Consolidated Database is accessible',raw=True)" ] @@ -56,135 +46,88 @@ { "cell_type": "code", "execution_count": null, - "id": "4", + "id": "3", "metadata": {}, "outputs": [], "source": [ + "# Put Plot in backend\n", + "plt.style.use('seaborn-v0_8-bright')\n", + "def plot(y, x):\n", + " # plot\n", + " fig, ax = plt.subplots()\n", + " ax.scatter(x, y)#, linewidth=2.0, color=\"#058b8c\")\n", "\n", - "psf_query = f'''\n", - " SELECT psf_area FROM cdb_lsstcomcamsim.ccdvisit1_quicklook\n", - " where day_obs = {day_obs_int}\n", - "'''\n", - "\n", - "sky_brightness_query = f'''\n", - "'''\n", - "\n", - "zero_point_query = f'''\n", - " SELECT zero_point FROM cdb_lsstcomcamsim.ccdvisit1_quicklook\n", - " where day_obs = {day_obs_int}\n", - "'''\n", - "\n", - "ra_dec_query = f'''\n", - " SELECT s_ra, s_dec FROM cdb_lsstcomcamsim.ccdvisit1_quicklook\n", - " where day_obs = {day_obs_int}\n", - "'''\n" + " plt.show()\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "5", + "id": "4", "metadata": {}, "outputs": [], "source": [ "day_obs_int = int(day_obs.replace('-', ''))\n", - "instrument = 'lsstcomcamsim'\n", - "ccdvisit1_quicklook = f''' \n", - " SELECT * FROM cdb_{instrument}.ccdvisit1_quicklook\n", - "'''\n", - "\n", - "visit1 = f'''\n", - " SELECT * FROM cdb_{instrument}.visit1\n", - "'''\n", + "instrument_list = [ins.strip() for ins in instruments.split(',')]\n", "\n", - "# Join Visit1 and ccdVisit1 to access data and day obs\n", - "visits = client.query(visit1).to_pandas()\n", - "quicklook = client.query(ccdvisit1_quicklook).to_pandas()\n", - "\n", - "visits = visits.join(quicklook, on='visit_id',lsuffix='',rsuffix='_q')\n", - "\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "visits_w_psf = visits[visits['psf_area'].notna()]\n", - "\n", - "visits_w_zero = visits[visits['zero_point'].notna()]\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], - "source": [ - "# Pare down to only day obs \n", - "visits_today = visits[(visits['day_obs'] == day_obs_int)]# & (visits['psf_area'].notna())]\n", + "for instrument in instrument_list:\n", + " print(f'instrumental {instrument}')\n", + " #################### Put in Backend\n", + " ccdvisit1_quicklook = f'''\n", + " SELECT * FROM cdb_{instrument}.ccdvisit1_quicklook\n", + " '''\n", "\n", - "ra = visits_today['s_ra']\n", - "dec = visits_today['s_dec']" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8", - "metadata": {}, - "outputs": [], - "source": [ - "plt.style.use('seaborn-v0_8-bright')\n", - "def plot(y, x):\n", - " # Times Square Tutorial\n", - " #x = np.linspace(0, 10, 100)\n", - " #y = y0 + A * np.sin(2 * np.pi * x / lambd)\n", + " visit1 = f'''\n", + " SELECT * FROM cdb_{instrument}.visit1\n", + " '''\n", "\n", - " # plot\n", - " fig, ax = plt.subplots()\n", + " # Join Visit1 and ccdVisit1 to access data and day obs\n", + " visits = client.query(visit1).to_pandas()\n", + " quicklook = client.query(ccdvisit1_quicklook).to_pandas()\n", "\n", - " ax.scatter(x, y)#, linewidth=2.0, color=\"#058b8c\")\n", + " visits = visits.join(quicklook, on='visit_id',lsuffix='',rsuffix='_q')\n", + " #################### Put in Backend - end\n", "\n", - " #ax.set(\n", - " # xlim=(0, 8), xticks=np.arange(1, 8),\n", - " # ylim=(0, 8), yticks=np.arange(1, 8)\n", - " #)\n", + " # If we see data exist in psf, zero, then we should pare down like visits_today below\n", + " try:\n", + " visits_w_psf = visits[visits['psf_area'].notna()]\n", + " time = visits_w_psf['obs_start']\n", + " print(f'length of visits with psf_area populated {len(visits_w_psf)}')\n", + " display_markdown('# psf_area vs obs_start', raw=True)\n", + " plot(time, visits_w_psf['psf_area'])\n", + " except KeyError as err:\n", + " print(\"No psf\")\n", + " \n", + " try:\n", + " visits_w_zero = visits[visits['zero_point'].notna()]\n", + " time = visits_w_zero['obs_start']\n", + " print(f'length of visits with zero_point populated {len(visits_w_zero)}')\n", + " display_markdown('# zero_point vs obs_start', raw=True)\n", + " plot(time, visits_w_zero['zero_point'])\n", + " except KeyError as err:\n", + " print(\"No zero_point\")\n", + " \n", + " # Pare down to only day obs\n", + " visits_today = visits[(visits['day_obs'] == day_obs_int)]\n", "\n", - " plt.show()" + " ra = visits_today['s_ra']\n", + " dec = visits_today['s_dec']\n", + " display_markdown('# Ra Dec proof they populated', raw=True)\n", + " plot(ra, dec)" ] }, { "cell_type": "code", "execution_count": null, - "id": "9", - "metadata": {}, - "outputs": [], - "source": [ - "display_markdown('# This plot is only proof that data exists', raw=True)\n", - "plot(ra, dec)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "10", + "id": "5", "metadata": {}, "outputs": [], - "source": [ - "#try:\n", - "# psf_data = client.query(psf_query).to_pandas()\n", - "#except requests.HTTPError or requests.JSONDecodeError:\n", - "# psf_data = client.query(psf_query).to_pandas()\n" - ] + "source": [] }, { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "6", "metadata": {}, "outputs": [], "source": [] @@ -192,7 +135,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "7", "metadata": {}, "outputs": [], "source": [] @@ -200,7 +143,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "8", "metadata": {}, "outputs": [], "source": [] @@ -208,7 +151,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "9", "metadata": {}, "outputs": [], "source": [] @@ -216,7 +159,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "10", "metadata": {}, "outputs": [], "source": [] @@ -224,9 +167,9 @@ ], "metadata": { "kernelspec": { - "display_name": "logrep", + "display_name": "LSST", "language": "python", - "name": "python3" + "name": "lsst" }, "language_info": { "codemirror_mode": { @@ -238,7 +181,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.5" + "version": "3.11.9" } }, "nbformat": 4, diff --git a/notebooks_tsqr/consdb/assorted_plots.yaml b/notebooks_tsqr/consdb/assorted_plots.yaml index 88a3477..545b20e 100644 --- a/notebooks_tsqr/consdb/assorted_plots.yaml +++ b/notebooks_tsqr/consdb/assorted_plots.yaml @@ -15,4 +15,4 @@ parameters: instrument: type: string description: Which instrument to query - default: 'latiss' \ No newline at end of file + default: 'latiss, lsstcomcamsim, lsstcomcam' \ No newline at end of file