From fed9619913414dfdf0562d862a12813de37e1b87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 19 Sep 2024 01:25:22 -0700 Subject: [PATCH 1/6] MAINT: no need for weekly dependabot (#626) * MAINT: no need for weekly dependabot * MAINT: grouping dependabot updates --- .github/dependabot.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index d21cf00b..b12794e5 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,9 +8,15 @@ updates: - package-ecosystem: "pip" directory: "/" schedule: - interval: "weekly" + interval: "monthly" - package-ecosystem: "github-actions" directory: "/" schedule: - interval: "weekly" + interval: "monthly" + groups: + actions: + patterns: + - "*" + labels: + - "github_actions" From a3c75bede6ee3ea2f0e079c1506c7d722b9c9688 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 19 Sep 2024 01:26:19 -0700 Subject: [PATCH 2/6] DOC: adding changelog for 1.1.0 and 1.1.1 (#625) --- CHANGELOG.md | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b96cd90..8c86603f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,66 @@ # Change Log +## v1.1.1 - 2024-06-27 + +([full changelog](https://github.com/executablebooks/MyST-NB/compare/v1.1.0...6ce30cd41fa82543e0f315ac8bbee82669b0cc82)) + +### Bugs fixed + +- FIX: output metadata overwrites image size for all following images [#609](https://github.com/executablebooks/MyST-NB/pull/609) ([@aeisenbarth](https://github.com/aeisenbarth)) + +- FIX: remove incorrect license classifier [#603](https://github.com/executablebooks/MyST-NB/pull/603) ([@agoose77](https://github.com/agoose77)) + +### Maintenance and upkeep improvements + +- MAINT: bump version [#614](https://github.com/executablebooks/MyST-NB/pull/614) ([@agoose77](https://github.com/agoose77)) + +- MAINT: appease mypy [#612](https://github.com/executablebooks/MyST-NB/pull/612) ([@agoose77](https://github.com/agoose77)) + +- MAINT: fix specs for CI matrix [#611](https://github.com/executablebooks/MyST-NB/pull/611) ([@agoose77](https://github.com/agoose77)) + +- MAINT: bump version [#592](https://github.com/executablebooks/MyST-NB/pull/592) ([@agoose77](https://github.com/agoose77)) + +### Documentation improvements + +- DOCS: set printoptions to disable modern scalar printing [#613](https://github.com/executablebooks/MyST-NB/pull/613) ([@agoose77](https://github.com/agoose77)) + +- DOCS: extra comma forgotten [#606](https://github.com/executablebooks/MyST-NB/pull/606) ([@jeertmans](https://github.com/jeertmans)) + +- DOCS: update shown code to match source [#598](https://github.com/executablebooks/MyST-NB/pull/598) ([@OriolAbril](https://github.com/OriolAbril)) + +### Contributors to this release + +([GitHub contributors page for this release](https://github.com/executablebooks/MyST-NB/graphs/contributors?from=2024-04-12&to=2024-06-27&type=c)) + +[@aeisenbarth](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3Aaeisenbarth+updated%3A2024-04-12..2024-06-27&type=Issues) | [@agoose77](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3Aagoose77+updated%3A2024-04-12..2024-06-27&type=Issues) | [@jeertmans](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3Ajeertmans+updated%3A2024-04-12..2024-06-27&type=Issues) | [@OriolAbril](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3AOriolAbril+updated%3A2024-04-12..2024-06-27&type=Issues) | [@sstroemer](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3Asstroemer+updated%3A2024-04-12..2024-06-27&type=Issues) | [@welcome](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3Awelcome+updated%3A2024-04-12..2024-06-27&type=Issues) + + +## v1.1.0 - 2024-04-12 + +([full changelog](https://github.com/executablebooks/MyST-NB/compare/v1.0.0...9943ec214c35844c4535d0184f7840574fc7ab03)) + +### Enhancements made + +- ENH: pass-through image metadata [#588](https://github.com/executablebooks/MyST-NB/pull/588) ([@flying-sheep](https://github.com/flying-sheep)) + +### Maintenance and upkeep improvements + +- MAINT: bump version [#592](https://github.com/executablebooks/MyST-NB/pull/592) ([@agoose77](https://github.com/agoose77)) + +- MAINT: use `findall` instead of `traverse` [#585](https://github.com/executablebooks/MyST-NB/pull/585) ([@agoose77](https://github.com/agoose77)) + +- MAINT: restore default line length [#577](https://github.com/executablebooks/MyST-NB/pull/577) ([@agoose77](https://github.com/agoose77)) + +### Other merged PRs + +- build(deps): bump actions/setup-python from 4 to 5 [#576](https://github.com/executablebooks/MyST-NB/pull/576) ([@dependabot](https://github.com/dependabot)) + +### Contributors to this release + +([GitHub contributors page for this release](https://github.com/executablebooks/MyST-NB/graphs/contributors?from=2023-11-08&to=2024-04-12&type=c)) + +[@agoose77](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3Aagoose77+updated%3A2023-11-08..2024-04-12&type=Issues) | [@cisaacstern](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3Acisaacstern+updated%3A2023-11-08..2024-04-12&type=Issues) | [@dependabot](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3Adependabot+updated%3A2023-11-08..2024-04-12&type=Issues) | [@flying-sheep](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3Aflying-sheep+updated%3A2023-11-08..2024-04-12&type=Issues) | [@ma-sadeghi](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3Ama-sadeghi+updated%3A2023-11-08..2024-04-12&type=Issues) | [@peytondmurray](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3Apeytondmurray+updated%3A2023-11-08..2024-04-12&type=Issues) | [@PhilipVinc](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3APhilipVinc+updated%3A2023-11-08..2024-04-12&type=Issues) | [@sphuber](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3Asphuber+updated%3A2023-11-08..2024-04-12&type=Issues) | [@welcome](https://github.com/search?q=repo%3Aexecutablebooks%2FMyST-NB+involves%3Awelcome+updated%3A2023-11-08..2024-04-12&type=Issues) + ## v1.0.0 - 2023-11-08 From d4ec1e98568f0c1ec97853e27f70c5a3732f10ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Brigitta=20Sip=C5=91cz?= Date: Thu, 19 Sep 2024 01:26:36 -0700 Subject: [PATCH 3/6] MAINT: only doing monthly pre-commit update (#627) --- .pre-commit-config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b6021404..760adcd8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,6 +9,9 @@ exclude: > tests/.*\.txt )$ +ci: + autoupdate_schedule: 'monthly' + repos: - repo: https://github.com/pre-commit/pre-commit-hooks From c90a70e6d6c2c4bad416c53e17771eb9fe6a3caf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 19 Sep 2024 08:58:21 -0700 Subject: [PATCH 4/6] build(deps): bump codecov/codecov-action from 3.1.4 to 4.5.0 in the actions group (#630) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * build(deps): bump codecov/codecov-action in the actions group Bumps the actions group with 1 update: [codecov/codecov-action](https://github.com/codecov/codecov-action). Updates `codecov/codecov-action` from 3.1.4 to 4.5.0 - [Release notes](https://github.com/codecov/codecov-action/releases) - [Changelog](https://github.com/codecov/codecov-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/codecov/codecov-action/compare/v3.1.4...v4.5.0) --- updated-dependencies: - dependency-name: codecov/codecov-action dependency-type: direct:production update-type: version-update:semver-major dependency-group: actions ... Signed-off-by: dependabot[bot] * codecov action's version --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Brigitta Sipőcz --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5c760131..60961efd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -87,7 +87,7 @@ jobs: # TEMPORARY FIX: Disable codecov until we can get it working again - name: Upload to Codecov - uses: codecov/codecov-action@v3.1.4 + uses: codecov/codecov-action@v4 if: false with: name: myst-nb-pytests From db8cedb1b6333ca2cde29abb53d88ee799201661 Mon Sep 17 00:00:00 2001 From: Bryan Weber Date: Thu, 19 Sep 2024 12:16:05 -0400 Subject: [PATCH 5/6] Update glue docs to mention NumPy 2.0 changes (#615) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changes to the text/plain representation of NumPy objects for NumPy >= 2.0 mean that we need a workaround for formatting. Co-authored-by: Brigitta Sipőcz --- docs/render/glue.md | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/docs/render/glue.md b/docs/render/glue.md index 66caeeb6..6b2cdcf9 100644 --- a/docs/render/glue.md +++ b/docs/render/glue.md @@ -25,7 +25,7 @@ To glue keys from other notebooks, see {ref}`glue/crossdoc`. ## Save variables in code cells You can use `myst_nb.glue()` to assign the output of a variable to a key of your choice. -`glue` will store all of the information that is normally used to display that variable (ie, whatever happens when you display the variable by putting it at the end of a cell). +`glue` will store all of the information that is normally used to display that variable (that is, whatever happens when you display the variable by putting it at the end of a cell). Choose a key that you will remember, as you will use it later. The following code glues a variable inside the notebook: @@ -137,12 +137,12 @@ These variables can be pasted using one of the roles or directives in the `glue: ### The `glue` role/directive -The simplest role and directive are `glue` (a.k.a. `glue:any`), +The simplest role and directive are `glue` (also known as `glue:any`), which paste the glued output inline or as a block respectively, with no additional formatting. Simply add: -```` +````md ```{glue} your-key ``` ```` @@ -219,6 +219,27 @@ For example, the following: ``My rounded mean: {glue:text}`boot_mean:.2f` `` wil My rounded mean: {glue:text}`boot_mean:.2f` (95% CI: {glue:text}`boot_clo:.2f`/{glue:text}`boot_chi:.2f`). +````{warning} +As of NumPy 2.0, the `text/plain` representation of [NumPy objects has changed](https://numpy.org/devdocs/release/2.0.0-notes.html#representation-of-numpy-scalars-changed). +Using text formatting with NumPy>=2.0 will give warnings like: + +``` +sphinx.errors.SphinxWarning: :257:Failed to format text/plain data: could not convert string to float: 'np.float64(0.9643970836113095)' [mystnb.glue] +``` + +This can be resolved by either formatting the number before glueing or by setting NumPy to use legacy print options, as shown below. + +```python +var = np.float(1.0) +# Format the variable before glueing +glue("var_glue", f"{var:.2f}") + +# Or set NumPy legacy print options +np.setprintoptions(legacy="1.25") +glue("var_glue", var) +``` +```` + +++ ### The `glue:figure` directive From 25fa81d71c14b554b5a18bc2f0f67dba099afbe8 Mon Sep 17 00:00:00 2001 From: Bas Nijholt Date: Thu, 19 Sep 2024 15:34:36 -0700 Subject: [PATCH 6/6] Fix incorrect output from prints originating from different processes (#604) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix incorrect output from prints originating from different processes In the PipeFunc documentation I got: ``` ``` { "cell_type": "code", "execution_count": 47, "id": "92", "metadata": { "execution": { "iopub.execute_input": "2024-05-31T05:42:39.297713Z", "iopub.status.busy": "2024-05-31T05:42:39.297474Z", "iopub.status.idle": "2024-05-31T05:42:40.477462Z", "shell.execute_reply": "2024-05-31T05:42:40.475729Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "2024-05-30 22:42:39.410279 - Running double_it for x=3" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2024-05-30 22:42:39.408318 - Running double_it for x=0" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2024-05-30 22:42:39.410888 - Running double_it for x=1" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2024-05-30 22:42:39.416024 - Running double_it for x=2" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2024-05-30 22:42:39.431485 - Running half_it for x=0" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2024-05-30 22:42:39.434285 - Running half_it for x=1" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2024-05-30 22:42:39.433559 - Running half_it for x=2" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2024-05-30 22:42:39.439223 - Running half_it for x=3" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "2024-05-30 22:42:40.459668 - Running take_sum" ] }, { "name": "stdout", "output_type": "stream", "text": [ "\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "14\n" ] } ], "source": [ "from concurrent.futures import ProcessPoolExecutor\n", "import datetime\n", "import numpy as np\n", "import time\n", "from pipefunc import Pipeline, pipefunc\n", "\n", "\n", "@pipefunc(output_name=\"double\", mapspec=\"x[i] -> double[i]\")\n", "def double_it(x: int) -> int:\n", " print(f\"{datetime.datetime.now()} - Running double_it for x={x}\")\n", " time.sleep(1)\n", " return 2 * x\n", "\n", "\n", "@pipefunc(output_name=\"half\", mapspec=\"x[i] -> half[i]\")\n", "def half_it(x: int) -> int:\n", " print(f\"{datetime.datetime.now()} - Running half_it for x={x}\")\n", " time.sleep(1)\n", " return x // 2\n", "\n", "\n", "@pipefunc(output_name=\"sum\")\n", "def take_sum(half: np.ndarray, double: np.ndarray) -> int:\n", " print(f\"{datetime.datetime.now()} - Running take_sum\")\n", " return sum(half + double)\n", "\n", "\n", "pipeline = Pipeline([double_it, half_it, take_sum])\n", "inputs = {\"x\": [0, 1, 2, 3]}\n", "run_folder = \"my_run_folder\"\n", "executor = ProcessPoolExecutor(max_workers=8) # use 8 processes\n", "results = pipeline.map(\n", " inputs,\n", " run_folder=run_folder,\n", " parallel=True,\n", " executor=executor,\n", " storage=\"shared_memory_dict\",\n", ")\n", "print(results[\"sum\"].output)" ] }, ``` * more line up * Add parallel streams test --------- Co-authored-by: Brigitta Sipőcz --- myst_nb/core/utils.py | 5 +- tests/notebooks/merge_streams_parallel.ipynb | 186 ++++++++++++++++++ tests/test_render_outputs.py | 13 ++ .../test_merge_streams_parallel.xml | 21 ++ 4 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 tests/notebooks/merge_streams_parallel.ipynb create mode 100644 tests/test_render_outputs/test_merge_streams_parallel.xml diff --git a/myst_nb/core/utils.py b/myst_nb/core/utils.py index d1165712..b3f428f5 100644 --- a/myst_nb/core/utils.py +++ b/myst_nb/core/utils.py @@ -25,8 +25,11 @@ def coalesce_streams(outputs: list[NotebookNode]) -> list[NotebookNode]: for output in outputs: if output["output_type"] == "stream": if output["name"] in streams: - streams[output["name"]]["text"] += output["text"] + out = output["text"].strip() + if out: + streams[output["name"]]["text"] += f"{out}\n" else: + output["text"] = output["text"].strip() + "\n" new_outputs.append(output) streams[output["name"]] = output else: diff --git a/tests/notebooks/merge_streams_parallel.ipynb b/tests/notebooks/merge_streams_parallel.ipynb new file mode 100644 index 00000000..9a9e5adc --- /dev/null +++ b/tests/notebooks/merge_streams_parallel.ipynb @@ -0,0 +1,186 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "execution": { + "iopub.execute_input": "2024-09-19T21:44:29.809012Z", + "iopub.status.busy": "2024-09-19T21:44:29.808809Z", + "iopub.status.idle": "2024-09-19T21:44:29.978481Z", + "shell.execute_reply": "2024-09-19T21:44:29.977891Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n" + ] + } + ], + "source": [ + "from concurrent.futures import ProcessPoolExecutor\n", + "\n", + "with ProcessPoolExecutor() as executor:\n", + " for i in executor.map(print, [0] * 10):\n", + " pass" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 + } diff --git a/tests/test_render_outputs.py b/tests/test_render_outputs.py index 1b58e181..d24d7586 100644 --- a/tests/test_render_outputs.py +++ b/tests/test_render_outputs.py @@ -1,4 +1,5 @@ """Tests for rendering code cell outputs.""" + import pytest from myst_nb.core.render import EntryPointError, load_renderer @@ -103,6 +104,18 @@ def test_merge_streams(sphinx_run, file_regression): file_regression.check(doctree.pformat(), extension=".xml", encoding="utf-8") +@pytest.mark.sphinx_params( + "merge_streams_parallel.ipynb", + conf={"nb_execution_mode": "off", "nb_merge_streams": True}, +) +def test_merge_streams_parallel(sphinx_run, file_regression): + """Test configuring multiple concurrent stdout/stderr outputs to be merged.""" + sphinx_run.build() + assert sphinx_run.warnings() == "" + doctree = sphinx_run.get_resolved_doctree("merge_streams_parallel") + file_regression.check(doctree.pformat(), extension=".xml", encoding="utf-8") + + @pytest.mark.sphinx_params( "metadata_image.ipynb", conf={"nb_execution_mode": "off", "nb_cell_metadata_key": "myst"}, diff --git a/tests/test_render_outputs/test_merge_streams_parallel.xml b/tests/test_render_outputs/test_merge_streams_parallel.xml new file mode 100644 index 00000000..17254962 --- /dev/null +++ b/tests/test_render_outputs/test_merge_streams_parallel.xml @@ -0,0 +1,21 @@ + + + + + from concurrent.futures import ProcessPoolExecutor + + with ProcessPoolExecutor() as executor: + for i in executor.map(print, [0] * 10): + pass + + + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0 + 0