diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d0e0e30..02b6365 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -7,11 +7,12 @@ permissions: env: CIBW_BEFORE_BUILD: pip install setuptools oldest-supported-numpy - CIBW_BUILD_VERBOSITY: 3 - # CIBW_TEST_REQUIRES: pytest + CIBW_BUILD_VERBOSITY: 1 CIBW_TEST_COMMAND: python -c "import sys, numexpr; sys.exit(0 if numexpr.test().wasSuccessful() else 1)" CIBW_TEST_SKIP: "*macosx*arm64*" - CIBW_SKIP: "*musllinux*" + # Building for musllinux and aarch64 takes way too much time. + # NumPy is adding musllinux for just x86_64 too, so this is not too bad. + CIBW_SKIP: "*musllinux*aarch64*" jobs: build_wheels: @@ -22,38 +23,19 @@ jobs: env: CIBW_BUILD: ${{ matrix.cibw_build }} CIBW_ARCHS_LINUX: ${{ matrix.arch }} - CIBW_ARCHS_MACOS: ${{ matrix.arch }} + CIBW_ARCHS_MACOS: "x86_64 arm64" strategy: matrix: os: [ubuntu-latest, windows-latest, macos-latest] - arch: [auto64] - cibw_build: ["cp3{7,8,9,10,11}-*"] - p_ver: ["3.7-3.11"] - include: - - arch: arm64 - os: macos-latest - cibw_build: "cp3{7,8,9,10,11}-*" - p_ver: "3.7-3.11" - - arch: aarch64 - os: ubuntu-latest - cibw_build: "cp37*" - p_ver: "3.7" - - arch: aarch64 - os: ubuntu-latest - cibw_build: "cp38*" - p_ver: "3.8" - - arch: aarch64 - os: ubuntu-latest - cibw_build: "cp39*" - p_ver: "3.9" - - arch: aarch64 - os: ubuntu-latest - cibw_build: "cp310*" - p_ver: "3.10" - - arch: aarch64 - os: ubuntu-latest - cibw_build: "cp311*" - p_ver: "3.11" + arch: [x86_64, aarch64] + cibw_build: ["cp3{9,10,11,12}-*"] + p_ver: ["3.9-3.12"] + exclude: + - os: windows-latest + arch: aarch64 + # cibuild is already in charge to build aarch64 (see CIBW_ARCHS_MACOS) + - os: macos-latest + arch: aarch64 steps: - uses: actions/checkout@v3 @@ -70,11 +52,11 @@ jobs: - uses: docker/setup-qemu-action@v2 if: ${{ matrix.arch == 'aarch64' }} name: Set up QEMU - - - name: Build wheels for Python + + - name: Build wheels run: | python -m cibuildwheel --output-dir wheelhouse - + - name: Make sdist if: ${{ matrix.os == 'windows-latest' }} run: | diff --git a/ANNOUNCE.rst b/ANNOUNCE.rst index 2c74b1d..99139a7 100644 --- a/ANNOUNCE.rst +++ b/ANNOUNCE.rst @@ -1,15 +1,40 @@ ======================== -Announcing NumExpr 2.8.6 +Announcing NumExpr 2.8.7 ======================== -Hi everyone, +Hi everyone, -... +NumExpr 2.8.7 is a release to deal with issues related to downstream `pandas` +and other projects where the sanitization blacklist was triggering issue in their +evaluate. Hopefully, the new sanitization code would be much more robust now. + +For those who do not wish to have sanitization on by default, it can be changed +by setting an environment variable, `NUMEXPR_SANITIZE=0`. + +If you use `pandas` in your packages it is advisable you pin + +`numexpr >= 2.8.7` + +in your requirements. + +Project documentation is available at: + +http://numexpr.readthedocs.io/ Changes from 2.8.5 to 2.8.6 --------------------------- -** Under development ** +* More permissive rules in sanitizing regular expression: allow to access digits + after the . with scientific notation. Thanks to Thomas Vincent. + +* Don't reject double underscores that are not at the start or end of a variable + name (pandas uses those), or scientific-notation numbers with digits after the + decimal point. Thanks to Rebecca Palmer. + +* Do not use `numpy.alltrue` in the test suite, as it has been deprecated + (replaced by `numpy.all`). Thanks to Rebecca Chen. + +* Wheels for Python 3.12. Wheels for 3.7 and 3.8 are not generated anymore. What's Numexpr? --------------- diff --git a/README.rst b/README.rst index e0ae031..9bed14c 100644 --- a/README.rst +++ b/README.rst @@ -24,6 +24,22 @@ NumExpr: Fast numerical expression evaluator for NumPy .. |version| image:: https://img.shields.io/pypi/v/numexpr.png :target: https://pypi.python.org/pypi/numexpr +IMPORTANT NOTE: NumExpr is looking for maintainers! +--------------------------------------------------- + +After 5 years as a solo maintainer (and performing a most excellent work), Robert McLeod +is asking for a well deserved break. So the NumExpr project is looking for a new +maintainer for a package that is used in pandas, PyTables and many other packages. +If have benefited of NumExpr capabilities in the past, and willing to contribute back to +the community, we would be happy to hear about you! + +We are looking for someone that is knowledgeable about compiling extensions, and that is +ready to spend some cycles in making releases (2 or 3 a year, maybe even less!). +Interested? just open a new ticket here and we will help you onboarding! + +Thank you! + + What is NumExpr? ---------------- diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 4804fee..429f806 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -5,7 +5,17 @@ Release notes for NumExpr 2.8 series Changes from 2.8.6 to 2.8.7 --------------------------- -** Under development ** +* More permissive rules in sanitizing regular expression: allow to access digits + after the . with scientific notation. Thanks to Thomas Vincent. + +* Don't reject double underscores that are not at the start or end of a variable + name (pandas uses those), or scientific-notation numbers with digits after the + decimal point. Thanks to Rebecca Palmer. + +* Do not use `numpy.alltrue` in the test suite, as it has been deprecated + (replaced by `numpy.all`). Thanks to Rebecca Chen. + +* Wheels for Python 3.12. Wheels for 3.7 and 3.8 are not generated anymore. Changes from 2.8.5 to 2.8.6 --------------------------- @@ -14,28 +24,28 @@ Changes from 2.8.5 to 2.8.6 `set NUMEXPR_SANITIZE=0` -* Improved behavior of the blacklist to avoid triggering on private variables +* Improved behavior of the blacklist to avoid triggering on private variables and scientific notation numbers. Changes from 2.8.4 to 2.8.5 --------------------------- -* A `validate` function has been added. This function checks the inputs, returning - `None` on success or raising an exception on invalid inputs. This function was +* A `validate` function has been added. This function checks the inputs, returning + `None` on success or raising an exception on invalid inputs. This function was added as numerous projects seem to be using NumExpr for parsing user inputs. `re_evaluate` may be called directly following `validate`. * As an addendum to the use of NumExpr for parsing user inputs, is that NumExpr - calls `eval` on the inputs. A regular expression is now applied to help sanitize - the input expression string, forbidding '__', ':', and ';'. Attribute access + calls `eval` on the inputs. A regular expression is now applied to help sanitize + the input expression string, forbidding '__', ':', and ';'. Attribute access is also banned except for '.r' for real and '.i' for imag. -* Thanks to timbrist for a fix to behavior of NumExpr with integers to negative - powers. NumExpr was pre-checking integer powers for negative values, which +* Thanks to timbrist for a fix to behavior of NumExpr with integers to negative + powers. NumExpr was pre-checking integer powers for negative values, which was both inefficient and caused parsing errors in some situations. Now NumExpr - will simply return 0 as a result for such cases. While NumExpr generally tries - to follow NumPy behavior, performance is also critical. -* Thanks to peadar for some fixes to how NumExpr launches threads for embedded + will simply return 0 as a result for such cases. While NumExpr generally tries + to follow NumPy behavior, performance is also critical. +* Thanks to peadar for some fixes to how NumExpr launches threads for embedded applications. -* Thanks to de11n for making parsing of the `site.cfg` for MKL consistent among +* Thanks to de11n for making parsing of the `site.cfg` for MKL consistent among all shared platforms. @@ -46,19 +56,19 @@ Changes from 2.8.3 to 2.8.4 * Thanks to Tobias Hangleiter for an improved accuracy complex `expm1` function. While it is 25 % slower, it is significantly more accurate for the real component over a range of values and matches NumPy outputs much more closely. -* Thanks to Kirill Kouzoubov for a range of fixes to constants parsing that was +* Thanks to Kirill Kouzoubov for a range of fixes to constants parsing that was resulting in duplicated constants of the same value. -* Thanks to Mark Harfouche for noticing that we no longer need `numpy` version +* Thanks to Mark Harfouche for noticing that we no longer need `numpy` version checks. `packaging` is no longer a requirement as a result. Changes from 2.8.1 to 2.8.3 --------------------------- * 2.8.2 was skipped due to an error in uploading to PyPi. -* Support for Python 3.6 has been dropped due to the need to substitute the flag - `NPY_ARRAY_WRITEBACKIFCOPY` for `NPY_ARRAY_UPDATEIFCOPY`. This flag change was - initiated in NumPy 1.14 and finalized in 1.23. The only changes were made to - cases where an unaligned constant was passed in with a pre-allocated output +* Support for Python 3.6 has been dropped due to the need to substitute the flag + `NPY_ARRAY_WRITEBACKIFCOPY` for `NPY_ARRAY_UPDATEIFCOPY`. This flag change was + initiated in NumPy 1.14 and finalized in 1.23. The only changes were made to + cases where an unaligned constant was passed in with a pre-allocated output variable: ``` @@ -66,13 +76,13 @@ x = np.empty(5, dtype=np.uint8)[1:].view(np.int32) ne.evaluate('3', out=x) ``` - We think the risk of issues is very low, but if you are using NumExpr as a + We think the risk of issues is very low, but if you are using NumExpr as a expression evaluation tool you may want to write a test for this edge case. * Thanks to Matt Einhorn (@matham) for improvements to the GitHub Actions build process to add support for Apple Silicon and aarch64. * Thanks to Biswapriyo Nath (@biswa96) for a fix to allow `mingw` builds on Windows. * There have been some changes made to not import `platform.machine()` on `sparc` - but it is highly advised to upgrade to Python 3.9+ to avoid this issue with + but it is highly advised to upgrade to Python 3.9+ to avoid this issue with the Python core package `platform`. @@ -80,25 +90,25 @@ Changes from 2.8.0 to 2.8.1 --------------------------- * Fixed dependency list. -* Added ``pyproject.toml`` and modernize the ``setup.py`` script. Thanks to +* Added ``pyproject.toml`` and modernize the ``setup.py`` script. Thanks to Antonio Valentino for the PR. Changes from 2.7.3 to 2.8.0 --------------------------- * Wheels for Python 3.10 are now provided. -* Support for Python 2.7 and 3.5 has been discontinued. -* All residual support for Python 2.X syntax has been removed, and therefore - the setup build no longer makes calls to the `2to3` script. The `setup.py` +* Support for Python 2.7 and 3.5 has been discontinued. +* All residual support for Python 2.X syntax has been removed, and therefore + the setup build no longer makes calls to the `2to3` script. The `setup.py` has been refactored to be more modern. -* The examples on how to link into Intel VML/MKL/oneAPI now use the dynamic +* The examples on how to link into Intel VML/MKL/oneAPI now use the dynamic library. Changes from 2.7.2 to 2.7.3 --------------------------- -- Pinned Numpy versions to minimum supported version in an effort to alleviate - issues seen in Windows machines not having the same MSVC runtime installed as +- Pinned Numpy versions to minimum supported version in an effort to alleviate + issues seen in Windows machines not having the same MSVC runtime installed as was used to build the wheels. - ARMv8 wheels are now available, thanks to `odidev` for the pull request. @@ -106,22 +116,22 @@ Changes from 2.7.2 to 2.7.3 Changes from 2.7.1 to 2.7.2 --------------------------- -- Support for Python 2.7 and 3.5 is deprecated and will be discontinued when +- Support for Python 2.7 and 3.5 is deprecated and will be discontinued when `cibuildwheels` and/or GitHub Actions no longer support these versions. -- Wheels are now provided for Python 3.7, 3.5, 3.6, 3.7, 3.8, and 3.9 via +- Wheels are now provided for Python 3.7, 3.5, 3.6, 3.7, 3.8, and 3.9 via GitHub Actions. - The block size is now exported into the namespace as `numexpr.__BLOCK_SIZE1__` as a read-only value. -- If using MKL, the number of threads for VML is no longer forced to 1 on loading - the module. Testing has shown that VML never runs in multi-threaded mode for - the default BLOCKSIZE1 of 1024 elements, and forcing to 1 can have deleterious +- If using MKL, the number of threads for VML is no longer forced to 1 on loading + the module. Testing has shown that VML never runs in multi-threaded mode for + the default BLOCKSIZE1 of 1024 elements, and forcing to 1 can have deleterious effects on NumPy functions when built with MKL. See issue #355 for details. -- Use of `ndarray.tostring()` in tests has been switch to `ndarray.tobytes()` - for future-proofing deprecation of `.tostring()`, if the version of NumPy is +- Use of `ndarray.tostring()` in tests has been switch to `ndarray.tobytes()` + for future-proofing deprecation of `.tostring()`, if the version of NumPy is greater than 1.9. -- Added a utility method `get_num_threads` that returns the (maximum) number of - threads currently in use by the virtual machine. The functionality of - `set_num_threads` whereby it returns the previous value has been deprecated +- Added a utility method `get_num_threads` that returns the (maximum) number of + threads currently in use by the virtual machine. The functionality of + `set_num_threads` whereby it returns the previous value has been deprecated and will be removed in 2.8.X. Changes from 2.7.0 to 2.7.1 @@ -136,75 +146,75 @@ Changes from 2.7.0 to 2.7.1 Changes from 2.6.9 to 2.7.0 ---------------------------- -- The default number of 'safe' threads has been restored to the historical limit +- The default number of 'safe' threads has been restored to the historical limit of 8, if the environment variable "NUMEXPR_MAX_THREADS" has not been set. - Thanks to @eltoder who fixed a small memory leak. -- Support for Python 2.6 has been dropped, as it is no longer available via +- Support for Python 2.6 has been dropped, as it is no longer available via TravisCI. -- A typo in the test suite that had a less than rather than greater than symbol +- A typo in the test suite that had a less than rather than greater than symbol in the NumPy version check has been corrected thanks to dhomeier. -- The file `site.cfg` was being accidently included in the sdists on PyPi. +- The file `site.cfg` was being accidently included in the sdists on PyPi. It has now been excluded. Changes from 2.6.8 to 2.6.9 --------------------------- -- Thanks to Mike Toews for more robust handling of the thread-setting +- Thanks to Mike Toews for more robust handling of the thread-setting environment variables. -- With Appveyor updating to Python 3.7.1, wheels for Python 3.7 are now +- With Appveyor updating to Python 3.7.1, wheels for Python 3.7 are now available in addition to those for other OSes. Changes from 2.6.7 to 2.6.8 --------------------------- -- Add check to make sure that `f_locals` is not actually `f_globals` when we +- Add check to make sure that `f_locals` is not actually `f_globals` when we do the `f_locals` clear to avoid the #310 memory leak issue. - Compare NumPy versions using `distutils.version.LooseVersion` to avoid issue #312 when working with NumPy development versions. -- As part of `multibuild`, wheels for Python 3.7 for Linux and MacOSX are now +- As part of `multibuild`, wheels for Python 3.7 for Linux and MacOSX are now available on PyPI. Changes from 2.6.6 to 2.6.7 --------------------------- - Thanks to Lehman Garrison for finding and fixing a bug that exhibited memory - leak-like behavior. The use in `numexpr.evaluate` of `sys._getframe` combined - with `.f_locals` from that frame object results an extra refcount on objects - in the frame that calls `numexpr.evaluate`, and not `evaluate`'s frame. So if - the calling frame remains in scope for a long time (such as a procedural - script where `numexpr` is called from the base frame) garbage collection would + leak-like behavior. The use in `numexpr.evaluate` of `sys._getframe` combined + with `.f_locals` from that frame object results an extra refcount on objects + in the frame that calls `numexpr.evaluate`, and not `evaluate`'s frame. So if + the calling frame remains in scope for a long time (such as a procedural + script where `numexpr` is called from the base frame) garbage collection would never occur. - Imports for the `numexpr.test` submodule were made lazy in the `numexpr` module. Changes from 2.6.5 to 2.6.6 --------------------------- -- Thanks to Mark Dickinson for a fix to the thread barrier that occassionally +- Thanks to Mark Dickinson for a fix to the thread barrier that occassionally suffered from spurious wakeups on MacOSX. Changes from 2.6.4 to 2.6.5 --------------------------- -- The maximum thread count can now be set at import-time by setting the - environment variable 'NUMEXPR_MAX_THREADS'. The default number of +- The maximum thread count can now be set at import-time by setting the + environment variable 'NUMEXPR_MAX_THREADS'. The default number of max threads was lowered from 4096 (which was deemed excessive) to 64. -- A number of imports were removed (pkg_resources) or made lazy (cpuinfo) in - order to speed load-times for downstream packages (such as `pandas`, `sympy`, - and `tables`). Import time has dropped from about 330 ms to 90 ms. Thanks to +- A number of imports were removed (pkg_resources) or made lazy (cpuinfo) in + order to speed load-times for downstream packages (such as `pandas`, `sympy`, + and `tables`). Import time has dropped from about 330 ms to 90 ms. Thanks to Jason Sachs for pointing out the source of the slow-down. -- Thanks to Alvaro Lopez Ortega for updates to benchmarks to be compatible with +- Thanks to Alvaro Lopez Ortega for updates to benchmarks to be compatible with Python 3. - Travis and AppVeyor now fail if the test module fails or errors. -- Thanks to Mahdi Ben Jelloul for a patch that removed a bug where constants +- Thanks to Mahdi Ben Jelloul for a patch that removed a bug where constants in `where` calls would raise a ValueError. -- Fixed a bug whereby all-constant power operations would lead to infinite +- Fixed a bug whereby all-constant power operations would lead to infinite recursion. Changes from 2.6.3 to 2.6.4 --------------------------- -- Christoph Gohlke noticed a lack of coverage for the 2.6.3 - `floor` and `ceil` functions for MKL that caused seg-faults in +- Christoph Gohlke noticed a lack of coverage for the 2.6.3 + `floor` and `ceil` functions for MKL that caused seg-faults in test, so thanks to him for that. Changes from 2.6.2 to 2.6.3 @@ -213,19 +223,19 @@ Changes from 2.6.2 to 2.6.3 - Documentation now available at readthedocs.io_. - Support for floor() and ceil() functions added by Caleb P. Burns. - + - NumPy requirement increased from 1.6 to 1.7 due to changes in iterator flags (#245). - + - Sphinx autodocs support added for documentation on readthedocs.org. -- Fixed a bug where complex constants would return an error, fixing +- Fixed a bug where complex constants would return an error, fixing problems with `sympy` when using NumExpr as a backend. - -- Fix for #277 whereby arrays of shape (1,...) would be reduced as + +- Fix for #277 whereby arrays of shape (1,...) would be reduced as if they were full reduction. Behavoir now matches that of NumPy. -- String literals are automatically encoded into 'ascii' bytes for +- String literals are automatically encoded into 'ascii' bytes for convience (see #281). .. _readthedocs.io: http://numexpr.readthedocs.io @@ -428,7 +438,7 @@ Changes from 2.2.1 to 2.2.2 #115. * The `__nonzero__` method in `ExpressionNode` class has been - commented out. This is also for compatibility with `PyTables < 3.0`. + commented out. This is also for compatibility with `PyTables < 3.0`. See #24 for details. * Fixed the type of some parameters in the C extension so that s390 diff --git a/numexpr/tests/test_numexpr.py b/numexpr/tests/test_numexpr.py index f338dba..c0a016e 100644 --- a/numexpr/tests/test_numexpr.py +++ b/numexpr/tests/test_numexpr.py @@ -511,7 +511,6 @@ def test_illegal_value(self): def test_sanitize(self): with _environment('NUMEXPR_SANITIZE', '1'): - print('Sanitize') # Forbid dunder try: evaluate('__builtins__') diff --git a/setup.cfg b/setup.cfg index 8d2514e..36856a6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = numexpr -version = 2.8.7.dev1 +version = 2.8.7 description = Fast numerical expression evaluator for NumPy author = David M. Cooke, Francesc Alted, and others maintainer = Robert A. McLeod @@ -15,11 +15,10 @@ classifiers = Intended Audience :: Science/Research License :: OSI Approved :: MIT License Programming Language :: Python :: 3 - Programming Language :: Python :: 3.7 - Programming Language :: Python :: 3.8 Programming Language :: Python :: 3.9 Programming Language :: Python :: 3.10 Programming Language :: Python :: 3.11 + Programming Language :: Python :: 3.12 Operating System :: Microsoft :: Windows Operating System :: POSIX Operating System :: MacOS @@ -27,7 +26,7 @@ classifiers = [options] zip_safe = False packages = find: -python_requires = >=3.7 +python_requires = >=3.9 # install_requires = # numpy