Merge pull request #120 from ashvardanian/main-dev

Unlocking GIL on writes
ashvardanian · Mar 18, 2024 · e0bcdf8 · e0bcdf8
2 parents f929eee + 7c910bf
commit e0bcdf8
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 24 deletions.
diff --git a/.github/workflows/prerelease.yml b/.github/workflows/prerelease.yml
@@ -106,14 +106,6 @@ jobs:
     env:
       CC: clang-16
       CXX: clang++-16
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-          - arch: amd64
-            target: x86_64-linux-gnu
-          - arch: arm64
-            target: aarch64-linux-gnu
 
     steps:
       - uses: actions/checkout@v4
@@ -253,9 +245,9 @@ jobs:
         # We can't run the produced builds, but we can make sure they exist
       - name: Test artifacts presense
         run: |
-          test -f build_artifacts/stringzilla_test_cpp20
-          test -f build_artifacts/libstringzilla.so
-          test -f build_artifacts/libstringzillite.so
+          test -e build_artifacts/libstringzillite.so
+          test -e build_artifacts/libstringzilla_shared.so
+          test -e build_artifacts/stringzilla_test_cpp20
 
   test_macos:
     name: MacOS

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -204,18 +204,15 @@ jobs:
 
           cmake --build build_release --config Release
 
-          # Copy and package the library
-          echo "ARTIFACT_NAME=stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}" >> $GITHUB_ENV
-          cp build_release/libstringzillite.so "$ARTIFACT_NAME.so"
-
-          mkdir -p "$ARTIFACT_NAME/DEBIAN"
-          touch "$ARTIFACT_NAME/DEBIAN/control"
-          mkdir -p "$ARTIFACT_NAME/usr/local/lib"
-          mkdir "$ARTIFACT_NAME/usr/local/include"
-          cp include/stringzilla/stringzilla.h "$ARTIFACT_NAME/usr/local/include/"
-          cp build_release/libstringzillite.so "$ARTIFACT_NAME/usr/local/lib/"
-          echo "Package: stringzilla\nVersion: ${{ steps.set_version.outputs.version }}\nMaintainer: Ash Vardanian\nArchitecture: ${{ matrix.arch }}\nDescription: SIMD-accelerated string search, sort, hashes, fingerprints, & edit distances" > "$ARTIFACT_NAME/DEBIAN/control"
-          dpkg-deb --build "$ARTIFACT_NAME"
+          cp build_release/libstringzillite.so "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}.so"
+          mkdir -p "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/DEBIAN"
+          touch "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/DEBIAN/control"
+          mkdir -p "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/lib"
+          mkdir "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/include"
+          cp include/stringzilla/stringzilla.h "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/include/"
+          cp build_release/libstringzillite.so "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/usr/local/lib/"
+          echo "Package: stringzilla\nVersion: ${{ steps.set_version.outputs.version }}\nMaintainer: Ash Vardanian\nArchitecture: ${{ matrix.arch }}\nDescription: SIMD-accelerated string search, sort, hashes, fingerprints, & edit distances" > "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}/DEBIAN/control"
+          dpkg-deb --build "stringzillite_linux_${{ matrix.arch }}_${{ steps.set_version.outputs.version }}"
 
       - name: Upload library
         uses: xresloader/upload-to-github-release@v1

diff --git a/README.md b/README.md
@@ -518,7 +518,7 @@ The `Str` class has `write_to` to write the string to a file, and `offset_within
 web_archieve = Str("<html>...</html><html>...</html>")
 _, end_tag, next_doc = web_archieve.partition("</html>") # or use `find`
 next_doc_offset = next_doc.offset_within(web_archieve)
-web_archieve.write_to("next_doc.html")
+web_archieve.write_to("next_doc.html") # no GIL, no copies, just a view
 ```
 
 #### PyArrow

diff --git a/python/lib.c b/python/lib.c
@@ -1271,16 +1271,23 @@ static PyObject *Str_write_to(PyObject *self, PyObject *args, PyObject *kwargs)
         return NULL;
     }
     memcpy(path_buffer, path.start, path.length);
+    path_buffer[path.length] = '\0';
 
+    // Unlock the Global Interpreter Lock (GIL) to allow other threads to run
+    // while the current thread is waiting for the file to be written.
+    PyThreadState *gil_state = PyEval_SaveThread();
     FILE *file_pointer = fopen(path_buffer, "wb");
     if (file_pointer == NULL) {
+        PyEval_RestoreThread(gil_state);
         PyErr_SetFromErrnoWithFilename(PyExc_OSError, path_buffer);
         free(path_buffer);
+        PyEval_RestoreThread(gil_state);
         return NULL;
     }
 
     setbuf(file_pointer, NULL); // Set the stream to unbuffered
     int status = fwrite(text.start, 1, text.length, file_pointer);
+    PyEval_RestoreThread(gil_state);
     if (status != text.length) {
         PyErr_SetFromErrnoWithFilename(PyExc_OSError, path_buffer);
         free(path_buffer);

diff --git a/scripts/test.py b/scripts/test.py
@@ -1,6 +1,8 @@
 from random import choice, randint
 from string import ascii_lowercase
 from typing import Optional
+import tempfile
+import os
 
 import pytest
 
@@ -104,6 +106,25 @@ def test_unit_buffer_protocol():
     assert "".join([c.decode("utf-8") for c in arr.tolist()]) == "hello"
 
 
+def test_str_write_to():
+    native = "line1\nline2\nline3"
+    big = Str(native)
+
+    # Create a temporary file
+    with tempfile.NamedTemporaryFile(delete=False) as tmpfile:
+        temp_filename = tmpfile.name  # Store the name for later use
+
+    try:
+        big.write_to(temp_filename)
+        with open(temp_filename, "r") as file:
+            content = file.read()
+            assert (
+                content == native
+            ), "The content of the file does not match the expected output"
+    finally:
+        os.remove(temp_filename)
+
+
 def test_unit_split():
     native = "line1\nline2\nline3"
     big = Str(native)