Initialize basic tests

.github/workflows/CI.yml

@@ -0,0 +1,48 @@
+name: CI
+on:
+  push:
+    branches: [main]
+    tags: ["*"]
+  pull_request:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: true
+jobs:
+  test:
+    name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }}
+    env:
+      JULIA_NUM_THREADS: 2
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - '1.9'
+          - '1.10-nightly'
+        os:
+          - ubuntu-latest
+          - macOS-latest
+        arch:
+          - x64
+    steps:
+      - uses: actions/[email protected]
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: actions/cache@v2
+        env:
+          cache-name: cache-artifacts
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-test-${{ env.cache-name }}-
+            ${{ runner.os }}-test-
+            ${{ runner.os }}-
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v3
+        with:
+          file: lcov.info

.github/workflows/CompatHelper.yml

@@ -0,0 +1,45 @@
+name: CompatHelper
+on:
+  schedule:
+    - cron: 0 0 * * *
+  workflow_dispatch:
+permissions:
+  contents: write
+  pull-requests: write
+jobs:
+  CompatHelper:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if Julia is already available in the PATH
+        id: julia_in_path
+        run: which julia
+        continue-on-error: true
+      - name: Install Julia, but only if it is not already available in the PATH
+        uses: julia-actions/setup-julia@v1
+        with:
+          version: '1'
+          # arch: ${{ runner.arch }}
+        if: steps.julia_in_path.outcome != 'success'
+      - name: "Add the General registry via Git"
+        run: |
+          import Pkg
+          ENV["JULIA_PKG_SERVER"] = ""
+          Pkg.Registry.add("General")
+        shell: julia --color=yes {0}
+      - name: "Install CompatHelper"
+        run: |
+          import Pkg
+          name = "CompatHelper"
+          uuid = "aa819f21-2bde-4658-8897-bab36330d9b7"
+          version = "3"
+          Pkg.add(; name, uuid, version)
+        shell: julia --color=yes {0}
+      - name: "Run CompatHelper"
+        run: |
+          import CompatHelper
+          CompatHelper.main()
+        shell: julia --color=yes {0}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }}
+          # COMPATHELPER_PRIV: ${{ secrets.COMPATHELPER_PRIV }}

.github/workflows/JuliaNightly.yml

@@ -0,0 +1,50 @@
+# CI for Julia nightly, separate workflow to avoid failing CI badge on nightly fail
+name: JuliaNightly
+on:
+  push:
+    branches: [master]
+    tags: [v*]
+  pull_request:
+  schedule:
+    - cron: "0 0 * * *"
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
+  cancel-in-progress: true
+jobs:
+  test:
+    name: ${{ matrix.os }} - ${{ matrix.arch }}
+    env:
+      JULIA_NUM_THREADS: 2
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        version:
+          - 'nightly'
+        os:
+          - ubuntu-latest
+        arch:
+          - x64
+    steps:
+      - uses: actions/[email protected]
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: ${{ matrix.version }}
+          arch: ${{ matrix.arch }}
+      - uses: actions/cache@v2
+        env:
+          cache-name: cache-artifacts
+        with:
+          path: ~/.julia/artifacts
+          key: ${{ runner.os }}-${{ matrix.arch }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.arch }}-test-${{ env.cache-name }}-
+            ${{ runner.os }}-${{ matrix.arch }}-test-
+            ${{ runner.os }}-${{ matrix.arch }}-
+            ${{ runner.os }}-
+      - uses: julia-actions/julia-buildpkg@v1
+      - uses: julia-actions/julia-runtest@v1
+      - uses: julia-actions/julia-processcoverage@v1
+      - uses: codecov/codecov-action@v1
+        with:
+          file: lcov.info

.github/workflows/TagBot.yml

@@ -0,0 +1,28 @@
+name: TagBot
+on:
+  issue_comment:
+    types:
+      - created
+  workflow_dispatch:
+permissions:
+  actions: read
+  checks: read
+  contents: write
+  deployments: read
+  issues: read
+  discussions: read
+  packages: read
+  pages: read
+  pull-requests: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+jobs:
+  TagBot:
+    if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: JuliaRegistries/TagBot@v1
+        with:
+            ssh: ${{ secrets.DOCUMENTER_KEY }}
+            token: ${{ secrets.GITHUB_TOKEN }}

Project.toml

@@ -1,3 +1,17 @@
 name = "ObjectStore"
 uuid = "1b5eed3d-1f46-4baa-87f3-a4a892b23610"
 version = "0.1.0"
+
+[compat]
+CloudBase = "1"
+ReTestItems = "1"
+Test = "1"
+julia = "1.8"
+
+[extras]
+CloudBase = "85eb1798-d7c4-4918-bb13-c944d38e27ed"
+ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["CloudBase", "ReTestItems", "Test"]

src/ObjectStore.jl

@@ -1,5 +1,7 @@
 module ObjectStore
 
+export init_rust_store, blob_get!, blob_put, AzureCredentials
+
 const rust_lib_dir = @static if Sys.islinux()
     joinpath(
         @__DIR__,
@@ -35,14 +37,16 @@ end
 
 const rust_lib = joinpath(rust_lib_dir, "librust_store.$extension")
 
-RUST_STORE_STARTED = false
+const RUST_STORE_STARTED = Ref(false)
+const _INIT_LOCK::ReentrantLock = ReentrantLock()
 function init_rust_store()
-    global RUST_STORE_STARTED
-    if RUST_STORE_STARTED
-        return
+    Base.@lock _INIT_LOCK begin
+        if RUST_STORE_STARTED[]
+            return
+        end
+        @ccall rust_lib.start()::Cint
+        RUST_STORE_STARTED[] = true
     end
-    @ccall rust_lib.start()::Cint
-    RUST_STORE_STARTED = true
 end
 
 struct AzureCredentials
@@ -51,23 +55,22 @@ struct AzureCredentials
     key::String
     host::String
 end
+const _AzureCredentialsFFI = NTuple{4,Cstring}
 
-struct AzureCredentialsFFI
-    account::Cstring
-    container::Cstring
-    key::Cstring
-    host::Cstring
-end
-
-function to_ffi(credentials::AzureCredentials)
-    AzureCredentialsFFI(
+function Base.cconvert(::Type{Ref{AzureCredentials}}, credentials::AzureCredentials)
+   credentials_ffi = (
         Base.unsafe_convert(Cstring, Base.cconvert(Cstring, credentials.account)),
         Base.unsafe_convert(Cstring, Base.cconvert(Cstring, credentials.container)),
         Base.unsafe_convert(Cstring, Base.cconvert(Cstring, credentials.key)),
         Base.unsafe_convert(Cstring, Base.cconvert(Cstring, credentials.host))
-    )
+    )::_AzureCredentialsFFI
+    # cconvert ensures its outputs are preserved during a ccall, so we can crate a pointer
+    # safely in the unsafe_convert call.
+    return credentials_ffi, Ref(credentials_ffi)
+end
+function Base.unsafe_convert(::Type{Ref{AzureCredentials}}, x::Tuple{T,Ref{T}}) where {T<:_AzureCredentialsFFI}
+    return Base.unsafe_convert(Ptr{_AzureCredentialsFFI}, x[2])
 end
-
 
 struct Response
     result::Cint
@@ -79,7 +82,6 @@ end
 
 function blob_get!(path::String, buffer::AbstractVector{UInt8}, credentials::AzureCredentials)
     response = Ref(Response())
-    credentials_ffi = Ref(to_ffi(credentials))
     size = length(buffer)
     cond = Base.AsyncCondition()
     cond_handle = cond.handle
@@ -88,7 +90,7 @@ function blob_get!(path::String, buffer::AbstractVector{UInt8}, credentials::Azu
             path::Cstring,
             buffer::Ref{Cuchar},
             size::Culonglong,
-            credentials_ffi::Ref{AzureCredentialsFFI},
+            credentials::Ref{AzureCredentials},
             response::Ref{Response},
             cond_handle::Ptr{Cvoid}
         )::Cint
@@ -115,9 +117,8 @@ function blob_get!(path::String, buffer::AbstractVector{UInt8}, credentials::Azu
     end
 end
 
-function blob_put!(path::String, buffer::AbstractVector{UInt8}, credentials::AzureCredentials)
+function blob_put(path::String, buffer::AbstractVector{UInt8}, credentials::AzureCredentials)
     response = Ref(Response())
-    credentials_ffi = Ref(to_ffi(credentials))
     size = length(buffer)
     cond = Base.AsyncCondition()
     cond_handle = cond.handle
@@ -126,7 +127,7 @@ function blob_put!(path::String, buffer::AbstractVector{UInt8}, credentials::Azu
             path::Cstring,
             buffer::Ref{Cuchar},
             size::Culonglong,
-            credentials_ffi::Ref{AzureCredentialsFFI},
+            credentials::Ref{AzureCredentials},
             response::Ref{Response},
             cond_handle::Ptr{Cvoid}
         )::Cint

test/azure_blobs_tests.jl

@@ -0,0 +1,80 @@
+@testitem "Basic BlobStorage tests" begin
+
+using CloudBase.CloudTest: Azurite
+using ObjectStore: blob_get!, blob_put, AzureCredentials
+import ObjectStore
+
+ObjectStore.init_rust_store()
+
+# For interactive testing, use Azurite.run() instead of Azurite.with()
+# conf, p = Azurite.run(; debug=true); atexit(() -> kill(p))
+Azurite.with(; debug=true, public=false) do conf
+    _credentials, _container = conf
+    base_url = _container.baseurl
+    credentials = AzureCredentials(_credentials.auth.account, _container.name, _credentials.auth.key, base_url)
+
+    @testset "0B file" begin
+        buffer = Vector{UInt8}(undef, 1000)
+
+        nbytes_written = blob_put(joinpath(base_url, "empty.csv"), codeunits(""), credentials)
+        @test nbytes_written == 0
+
+        nbytes_read = blob_get!(joinpath(base_url, "empty.csv"), buffer, credentials)
+        @test nbytes_read == 0
+    end
+
+    @testset "100B file" begin
+        input = "1,2,3,4,5,6,7,8,9,1\n" ^ 5
+        buffer = Vector{UInt8}(undef, 1000)
+        @assert sizeof(input) == 100
+        @assert sizeof(buffer) > sizeof(input)
+
+        nbytes_written = blob_put(joinpath(base_url, "test100B.csv"), codeunits(input), credentials)
+        @test nbytes_written == 100
+
+        nbytes_read = blob_get!(joinpath(base_url, "test100B.csv"), buffer, credentials)
+        @test nbytes_read == 100
+        @test String(buffer[1:nbytes_read]) == input
+    end
+
+    @testset "1MB file" begin
+        input = "1,2,3,4,5,6,7,8,9,1\n" ^ 50_000
+        buffer = Vector{UInt8}(undef, 1_000_000)
+        @assert sizeof(input) == 1_000_000 == sizeof(buffer)
+
+        nbytes_written = blob_put(joinpath(base_url, "test100B.csv"), codeunits(input), credentials)
+        @test nbytes_written == 1_000_000
+
+        nbytes_read = blob_get!(joinpath(base_url, "test100B.csv"), buffer, credentials)
+        @test nbytes_read == 1_000_000
+        @test String(buffer[1:nbytes_read]) == input
+    end
+
+    @testset "20MB file" begin
+        input = "1,2,3,4,5,6,7,8,9,1\n" ^ 1_000_000
+        buffer = Vector{UInt8}(undef, 20_000_000)
+        @assert sizeof(input) == 20_000_000 == sizeof(buffer)
+
+        nbytes_written = blob_put(joinpath(base_url, "test100B.csv"), codeunits(input), credentials)
+        @test nbytes_written == 20_000_000
+
+        nbytes_read = blob_get!(joinpath(base_url, "test100B.csv"), buffer, credentials)
+        @test nbytes_read == 20_000_000
+        @test String(buffer[1:nbytes_read]) == input
+    end
+
+    # If the buffer is too small, we hang
+    # @testset "100B file, buffer too small" begin
+    #     input = "1,2,3,4,5,6,7,8,9,1\n" ^ 5
+    #     buffer = Vector{UInt8}(undef, 10)
+    #     @assert sizeof(input) == 100
+    #     @assert sizeof(buffer) < sizeof(input)
+
+    #     nbytes_written = blob_put(joinpath(base_url, "test100B.csv"), codeunits(input), credentials)
+    #     @test nbytes_written == 100
+
+    #     nbytes_read = blob_get!(joinpath(base_url, "test100B.csv"), buffer, credentials)
+    # end
+end # Azurite.with
+
+end # @testitem

test/runtests.jl

@@ -0,0 +1,6 @@
+using ReTestItems
+using ObjectStore
+
+withenv("RUST_BACKTRACE"=>1) do
+    runtests(ObjectStore, testitem_timeout=30, nworkers=1)
+end