diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000..6726852 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,48 @@ +name: CI +on: + push: + branches: [main] + tags: ["*"] + pull_request: +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true +jobs: + test: + name: Julia ${{ matrix.version }} - ${{ matrix.os }} - ${{ matrix.arch }} - ${{ github.event_name }} + env: + JULIA_NUM_THREADS: 2 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - '1.9' + - '1.10-nightly' + os: + - ubuntu-latest + - macOS-latest + arch: + - x64 + steps: + - uses: actions/checkout@v3.5.0 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: actions/cache@v2 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v3 + with: + file: lcov.info diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml new file mode 100644 index 0000000..7ddad2a --- /dev/null +++ b/.github/workflows/CompatHelper.yml @@ -0,0 +1,45 @@ +name: CompatHelper +on: + schedule: + - cron: 0 0 * * * + workflow_dispatch: +permissions: + contents: write + pull-requests: write +jobs: + CompatHelper: + runs-on: ubuntu-latest + steps: + - name: Check if Julia is already available in the PATH + id: julia_in_path + run: which julia + continue-on-error: true + - name: Install Julia, but only if it is not already available in the PATH + uses: julia-actions/setup-julia@v1 + with: + version: '1' + # arch: ${{ runner.arch }} + if: steps.julia_in_path.outcome != 'success' + - name: "Add the General registry via Git" + run: | + import Pkg + ENV["JULIA_PKG_SERVER"] = "" + Pkg.Registry.add("General") + shell: julia --color=yes {0} + - name: "Install CompatHelper" + run: | + import Pkg + name = "CompatHelper" + uuid = "aa819f21-2bde-4658-8897-bab36330d9b7" + version = "3" + Pkg.add(; name, uuid, version) + shell: julia --color=yes {0} + - name: "Run CompatHelper" + run: | + import CompatHelper + CompatHelper.main() + shell: julia --color=yes {0} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }} + # COMPATHELPER_PRIV: ${{ secrets.COMPATHELPER_PRIV }} diff --git a/.github/workflows/JuliaNightly.yml b/.github/workflows/JuliaNightly.yml new file mode 100644 index 0000000..66d5cdf --- /dev/null +++ b/.github/workflows/JuliaNightly.yml @@ -0,0 +1,50 @@ +# CI for Julia nightly, separate workflow to avoid failing CI badge on nightly fail +name: JuliaNightly +on: + push: + branches: [master] + tags: [v*] + pull_request: + schedule: + - cron: "0 0 * * *" +concurrency: + group: ${{ github.workflow }}-${{ github.ref || github.run_id }} + cancel-in-progress: true +jobs: + test: + name: ${{ matrix.os }} - ${{ matrix.arch }} + env: + JULIA_NUM_THREADS: 2 + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + version: + - 'nightly' + os: + - ubuntu-latest + arch: + - x64 + steps: + - uses: actions/checkout@v3.5.0 + - uses: julia-actions/setup-julia@v1 + with: + version: ${{ matrix.version }} + arch: ${{ matrix.arch }} + - uses: actions/cache@v2 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-${{ matrix.arch }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-${{ matrix.arch }}-test-${{ env.cache-name }}- + ${{ runner.os }}-${{ matrix.arch }}-test- + ${{ runner.os }}-${{ matrix.arch }}- + ${{ runner.os }}- + - uses: julia-actions/julia-buildpkg@v1 + - uses: julia-actions/julia-runtest@v1 + - uses: julia-actions/julia-processcoverage@v1 + - uses: codecov/codecov-action@v1 + with: + file: lcov.info diff --git a/.github/workflows/TagBot.yml b/.github/workflows/TagBot.yml new file mode 100644 index 0000000..103a5ac --- /dev/null +++ b/.github/workflows/TagBot.yml @@ -0,0 +1,28 @@ +name: TagBot +on: + issue_comment: + types: + - created + workflow_dispatch: +permissions: + actions: read + checks: read + contents: write + deployments: read + issues: read + discussions: read + packages: read + pages: read + pull-requests: read + repository-projects: read + security-events: read + statuses: read +jobs: + TagBot: + if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot' + runs-on: ubuntu-latest + steps: + - uses: JuliaRegistries/TagBot@v1 + with: + ssh: ${{ secrets.DOCUMENTER_KEY }} + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/Project.toml b/Project.toml index c9e89f9..a9fa39c 100644 --- a/Project.toml +++ b/Project.toml @@ -1,3 +1,17 @@ name = "ObjectStore" uuid = "1b5eed3d-1f46-4baa-87f3-a4a892b23610" version = "0.1.0" + +[compat] +CloudBase = "1" +ReTestItems = "1" +Test = "1" +julia = "1.8" + +[extras] +CloudBase = "85eb1798-d7c4-4918-bb13-c944d38e27ed" +ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[targets] +test = ["CloudBase", "ReTestItems", "Test"] diff --git a/src/ObjectStore.jl b/src/ObjectStore.jl index 6f55294..3e2fe61 100644 --- a/src/ObjectStore.jl +++ b/src/ObjectStore.jl @@ -1,5 +1,7 @@ module ObjectStore +export init_rust_store, blob_get!, blob_put, AzureCredentials + const rust_lib_dir = @static if Sys.islinux() joinpath( @__DIR__, @@ -35,14 +37,16 @@ end const rust_lib = joinpath(rust_lib_dir, "librust_store.$extension") -RUST_STORE_STARTED = false +const RUST_STORE_STARTED = Ref(false) +const _INIT_LOCK::ReentrantLock = ReentrantLock() function init_rust_store() - global RUST_STORE_STARTED - if RUST_STORE_STARTED - return + Base.@lock _INIT_LOCK begin + if RUST_STORE_STARTED[] + return + end + @ccall rust_lib.start()::Cint + RUST_STORE_STARTED[] = true end - @ccall rust_lib.start()::Cint - RUST_STORE_STARTED = true end struct AzureCredentials @@ -51,23 +55,22 @@ struct AzureCredentials key::String host::String end +const _AzureCredentialsFFI = NTuple{4,Cstring} -struct AzureCredentialsFFI - account::Cstring - container::Cstring - key::Cstring - host::Cstring -end - -function to_ffi(credentials::AzureCredentials) - AzureCredentialsFFI( +function Base.cconvert(::Type{Ref{AzureCredentials}}, credentials::AzureCredentials) + credentials_ffi = ( Base.unsafe_convert(Cstring, Base.cconvert(Cstring, credentials.account)), Base.unsafe_convert(Cstring, Base.cconvert(Cstring, credentials.container)), Base.unsafe_convert(Cstring, Base.cconvert(Cstring, credentials.key)), Base.unsafe_convert(Cstring, Base.cconvert(Cstring, credentials.host)) - ) + )::_AzureCredentialsFFI + # cconvert ensures its outputs are preserved during a ccall, so we can crate a pointer + # safely in the unsafe_convert call. + return credentials_ffi, Ref(credentials_ffi) +end +function Base.unsafe_convert(::Type{Ref{AzureCredentials}}, x::Tuple{T,Ref{T}}) where {T<:_AzureCredentialsFFI} + return Base.unsafe_convert(Ptr{_AzureCredentialsFFI}, x[2]) end - struct Response result::Cint @@ -79,7 +82,6 @@ end function blob_get!(path::String, buffer::AbstractVector{UInt8}, credentials::AzureCredentials) response = Ref(Response()) - credentials_ffi = Ref(to_ffi(credentials)) size = length(buffer) cond = Base.AsyncCondition() cond_handle = cond.handle @@ -88,7 +90,7 @@ function blob_get!(path::String, buffer::AbstractVector{UInt8}, credentials::Azu path::Cstring, buffer::Ref{Cuchar}, size::Culonglong, - credentials_ffi::Ref{AzureCredentialsFFI}, + credentials::Ref{AzureCredentials}, response::Ref{Response}, cond_handle::Ptr{Cvoid} )::Cint @@ -115,9 +117,8 @@ function blob_get!(path::String, buffer::AbstractVector{UInt8}, credentials::Azu end end -function blob_put!(path::String, buffer::AbstractVector{UInt8}, credentials::AzureCredentials) +function blob_put(path::String, buffer::AbstractVector{UInt8}, credentials::AzureCredentials) response = Ref(Response()) - credentials_ffi = Ref(to_ffi(credentials)) size = length(buffer) cond = Base.AsyncCondition() cond_handle = cond.handle @@ -126,7 +127,7 @@ function blob_put!(path::String, buffer::AbstractVector{UInt8}, credentials::Azu path::Cstring, buffer::Ref{Cuchar}, size::Culonglong, - credentials_ffi::Ref{AzureCredentialsFFI}, + credentials::Ref{AzureCredentials}, response::Ref{Response}, cond_handle::Ptr{Cvoid} )::Cint diff --git a/test/azure_blobs_tests.jl b/test/azure_blobs_tests.jl new file mode 100644 index 0000000..d166b75 --- /dev/null +++ b/test/azure_blobs_tests.jl @@ -0,0 +1,80 @@ +@testitem "Basic BlobStorage tests" begin + +using CloudBase.CloudTest: Azurite +using ObjectStore: blob_get!, blob_put, AzureCredentials +import ObjectStore + +ObjectStore.init_rust_store() + +# For interactive testing, use Azurite.run() instead of Azurite.with() +# conf, p = Azurite.run(; debug=true); atexit(() -> kill(p)) +Azurite.with(; debug=true, public=false) do conf + _credentials, _container = conf + base_url = _container.baseurl + credentials = AzureCredentials(_credentials.auth.account, _container.name, _credentials.auth.key, base_url) + + @testset "0B file" begin + buffer = Vector{UInt8}(undef, 1000) + + nbytes_written = blob_put(joinpath(base_url, "empty.csv"), codeunits(""), credentials) + @test nbytes_written == 0 + + nbytes_read = blob_get!(joinpath(base_url, "empty.csv"), buffer, credentials) + @test nbytes_read == 0 + end + + @testset "100B file" begin + input = "1,2,3,4,5,6,7,8,9,1\n" ^ 5 + buffer = Vector{UInt8}(undef, 1000) + @assert sizeof(input) == 100 + @assert sizeof(buffer) > sizeof(input) + + nbytes_written = blob_put(joinpath(base_url, "test100B.csv"), codeunits(input), credentials) + @test nbytes_written == 100 + + nbytes_read = blob_get!(joinpath(base_url, "test100B.csv"), buffer, credentials) + @test nbytes_read == 100 + @test String(buffer[1:nbytes_read]) == input + end + + @testset "1MB file" begin + input = "1,2,3,4,5,6,7,8,9,1\n" ^ 50_000 + buffer = Vector{UInt8}(undef, 1_000_000) + @assert sizeof(input) == 1_000_000 == sizeof(buffer) + + nbytes_written = blob_put(joinpath(base_url, "test100B.csv"), codeunits(input), credentials) + @test nbytes_written == 1_000_000 + + nbytes_read = blob_get!(joinpath(base_url, "test100B.csv"), buffer, credentials) + @test nbytes_read == 1_000_000 + @test String(buffer[1:nbytes_read]) == input + end + + @testset "20MB file" begin + input = "1,2,3,4,5,6,7,8,9,1\n" ^ 1_000_000 + buffer = Vector{UInt8}(undef, 20_000_000) + @assert sizeof(input) == 20_000_000 == sizeof(buffer) + + nbytes_written = blob_put(joinpath(base_url, "test100B.csv"), codeunits(input), credentials) + @test nbytes_written == 20_000_000 + + nbytes_read = blob_get!(joinpath(base_url, "test100B.csv"), buffer, credentials) + @test nbytes_read == 20_000_000 + @test String(buffer[1:nbytes_read]) == input + end + + # If the buffer is too small, we hang + # @testset "100B file, buffer too small" begin + # input = "1,2,3,4,5,6,7,8,9,1\n" ^ 5 + # buffer = Vector{UInt8}(undef, 10) + # @assert sizeof(input) == 100 + # @assert sizeof(buffer) < sizeof(input) + + # nbytes_written = blob_put(joinpath(base_url, "test100B.csv"), codeunits(input), credentials) + # @test nbytes_written == 100 + + # nbytes_read = blob_get!(joinpath(base_url, "test100B.csv"), buffer, credentials) + # end +end # Azurite.with + +end # @testitem diff --git a/test/runtests.jl b/test/runtests.jl new file mode 100644 index 0000000..44b52fe --- /dev/null +++ b/test/runtests.jl @@ -0,0 +1,6 @@ +using ReTestItems +using ObjectStore + +withenv("RUST_BACKTRACE"=>1) do + runtests(ObjectStore, testitem_timeout=30, nworkers=1) +end \ No newline at end of file