Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

removing roaring64.FromUnsafeBytes #446

Closed
wants to merge 3 commits into from
Closed

removing roaring64.FromUnsafeBytes #446

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
bbbdb70
removing roaring64.FromUnsafeBytes
lemire Sep 17, 2024
9aba17d
trim
lemire Sep 17, 2024
22f8612
trim
lemire Sep 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 0 additions & 48 deletions roaring64/roaring64.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"strconv"

"github.com/RoaringBitmap/roaring/v2"
"github.com/RoaringBitmap/roaring/v2/internal"
)

const (
Expand Down Expand Up @@ -81,53 +80,6 @@ func (rb *Bitmap) WriteTo(stream io.Writer) (int64, error) {
return n, nil
}

// FromUnsafeBytes reads a serialized version of this bitmap from the byte buffer without copy.
// It is the caller's responsibility to ensure that the input data is not modified and remains valid for the entire lifetime of this bitmap.
// This method avoids small allocations but holds references to the input data buffer. It is GC-friendly, but it may consume more memory eventually.
func (rb *Bitmap) FromUnsafeBytes(data []byte) (p int64, err error) {
stream := internal.NewByteBuffer(data)
sizeBuf := make([]byte, 8)
n, err := stream.Read(sizeBuf)
if err != nil {
return 0, err
}
p += int64(n)
size := binary.LittleEndian.Uint64(sizeBuf)

rb.highlowcontainer.resize(0)
if cap(rb.highlowcontainer.keys) >= int(size) {
rb.highlowcontainer.keys = rb.highlowcontainer.keys[:size]
} else {
rb.highlowcontainer.keys = make([]uint32, size)
}
if cap(rb.highlowcontainer.containers) >= int(size) {
rb.highlowcontainer.containers = rb.highlowcontainer.containers[:size]
} else {
rb.highlowcontainer.containers = make([]*roaring.Bitmap, size)
}
if cap(rb.highlowcontainer.needCopyOnWrite) >= int(size) {
rb.highlowcontainer.needCopyOnWrite = rb.highlowcontainer.needCopyOnWrite[:size]
} else {
rb.highlowcontainer.needCopyOnWrite = make([]bool, size)
}
for i := uint64(0); i < size; i++ {
keyBuf, err := stream.Next(4)
if err != nil {
return 0, fmt.Errorf("error in bitmap.UnsafeFromBytes: could not read key #%d: %w", i, err)
}
p += 4
rb.highlowcontainer.keys[i] = binary.LittleEndian.Uint32(keyBuf)
rb.highlowcontainer.containers[i] = roaring.NewBitmap()
n, err := rb.highlowcontainer.containers[i].ReadFrom(stream)
if n == 0 || err != nil {
return int64(n), fmt.Errorf("Could not deserialize bitmap for key #%d: %s", i, err)
}
p += int64(n)
}

return p, nil
}

// ReadFrom reads a serialized version of this bitmap from stream.
// The format is compatible with other 64-bit RoaringBitmap
// implementations (Java, Go, C++) and it has a specification :
Expand Down
37 changes: 0 additions & 37 deletions roaring64/serialization_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,18 +25,12 @@ func TestSerializationOfEmptyBitmap(t *testing.T) {

require.NoError(t, err)
assert.EqualValues(t, buf.Len(), rb.GetSerializedSizeInBytes())
data := buf.Bytes()

newrb := NewBitmap()
_, err = newrb.ReadFrom(buf)

require.NoError(t, err)
assert.True(t, rb.Equals(newrb))

newrb2 := NewBitmap()
_, err = newrb2.FromUnsafeBytes(data)
require.NoError(t, err)
assert.True(t, rb.Equals(newrb2))
}

func TestBase64_036(t *testing.T) {
Expand All @@ -61,18 +55,12 @@ func TestSerializationBasic037(t *testing.T) {

require.NoError(t, err)
assert.EqualValues(t, buf.Len(), rb.GetSerializedSizeInBytes())
data := buf.Bytes()

newrb := NewBitmap()
_, err = newrb.ReadFrom(buf)

require.NoError(t, err)
assert.True(t, rb.Equals(newrb))

newrb2 := NewBitmap()
_, err = newrb2.FromUnsafeBytes(data)
require.NoError(t, err)
assert.True(t, rb.Equals(newrb2))
}

func TestSerializationToFile038(t *testing.T) {
Expand Down Expand Up @@ -111,11 +99,6 @@ func TestSerializationToFile038(t *testing.T) {

_, _ = newrb.ReadFrom(teer)
assert.True(t, rb.Equals(newrb))

newrb2 := NewBitmap()
_, err = newrb2.FromUnsafeBytes(buf.Bytes())
require.NoError(t, err)
assert.True(t, rb.Equals(newrb2))
}

func TestSerializationBasic2_041(t *testing.T) {
Expand All @@ -127,18 +110,12 @@ func TestSerializationBasic2_041(t *testing.T) {

require.NoError(t, err)
assert.Equal(t, l, buf.Len())
data := buf.Bytes()

newrb := NewBitmap()
_, err = newrb.ReadFrom(buf)

require.NoError(t, err)
assert.True(t, rb.Equals(newrb))

newrb2 := NewBitmap()
_, err = newrb2.FromUnsafeBytes(data)
require.NoError(t, err)
assert.True(t, rb.Equals(newrb2))
}

// roaringarray.writeTo and .readFrom should serialize and unserialize when containing all 3 container types
Expand All @@ -153,18 +130,12 @@ func TestSerializationBasic3_042(t *testing.T) {

require.NoError(t, err)
assert.EqualValues(t, buf.Len(), int(rb.GetSerializedSizeInBytes()))
data := buf.Bytes()

newrb := NewBitmap()
_, err = newrb.ReadFrom(&buf)

require.NoError(t, err)
assert.True(t, newrb.Equals(rb))

newrb2 := NewBitmap()
_, err = newrb2.FromUnsafeBytes(data)
require.NoError(t, err)
assert.True(t, rb.Equals(newrb2))
}

func TestHoldReference(t *testing.T) {
Expand Down Expand Up @@ -207,14 +178,6 @@ func TestHoldReference(t *testing.T) {
})
}

func BenchmarkUnserializeFromUnsafeBytes(b *testing.B) {
benchmarkUnserializeFunc(b, "FromUnsafeBytes", func(bitmap *Bitmap, data []byte) (int64, error) {
copied := make([]byte, len(data))
copy(copied, data)
return bitmap.FromUnsafeBytes(copied)
})
}

func BenchmarkUnserializeReadFrom(b *testing.B) {
benchmarkUnserializeFunc(b, "ReadFrom", func(bitmap *Bitmap, data []byte) (int64, error) {
return bitmap.ReadFrom(bytes.NewReader(data))
Expand Down
Loading