Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduce BufferReader and eliminate a bunch of allocations #393

Merged
merged 31 commits into from
Feb 4, 2024
Merged
Show file tree
Hide file tree
Changes from 22 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
6392506
Eliminate allocation in PhotoshopReader
iamcarbon Jan 30, 2024
722774a
Use switch expressions
iamcarbon Jan 31, 2024
4df2738
Add BufferReader
iamcarbon Jan 31, 2024
224eade
Eliminate two allocations in FlirReader
iamcarbon Jan 31, 2024
461a6b4
Eliminate allocation in OlympusMakernoteDirectory
iamcarbon Jan 31, 2024
cbe2d09
Eliminate various allocations in BplistReader
iamcarbon Jan 31, 2024
a26f6f3
Eliminate allocation in JpegReader
iamcarbon Jan 31, 2024
d84ae43
Eliminate allocation in PngChromaticities
iamcarbon Jan 31, 2024
6f33bc8
Eliminate allocation in PngDescriptor
iamcarbon Jan 31, 2024
5d34c55
Eliminate allocation in PngHeader
iamcarbon Jan 31, 2024
189d66f
Eliminate a few allocations in PngMetadataReader
iamcarbon Jan 31, 2024
76c687e
Eliminate allocation in WavFactHandler
iamcarbon Jan 31, 2024
27c9846
Use Assert.Debug
iamcarbon Jan 31, 2024
5bb02b1
Ensure BufferReader operates on buffers <= 2GB
iamcarbon Jan 31, 2024
f53892b
Fix offset in BufferReader.GetBytes
iamcarbon Jan 31, 2024
a89e1f0
Rename BufferReader.Skip -> Advance
iamcarbon Jan 31, 2024
db65fa5
Fix assert order
iamcarbon Jan 31, 2024
f7ee272
Verify JpegSegment sizes before reading data
iamcarbon Feb 1, 2024
94eec47
Bring back exceptions on BufferReader
iamcarbon Feb 1, 2024
73143a7
Fix switch statement
iamcarbon Feb 1, 2024
945686f
Slice span to avoid extra copy in BufferReader
iamcarbon Feb 1, 2024
5205495
Validate buffer length when reading PNG chunks
iamcarbon Feb 1, 2024
e6d86bd
Ensure the BufferReader can't advance past the end of data
iamcarbon Feb 2, 2024
f4437f5
Improve code readability in BplistReader
iamcarbon Feb 2, 2024
59f4294
Fix length check in PngMetadataReader
iamcarbon Feb 2, 2024
509973c
Fix typo
drewnoakes Feb 4, 2024
a332d84
Revert change in 4CC strings
drewnoakes Feb 4, 2024
7173e0a
Make field readonly
drewnoakes Feb 4, 2024
b983a8f
Make property readonly
drewnoakes Feb 4, 2024
6c398fd
Change Advance to return a span
drewnoakes Feb 4, 2024
1b38f68
Merge remote-tracking branch 'origin/main' into cq4
drewnoakes Feb 4, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions MetadataExtractor/Formats/Apple/BplistReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public static PropertyListResults Parse(byte[] bplist)

Trailer trailer = ReadTrailer();

SequentialByteArrayReader reader = new(bplist, baseIndex: checked((int)(trailer.OffsetTableOffset + trailer.TopObject)));
var reader = new BufferReader(bplist.AsSpan(checked((int)(trailer.OffsetTableOffset + trailer.TopObject))), isBigEndian: true);
drewnoakes marked this conversation as resolved.
Show resolved Hide resolved

int[] offsets = new int[(int)trailer.NumObjects];

Expand All @@ -63,7 +63,7 @@ public static PropertyListResults Parse(byte[] bplist)

for (int i = 0; i < offsets.Length; i++)
{
reader = new SequentialByteArrayReader(bplist, offsets[i]);
reader = new BufferReader(bplist.AsSpan(offsets[i]), isBigEndian: true);

byte b = reader.GetByte();

Expand All @@ -73,13 +73,13 @@ public static PropertyListResults Parse(byte[] bplist)
object obj = objectFormat switch
{
// dict
0x0D => HandleDict(marker),
0x0D => HandleDict(ref reader, marker),
// string (ASCII)
0x05 => reader.GetString(bytesRequested: marker & 0x0F, Encoding.ASCII),
// data
0x04 => HandleData(marker),
0x04 => HandleData(ref reader, marker),
// int
0x01 => HandleInt(marker),
0x01 => HandleInt(ref reader, marker),
// unknown
_ => throw new NotSupportedException($"Unsupported object format {objectFormat:X2}.")
};
Expand All @@ -91,10 +91,10 @@ public static PropertyListResults Parse(byte[] bplist)

Trailer ReadTrailer()
{
SequentialByteArrayReader reader = new(bplist, bplist.Length - Trailer.SizeBytes);
var reader = new BufferReader(bplist.AsSpan(bplist.Length - Trailer.SizeBytes), isBigEndian: true);

// Skip 5-byte unused values, 1-byte sort version.
reader.Skip(6);
reader.Advance(6);

return new Trailer
{
Expand All @@ -106,7 +106,7 @@ Trailer ReadTrailer()
};
}

object HandleInt(byte marker)
static object HandleInt(ref BufferReader reader, byte marker)
{
return marker switch
{
Expand All @@ -118,7 +118,7 @@ object HandleInt(byte marker)
};
}

Dictionary<byte, byte> HandleDict(byte count)
static Dictionary<byte, byte> HandleDict(ref BufferReader reader, byte count)
{
var keyRefs = new byte[count];

Expand All @@ -137,7 +137,7 @@ Dictionary<byte, byte> HandleDict(byte count)
return map;
}

object HandleData(byte marker)
object HandleData(ref BufferReader reader, byte marker)
{
int byteCount = marker;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -160,30 +160,16 @@ public sealed class OlympusCameraSettingsMakernoteDescriptor(OlympusCameraSettin
return null;

var sb = new StringBuilder();
switch (values[0])
sb.Append(values[0] switch
{
case 0:
sb.Append("Single AF");
break;
case 1:
sb.Append("Sequential shooting AF");
break;
case 2:
sb.Append("Continuous AF");
break;
case 3:
sb.Append("Multi AF");
break;
case 4:
sb.Append("Face detect");
break;
case 10:
sb.Append("MF");
break;
default:
sb.Append("Unknown (" + values[0] + ")");
break;
}
0 => "Single AF",
1 => "Sequential shooting AF",
2 => "Continuous AF",
3 => "Multi AF",
4 => "Face detect",
10 => "MF",
_ => $"Unknown ({values[0]})"
});

if (values.Length > 1)
{
Expand Down Expand Up @@ -231,18 +217,12 @@ public sealed class OlympusCameraSettingsMakernoteDescriptor(OlympusCameraSettin

var sb = new StringBuilder();

switch (values[0])
sb.Append(values[0] switch
{
case 0:
sb.Append("AF not used");
break;
case 1:
sb.Append("AF used");
break;
default:
sb.Append("Unknown (" + values[0] + ")");
break;
}
0 => "AF not used",
1 => "AF used",
_ => $"Unknown ({values[0]})"
});

if (values.Length > 1)
sb.Append("; " + values[1]);
Expand Down Expand Up @@ -385,24 +365,14 @@ public sealed class OlympusCameraSettingsMakernoteDescriptor(OlympusCameraSettin

var sb = new StringBuilder();

switch (values[0])
sb.Append(values[0] switch
{
case 0:
sb.Append("Off");
break;
case 3:
sb.Append("TTL");
break;
case 4:
sb.Append("Auto");
break;
case 5:
sb.Append("Manual");
break;
default:
sb.Append("Unknown (" + values[0] + ")");
break;
}
0 => "Off",
3 => "TTL",
4 => "Auto",
5 => "Manual",
_ => $"Unknown ({values[0]})"
});

for (var i = 1; i < values.Length; i++)
sb.Append("; ").Append(values[i]);
Expand Down Expand Up @@ -692,33 +662,17 @@ public sealed class OlympusCameraSettingsMakernoteDescriptor(OlympusCameraSettin
return null;

var sb = new StringBuilder();
switch (values[0])
sb.Append(values[0] switch
{
case 1:
sb.Append("Vivid");
break;
case 2:
sb.Append("Natural");
break;
case 3:
sb.Append("Muted");
break;
case 4:
sb.Append("Portrait");
break;
case 5:
sb.Append("i-Enhance");
break;
case 256:
sb.Append("Monotone");
break;
case 512:
sb.Append("Sepia");
break;
default:
sb.Append("Unknown (").Append(values[0]).Append(')');
break;
}
1 => "Vivid",
2 => "Natural",
3 => "Muted",
4 => "Portrait",
5 => "i-Enhance",
256 => "Monotone",
512 => "Sepia",
_ => $"Unknown ({values[0]})"
});

if (values.Length > 1)
sb.Append("; ").Append(values[1]);
Expand Down Expand Up @@ -822,33 +776,18 @@ public sealed class OlympusCameraSettingsMakernoteDescriptor(OlympusCameraSettin
sb.Append("Partial Color " + values[i] + "; ");
else if (i == 4)
{
switch (values[i])
sb.Append(values[i] switch
{
case 0x0000:
sb.Append("No Effect");
break;
case 0x8010:
sb.Append("Star Light");
break;
case 0x8020:
sb.Append("Pin Hole");
break;
case 0x8030:
sb.Append("Frame");
break;
case 0x8040:
sb.Append("Soft Focus");
break;
case 0x8050:
sb.Append("White Edge");
break;
case 0x8060:
sb.Append("B&W");
break;
default:
sb.Append("Unknown (").Append(values[i]).Append(')');
break;
}
0x0000 => "No Effect",
0x8010 => "Star Light",
0x8020 => "Pin Hole",
0x8030 => "Frame",
0x8040 => "Soft Focus",
0x8050 => "White Edge",
0x8060 => "B&W",
_ => $"Unknown ({values[i]})"
});

sb.Append("; ");
}
else if (i == 6)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -448,9 +448,9 @@ public override void Set(int tagType, object value)
base.Set(tagType, value);
}

private void ProcessCameraSettings(byte[] bytes)
private void ProcessCameraSettings(ReadOnlySpan<byte> bytes)
{
var reader = new SequentialByteArrayReader(bytes);
var reader = new BufferReader(bytes, isBigEndian: true);
var count = bytes.Length / 4;

for (var i = 0; i < count; i++)
Expand Down
5 changes: 3 additions & 2 deletions MetadataExtractor/Formats/Flir/FlirReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -153,8 +153,9 @@ public IEnumerable<Directory> Extract(IndexedReader reader)
directory.Set(TagRawValueMedian, reader2.GetUInt16(TagRawValueMedian));
directory.Set(TagRawValueRange, reader2.GetUInt16(TagRawValueRange));

var dateTimeBytes = reader2.GetBytes(TagDateTimeOriginal, 10);
var dateTimeReader = new SequentialByteArrayReader(dateTimeBytes, isMotorolaByteOrder: false);
Span<byte> dateTimeBytes = stackalloc byte[10];
reader2.GetBytes(TagDateTimeOriginal, dateTimeBytes);
var dateTimeReader = new BufferReader(dateTimeBytes, isBigEndian: false);
var tm = dateTimeReader.GetUInt32();
var ss = dateTimeReader.GetUInt32() & 0xffff;
var tz = dateTimeReader.GetInt16();
Expand Down
60 changes: 35 additions & 25 deletions MetadataExtractor/Formats/Jpeg/JpegReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,35 +29,45 @@ public JpegDirectory Extract(JpegSegment segment)
// The value of TagCompressionType is determined by the segment type found
directory.Set(JpegDirectory.TagCompressionType, (int)segment.Type - (int)JpegSegmentType.Sof0);

SequentialReader reader = new SequentialByteArrayReader(segment.Bytes);
const int JpegHeaderSize = 1 + 2 + 2 + 1;

try
if (segment.Span.Length < JpegHeaderSize)
{
directory.Set(JpegDirectory.TagDataPrecision, reader.GetByte());
directory.Set(JpegDirectory.TagImageHeight, reader.GetUInt16());
directory.Set(JpegDirectory.TagImageWidth, reader.GetUInt16());

var componentCount = reader.GetByte();

directory.Set(JpegDirectory.TagNumberOfComponents, componentCount);

// For each component, there are three bytes of data:
// 1 - Component ID: 1 = Y, 2 = Cb, 3 = Cr, 4 = I, 5 = Q
// 2 - Sampling factors: bit 0-3 vertical, 4-7 horizontal
// 3 - Quantization table number

for (var i = 0; i < componentCount; i++)
{
var componentId = reader.GetByte();
var samplingFactorByte = reader.GetByte();
var quantizationTableNumber = reader.GetByte();
var component = new JpegComponent(componentId, samplingFactorByte, quantizationTableNumber);
directory.Set(JpegDirectory.TagComponentData1 + i, component);
}
directory.AddError("Insufficient bytes for JPEG segment header.");

return directory;
}
catch (IOException ex)

var reader = new BufferReader(segment.Span, isBigEndian: true);

directory.Set(JpegDirectory.TagDataPrecision, reader.GetByte());
directory.Set(JpegDirectory.TagImageHeight, reader.GetUInt16());
directory.Set(JpegDirectory.TagImageWidth, reader.GetUInt16());

var componentCount = reader.GetByte();

directory.Set(JpegDirectory.TagNumberOfComponents, componentCount);

const int JpegComponentSize = 1 + 1 + 1;

if (reader.Available < componentCount * JpegComponentSize)
{
directory.AddError("Insufficient bytes for JPEG the requested number of JPEG components.");
return directory;
}

// For each component, there are three bytes of data:
// 1 - Component ID: 1 = Y, 2 = Cb, 3 = Cr, 4 = I, 5 = Q
// 2 - Sampling factors: bit 0-3 vertical, 4-7 horizontal
// 3 - Quantization table number

for (var i = 0; i < componentCount; i++)
{
directory.AddError(ex.Message);
var componentId = reader.GetByte();
var samplingFactorByte = reader.GetByte();
var quantizationTableNumber = reader.GetByte();
var component = new JpegComponent(componentId, samplingFactorByte, quantizationTableNumber);
directory.Set(JpegDirectory.TagComponentData1 + i, component);
}

return directory;
Expand Down
7 changes: 5 additions & 2 deletions MetadataExtractor/Formats/Photoshop/PhotoshopReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -53,12 +53,15 @@ public IReadOnlyList<Directory> Extract(SequentialReader reader, int length)
// http://www.adobe.com/devnet-apps/photoshop/fileformatashtml/#50577409_pgfId-1037504
var pos = 0;
int clippingPathCount = 0;

Span<byte> signature = stackalloc byte[4];

while (pos < length)
{
try
{
// 4 bytes for the signature ("8BIM", "PHUT", etc.)
var signature = reader.GetString(4, Encoding.UTF8);
reader.GetBytes(signature);
pos += 4;

// 2 bytes for the resource identifier (tag type).
Expand Down Expand Up @@ -106,7 +109,7 @@ public IReadOnlyList<Directory> Extract(SequentialReader reader, int length)
}

// Skip any unsupported IRBs
if (signature != "8BIM")
if (!signature.SequenceEqual("8BIM"u8))
continue;

switch (tagType)
Expand Down
Loading
Loading