Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Introduce BufferReader and eliminate a bunch of allocations #393

Merged
merged 31 commits into from
Feb 4, 2024
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
31 commits
Select commit Hold shift + click to select a range
6392506
Eliminate allocation in PhotoshopReader
iamcarbon Jan 30, 2024
722774a
Use switch expressions
iamcarbon Jan 31, 2024
4df2738
Add BufferReader
iamcarbon Jan 31, 2024
224eade
Eliminate two allocations in FlirReader
iamcarbon Jan 31, 2024
461a6b4
Eliminate allocation in OlympusMakernoteDirectory
iamcarbon Jan 31, 2024
cbe2d09
Eliminate various allocations in BplistReader
iamcarbon Jan 31, 2024
a26f6f3
Eliminate allocation in JpegReader
iamcarbon Jan 31, 2024
d84ae43
Eliminate allocation in PngChromaticities
iamcarbon Jan 31, 2024
6f33bc8
Eliminate allocation in PngDescriptor
iamcarbon Jan 31, 2024
5d34c55
Eliminate allocation in PngHeader
iamcarbon Jan 31, 2024
189d66f
Eliminate a few allocations in PngMetadataReader
iamcarbon Jan 31, 2024
76c687e
Eliminate allocation in WavFactHandler
iamcarbon Jan 31, 2024
27c9846
Use Assert.Debug
iamcarbon Jan 31, 2024
5bb02b1
Ensure BufferReader operates on buffers <= 2GB
iamcarbon Jan 31, 2024
f53892b
Fix offset in BufferReader.GetBytes
iamcarbon Jan 31, 2024
a89e1f0
Rename BufferReader.Skip -> Advance
iamcarbon Jan 31, 2024
db65fa5
Fix assert order
iamcarbon Jan 31, 2024
f7ee272
Verify JpegSegment sizes before reading data
iamcarbon Feb 1, 2024
94eec47
Bring back exceptions on BufferReader
iamcarbon Feb 1, 2024
73143a7
Fix switch statement
iamcarbon Feb 1, 2024
945686f
Slice span to avoid extra copy in BufferReader
iamcarbon Feb 1, 2024
5205495
Validate buffer length when reading PNG chunks
iamcarbon Feb 1, 2024
e6d86bd
Ensure the BufferReader can't advance past the end of data
iamcarbon Feb 2, 2024
f4437f5
Improve code readability in BplistReader
iamcarbon Feb 2, 2024
59f4294
Fix length check in PngMetadataReader
iamcarbon Feb 2, 2024
509973c
Fix typo
drewnoakes Feb 4, 2024
a332d84
Revert change in 4CC strings
drewnoakes Feb 4, 2024
7173e0a
Make field readonly
drewnoakes Feb 4, 2024
b983a8f
Make property readonly
drewnoakes Feb 4, 2024
6c398fd
Change Advance to return a span
drewnoakes Feb 4, 2024
1b38f68
Merge remote-tracking branch 'origin/main' into cq4
drewnoakes Feb 4, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -367,10 +367,10 @@ public sealed class OlympusCameraSettingsMakernoteDescriptor(OlympusCameraSettin

sb.Append(values[0] switch
{
0 => sb.Append("Off"),
3 => sb.Append("TTL"),
4 => sb.Append("Auto"),
5 => sb.Append("Manual"),
0 => "Off",
3 => "TTL",
4 => "Auto",
5 => "Manual",
_ => $"Unknown ({values[0]})"
});

Expand Down
58 changes: 34 additions & 24 deletions MetadataExtractor/Formats/Jpeg/JpegReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,35 +29,45 @@ public JpegDirectory Extract(JpegSegment segment)
// The value of TagCompressionType is determined by the segment type found
directory.Set(JpegDirectory.TagCompressionType, (int)segment.Type - (int)JpegSegmentType.Sof0);

const int JpegHeaderSize = 1 + 2 + 2 + 1;

if (segment.Span.Length < JpegHeaderSize)
{
directory.AddError("Insufficient bytes for JPEG segment header.");

return directory;
}

var reader = new BufferReader(segment.Span, isBigEndian: true);

try
directory.Set(JpegDirectory.TagDataPrecision, reader.GetByte());
directory.Set(JpegDirectory.TagImageHeight, reader.GetUInt16());
directory.Set(JpegDirectory.TagImageWidth, reader.GetUInt16());

var componentCount = reader.GetByte();

directory.Set(JpegDirectory.TagNumberOfComponents, componentCount);

const int JpegComponentSize = 1 + 1 + 1;

if (reader.Available < componentCount * JpegComponentSize)
{
directory.Set(JpegDirectory.TagDataPrecision, reader.GetByte());
directory.Set(JpegDirectory.TagImageHeight, reader.GetUInt16());
directory.Set(JpegDirectory.TagImageWidth, reader.GetUInt16());

var componentCount = reader.GetByte();

directory.Set(JpegDirectory.TagNumberOfComponents, componentCount);

// For each component, there are three bytes of data:
// 1 - Component ID: 1 = Y, 2 = Cb, 3 = Cr, 4 = I, 5 = Q
// 2 - Sampling factors: bit 0-3 vertical, 4-7 horizontal
// 3 - Quantization table number

for (var i = 0; i < componentCount; i++)
{
var componentId = reader.GetByte();
var samplingFactorByte = reader.GetByte();
var quantizationTableNumber = reader.GetByte();
var component = new JpegComponent(componentId, samplingFactorByte, quantizationTableNumber);
directory.Set(JpegDirectory.TagComponentData1 + i, component);
}
directory.AddError("Insufficient bytes for JPEG the requested number of JPEG components.");
return directory;
}
catch (IOException ex)

// For each component, there are three bytes of data:
// 1 - Component ID: 1 = Y, 2 = Cb, 3 = Cr, 4 = I, 5 = Q
// 2 - Sampling factors: bit 0-3 vertical, 4-7 horizontal
// 3 - Quantization table number

for (var i = 0; i < componentCount; i++)
{
directory.AddError(ex.Message);
var componentId = reader.GetByte();
var samplingFactorByte = reader.GetByte();
var quantizationTableNumber = reader.GetByte();
var component = new JpegComponent(componentId, samplingFactorByte, quantizationTableNumber);
directory.Set(JpegDirectory.TagComponentData1 + i, component);
}

return directory;
Expand Down
44 changes: 31 additions & 13 deletions MetadataExtractor/Formats/Png/PngMetadataReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -295,32 +295,50 @@ private static IEnumerable<Directory> ProcessChunk(PngChunk chunk)
}
else if (chunkType == PngChunkType.tIME)
{
var reader = new BufferReader(bytes, isBigEndian: true);
var year = reader.GetUInt16();
var month = reader.GetByte();
int day = reader.GetByte();
int hour = reader.GetByte();
int minute = reader.GetByte();
int second = reader.GetByte();
var directory = new PngDirectory(PngChunkType.tIME);
if (DateUtil.IsValidDate(year, month, day) && DateUtil.IsValidTime(hour, minute, second))

if (bytes.Length < 2 + 1 + 1 + 1 + 1 + 1)
{
var time = new DateTime(year, month, day, hour, minute, second, DateTimeKind.Unspecified);
directory.Set(PngDirectory.TagLastModificationTime, time);
directory.AddError("Insufficient bytes for PNG tIME chunk.");
}
else
{
directory.AddError($"PNG tIME data describes an invalid date/time: year={year} month={month} day={day} hour={hour} minute={minute} second={second}");
var reader = new BufferReader(bytes, isBigEndian: true);

var year = reader.GetUInt16();
var month = reader.GetByte();
int day = reader.GetByte();
int hour = reader.GetByte();
int minute = reader.GetByte();
int second = reader.GetByte();

if (DateUtil.IsValidDate(year, month, day) && DateUtil.IsValidTime(hour, minute, second))
{
var time = new DateTime(year, month, day, hour, minute, second, DateTimeKind.Unspecified);
directory.Set(PngDirectory.TagLastModificationTime, time);
}
else
{
directory.AddError($"PNG tIME data describes an invalid date/time: year={year} month={month} day={day} hour={hour} minute={minute} second={second}");
}
yield return directory;
}
yield return directory;
}
else if (chunkType == PngChunkType.pHYs)
{
var directory = new PngDirectory(PngChunkType.pHYs);

if (bytes.Length < 4 + 4 + 1)
{
directory.AddError("Insufficient bytes for PNG pHYs chunk.");
}

var reader = new BufferReader(bytes, isBigEndian: true);

var pixelsPerUnitX = reader.GetInt32();
var pixelsPerUnitY = reader.GetInt32();
var unitSpecifier = reader.GetSByte();
var directory = new PngDirectory(PngChunkType.pHYs);

directory.Set(PngDirectory.TagPixelsPerUnitX, pixelsPerUnitX);
directory.Set(PngDirectory.TagPixelsPerUnitY, pixelsPerUnitY);
directory.Set(PngDirectory.TagUnitSpecifier, unitSpecifier);
Expand Down
40 changes: 20 additions & 20 deletions MetadataExtractor/IO/BufferReader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,15 +21,17 @@ public byte GetByte()

public void GetBytes(scoped Span<byte> bytes)
{
Debug.Assert(_position + bytes.Length <= _bytes.Length, "attempted to read past end of data");
if (_position + bytes.Length > _bytes.Length)
throw new IOException("End of data reached.");
drewnoakes marked this conversation as resolved.
Show resolved Hide resolved

_bytes.Slice(_position, bytes.Length).CopyTo(bytes);
_position += bytes.Length;
}

public byte[] GetBytes(int count)
{
Debug.Assert(_position + count <= _bytes.Length, "attempted to read past end of data");
if (_position + count > _bytes.Length)
throw new IOException("End of data reached.");

var bytes = new byte[count];

Expand All @@ -43,7 +45,9 @@ public byte[] GetBytes(int count)
public void Advance(int n)
{
Debug.Assert(n >= 0, "n must be zero or greater");
Debug.Assert(_position + n <= _bytes.Length, "attempted to advance past end of data");

if (_position + n > _bytes.Length)
throw new IOException("End of data reached.");

_position += n;
}
Expand All @@ -65,14 +69,13 @@ public bool TryAdvance(int n)

public sbyte GetSByte()
{
return unchecked((sbyte)GetByte());
return unchecked((sbyte)_bytes[_position++]);
}

public ushort GetUInt16()
{
Span<byte> bytes = stackalloc byte[2];

GetBytes(bytes);
var bytes = _bytes.Slice(_position, 2);
_position += 2;
drewnoakes marked this conversation as resolved.
Show resolved Hide resolved

return _isBigEndian
? BinaryPrimitives.ReadUInt16BigEndian(bytes)
Expand All @@ -81,9 +84,8 @@ public ushort GetUInt16()

public short GetInt16()
{
Span<byte> bytes = stackalloc byte[2];

GetBytes(bytes);
var bytes = _bytes.Slice(_position, 2);
_position += 2;

return _isBigEndian
? BinaryPrimitives.ReadInt16BigEndian(bytes)
Expand All @@ -92,9 +94,8 @@ public short GetInt16()

public uint GetUInt32()
{
Span<byte> bytes = stackalloc byte[4];

GetBytes(bytes);
var bytes = _bytes.Slice(_position, 4);
_position += 4;

return _isBigEndian
? BinaryPrimitives.ReadUInt32BigEndian(bytes)
Expand All @@ -103,9 +104,8 @@ public uint GetUInt32()

public int GetInt32()
{
Span<byte> bytes = stackalloc byte[4];

GetBytes(bytes);
var bytes = _bytes.Slice(_position, 4);
_position += 4;

return _isBigEndian
? BinaryPrimitives.ReadInt32BigEndian(bytes)
Expand All @@ -114,8 +114,8 @@ public int GetInt32()

public long GetInt64()
{
Span<byte> bytes = stackalloc byte[8];
GetBytes(bytes);
var bytes = _bytes.Slice(_position, 8);
_position += 8;

return _isBigEndian
? BinaryPrimitives.ReadInt64BigEndian(bytes)
Expand All @@ -124,8 +124,8 @@ public long GetInt64()

public ulong GetUInt64()
{
Span<byte> bytes = stackalloc byte[8];
GetBytes(bytes);
var bytes = _bytes.Slice(_position, 8);
_position += 8;

return _isBigEndian
? BinaryPrimitives.ReadUInt64BigEndian(bytes)
Expand Down
Loading