From b7dbc885445d5b3405036345f98555fe82a49c10 Mon Sep 17 00:00:00 2001 From: Christopher Burns Date: Tue, 2 Apr 2024 13:56:18 -0500 Subject: [PATCH] add support for heterogeneous files (#156) * add support for files with multiple images of different sizes or colortypes * fix is_homogeneous logic --- Project.toml | 2 +- src/TiffImages.jl | 1 + src/layout.jl | 19 ++++++++++++----- src/load.jl | 49 +++++++++++++++++++++++++++++++++++--------- src/types/strided.jl | 16 ++++----------- test/runtests.jl | 17 +++++++++++++++ 6 files changed, 76 insertions(+), 28 deletions(-) diff --git a/Project.toml b/Project.toml index b07ce0c5..2fd28cf5 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "TiffImages" uuid = "731e570b-9d59-4bfa-96dc-6df516fadf69" authors = ["Tamas Nagy "] -version = "0.9.1" +version = "1.0.0" [deps] ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" diff --git a/src/TiffImages.jl b/src/TiffImages.jl index 1b6027af..d2f26d52 100644 --- a/src/TiffImages.jl +++ b/src/TiffImages.jl @@ -30,6 +30,7 @@ include("compression.jl") include("layout.jl") include(joinpath("types", "common.jl")) include(joinpath("types", "dense.jl")) +include(joinpath("types", "strided.jl")) include(joinpath("types", "lazy.jl")) include(joinpath("types", "mmapped.jl")) include("load.jl") diff --git a/src/layout.jl b/src/layout.jl index b2f1c79c..7ee43995 100644 --- a/src/layout.jl +++ b/src/layout.jl @@ -9,10 +9,21 @@ predictor(ifd::IFD) = Int(getdata(ifd, PREDICTOR, 0)) bitspersample(ifd::IFD) = Int(first(ifd[BITSPERSAMPLE].data))::Int ispalette(ifd::IFD) = Int(getdata(ifd, PHOTOMETRIC, 0)) == 3 compression(ifd::IFD) = getdata(CompressionType, ifd, COMPRESSION, COMPRESSION_NONE) +colortype(ifd::IFD) = first(interpretation(ifd)){_mappedtype(rawtype(ifd), bitspersample(ifd))} is_irregular_bps(ifd::IFD) = bitspersample(ifd) != sizeof(rawtype(ifd)) * 8 is_complicated(ifd::IFD) = !iscontiguous(ifd) || compression(ifd) != COMPRESSION_NONE || is_irregular_bps(ifd) == true || predictor(ifd) > 1 +# returns true if all slices have the same size and color type +function is_homogeneous(ifds::Vector{<:IFD}) + return all(map(==(nrows(first(ifds))), nrows.(ifds))) && + all(map(==(ncols(first(ifds))), ncols.(ifds))) && + all(map(==(colortype(first(ifds))), colortype.(ifds))) && + # tiled images are padded during encoding, so 'nrows' and 'ncols' + # may not indicate the true space requirements for decoding + all(map(==(istiled(first(ifds))), istiled.(ifds))) +end + """ interpretation(ifd) @@ -123,15 +134,13 @@ end Allocate a cache for this IFD with correct type and size. """ function getcache(ifd::IFD) - T = rawtype(ifd) - colortype, extras = interpretation(ifd) - bps = bitspersample(ifd) + ctype = colortype(ifd) if istiled(ifd) tile_width = tilecols(ifd) tile_height = tilerows(ifd) - return Array{colortype{_mappedtype(T, bps)}}(undef, cld(ncols(ifd), tile_width) * tile_width, cld(nrows(ifd), tile_height) * tile_height) + return Array{ctype}(undef, cld(ncols(ifd), tile_width) * tile_width, cld(nrows(ifd), tile_height) * tile_height) else - return Array{colortype{_mappedtype(T, bps)}}(undef, ncols(ifd), nrows(ifd)) + return Array{ctype}(undef, ncols(ifd), nrows(ifd)) end end diff --git a/src/load.jl b/src/load.jl index 71da1a9b..31b0f477 100644 --- a/src/load.jl +++ b/src/load.jl @@ -30,29 +30,44 @@ function load(tf::TiffFile; verbose=true, mmap = false, lazyio = false) nplanes += 1 end + homogeneous = is_homogeneous(ifds) + ifd = first(ifds) if mmap && iscontiguous(ifd) && getdata(CompressionType, ifd, COMPRESSION, COMPRESSION_NONE) === COMPRESSION_NONE return MmappedTIFF(tf, ifds) elseif lazyio || mmap + homogeneous || error("lazy IO is only supported for homogeneous files") mmap && @warn "Compression and discontiguous planes are not supported by `mmap`, use `lazyio = true` instead" loaded = LazyBufferedTIFF(tf, ifds) else if nplanes == 1 loaded = load(tf, ifds, nothing; verbose=verbose) else - loaded = load(tf, ifds, nplanes; verbose=verbose) + loaded = load(tf, ifds, nplanes, Val(homogeneous); verbose=verbose) end end if (tf.need_bswap && !is_irregular_bps(ifd)) || predictor(ifd) == 3 @debug "bswap'ing data" - loaded .= bswap.(loaded) + if !homogeneous + for sub in loaded + sub .= bswap.(sub) + end + else + loaded .= bswap.(loaded) + end end - data = fixcolors(loaded, first(ifds)) - close(tf.io) - return DenseTaggedImage(data, ifds) + + if nplanes > 1 && !homogeneous + # multiple images of different sizes or color types + data = fixcolors.(loaded, ifds) + return StridedTaggedImage(data, ifds) + else + data = fixcolors(loaded, first(ifds)) + return DenseTaggedImage(data, ifds) + end end """ @@ -75,14 +90,14 @@ function fixcolors(loaded, ifd) end function load(tf::TiffFile, ifds::AbstractVector{<:IFD}, ::Nothing; verbose = true) - ifd = ifds[1] + ifd = first(ifds) cache = getcache(ifd) read!(cache, tf, ifd) - istiled(ifd) ? Matrix(tile(cache, ifd)) : Matrix(cache') + Matrix(transform(cache, ifd)) end -function load(tf::TiffFile, ifds::AbstractVector{<:IFD}, N; verbose = true) - ifd = ifds[1] +function load(tf::TiffFile, ifds::AbstractVector{<:IFD}, N, homogeneous::Val{true}; verbose = true) + ifd = first(ifds) cache = getcache(ifd) @@ -90,12 +105,26 @@ function load(tf::TiffFile, ifds::AbstractVector{<:IFD}, N; verbose = true) @showprogress desc="Loading:" enabled=verbose for (idx, ifd) in enumerate(ifds) read!(cache, tf, ifd) - data[:, :, idx] .= (istiled(ifd) ? tile(cache, ifd) : cache') + data[:, :, idx] .= transform(cache, ifd) + end + + return data + end + +function load(tf::TiffFile, ifds::AbstractVector{<:IFD}, N, homogeneous::Val{false}; verbose = true) + data = Vector{AbstractMatrix}() + + @showprogress desc="Loading:" enabled=verbose for (idx, ifd) in enumerate(ifds) + cache = getcache(ifd) + read!(cache, tf, ifd) + push!(data, transform(cache, ifd)) end return data end +transform(cache, ifd) = istiled(ifd) ? tile(cache, ifd) : cache' + function tile(cache, ifd) rows = nrows(ifd) cols = ncols(ifd) diff --git a/src/types/strided.jl b/src/types/strided.jl index a6317bc7..3564271f 100644 --- a/src/types/strided.jl +++ b/src/types/strided.jl @@ -1,15 +1,7 @@ -struct StridedTaggedImage{T, O} <: AbstractTIFF{T, 1} where {O <: Unsigned} - data::Vector{AbstractArray{T, 2}} - +struct StridedTaggedImage{O <: Unsigned, AA <: AbstractMatrix} <: AbstractTIFF{Any, 1} + data::Vector{AA} ifds::Vector{IFD{O}} end -function StridedTaggedImage(data::AbstractArray{T, 2}, ifd::IFD{O}) where {T, O} - StridedTaggedImage([data], IFD{O}[ifd]) -end - -Base.size(t::StridedTaggedImage) = length(t.data) -Base.getindex(img::StridedTaggedImage, i) = getindex(img.data, i) - - - +Base.size(t::StridedTaggedImage) = size(t.data) +Base.getindex(img::StridedTaggedImage, i) = getindex(img.data, i) \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index ad6ab8e9..bdb7299b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -320,3 +320,20 @@ end im = get_example("earthlab.tif") @test size(TiffImages.load(im)) == (2400, 2400) # no error end + +@testset "Ragged" begin + original = TiffImages.load(get_example("shapes_uncompressed.tif")) + half = TiffImages.load(get_example("shapes_uncompressed_half.tif")) + palette = TiffImages.load(get_example("shapes_lzw_palette.tif")) + multisize = TiffImages.load(get_example("shapes_multi_size.tif")) + multicolor = TiffImages.load(get_example("shapes_multi_color.tif")) + + @test original == multisize[1] + @test half == multisize[2] + + @test original == multicolor[1] + @test palette == multicolor[2] + @test original == multicolor[3] + @test sum(convert.(Float64, reinterpret(N0f8, original)) .- convert.(Float64, reinterpret(N4f12, multicolor[4]))) ./ (216*128) < 0.0001 + @test original == multicolor[5] +end \ No newline at end of file