add support for heterogeneous files (#156)

* add support for files with multiple images of different sizes or colortypes

* fix is_homogeneous logic

Project.toml

@@ -1,7 +1,7 @@
 name = "TiffImages"
 uuid = "731e570b-9d59-4bfa-96dc-6df516fadf69"
 authors = ["Tamas Nagy <[email protected]>"]
-version = "0.9.1"
+version = "1.0.0"
 
 [deps]
 ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"

src/TiffImages.jl

@@ -30,6 +30,7 @@ include("compression.jl")
 include("layout.jl")
 include(joinpath("types", "common.jl"))
 include(joinpath("types", "dense.jl"))
+include(joinpath("types", "strided.jl"))
 include(joinpath("types", "lazy.jl"))
 include(joinpath("types", "mmapped.jl"))
 include("load.jl")

src/layout.jl

@@ -9,10 +9,21 @@ predictor(ifd::IFD) = Int(getdata(ifd, PREDICTOR, 0))
 bitspersample(ifd::IFD) = Int(first(ifd[BITSPERSAMPLE].data))::Int
 ispalette(ifd::IFD) = Int(getdata(ifd, PHOTOMETRIC, 0)) == 3
 compression(ifd::IFD) = getdata(CompressionType, ifd, COMPRESSION, COMPRESSION_NONE)
+colortype(ifd::IFD) =  first(interpretation(ifd)){_mappedtype(rawtype(ifd), bitspersample(ifd))}
 
 is_irregular_bps(ifd::IFD) = bitspersample(ifd) != sizeof(rawtype(ifd)) * 8
 is_complicated(ifd::IFD) = !iscontiguous(ifd) || compression(ifd) != COMPRESSION_NONE || is_irregular_bps(ifd) == true || predictor(ifd) > 1
 
+# returns true if all slices have the same size and color type
+function is_homogeneous(ifds::Vector{<:IFD})
+    return all(map(==(nrows(first(ifds))), nrows.(ifds))) &&
+        all(map(==(ncols(first(ifds))), ncols.(ifds))) &&
+        all(map(==(colortype(first(ifds))), colortype.(ifds))) &&
+        # tiled images are padded during encoding, so 'nrows' and 'ncols'
+        # may not indicate the true space requirements for decoding
+        all(map(==(istiled(first(ifds))), istiled.(ifds)))
+end
+
 """
     interpretation(ifd)
 
@@ -123,15 +134,13 @@ end
 Allocate a cache for this IFD with correct type and size.
 """
 function getcache(ifd::IFD)
-    T = rawtype(ifd)
-    colortype, extras = interpretation(ifd)
-    bps = bitspersample(ifd)
+    ctype = colortype(ifd)
     if istiled(ifd)
         tile_width = tilecols(ifd)
         tile_height = tilerows(ifd)
-        return Array{colortype{_mappedtype(T, bps)}}(undef, cld(ncols(ifd), tile_width) * tile_width, cld(nrows(ifd), tile_height) * tile_height)
+        return Array{ctype}(undef, cld(ncols(ifd), tile_width) * tile_width, cld(nrows(ifd), tile_height) * tile_height)
     else
-        return Array{colortype{_mappedtype(T, bps)}}(undef, ncols(ifd), nrows(ifd))
+        return Array{ctype}(undef, ncols(ifd), nrows(ifd))
     end
 end
 

src/load.jl

@@ -30,29 +30,44 @@ function load(tf::TiffFile; verbose=true, mmap = false, lazyio = false)
         nplanes += 1
     end
 
+    homogeneous = is_homogeneous(ifds)
+
     ifd = first(ifds)
     if mmap && iscontiguous(ifd) && getdata(CompressionType, ifd, COMPRESSION, COMPRESSION_NONE) === COMPRESSION_NONE
         return MmappedTIFF(tf, ifds)
     elseif lazyio || mmap
+        homogeneous || error("lazy IO is only supported for homogeneous files")
         mmap && @warn "Compression and discontiguous planes are not supported by `mmap`, use `lazyio = true` instead"
         loaded = LazyBufferedTIFF(tf, ifds)
     else
         if nplanes == 1
             loaded = load(tf, ifds, nothing; verbose=verbose)
         else
-            loaded = load(tf, ifds, nplanes; verbose=verbose)
+            loaded = load(tf, ifds, nplanes, Val(homogeneous); verbose=verbose)
         end
     end
 
     if (tf.need_bswap && !is_irregular_bps(ifd)) || predictor(ifd) == 3
         @debug "bswap'ing data"
-        loaded .= bswap.(loaded)
+        if !homogeneous
+            for sub in loaded
+                sub .= bswap.(sub)
+            end
+        else
+            loaded .= bswap.(loaded)
+        end
     end
 
-    data = fixcolors(loaded, first(ifds))
-
     close(tf.io)
-    return DenseTaggedImage(data, ifds)
+
+    if nplanes > 1 && !homogeneous
+        # multiple images of different sizes or color types
+        data = fixcolors.(loaded, ifds)
+        return StridedTaggedImage(data, ifds)
+    else
+        data = fixcolors(loaded, first(ifds))
+        return DenseTaggedImage(data, ifds)
+    end
 end
 
 """
@@ -75,27 +90,41 @@ function fixcolors(loaded, ifd)
 end
 
 function load(tf::TiffFile, ifds::AbstractVector{<:IFD}, ::Nothing; verbose = true)
-    ifd = ifds[1]
+    ifd = first(ifds)
     cache = getcache(ifd)
     read!(cache, tf, ifd)
-    istiled(ifd) ? Matrix(tile(cache, ifd)) : Matrix(cache')
+    Matrix(transform(cache, ifd))
 end
 
-function load(tf::TiffFile, ifds::AbstractVector{<:IFD}, N; verbose = true)
-    ifd = ifds[1]
+function load(tf::TiffFile, ifds::AbstractVector{<:IFD}, N, homogeneous::Val{true}; verbose = true)
+    ifd = first(ifds)
 
     cache = getcache(ifd)
 
     data = similar(cache, nrows(ifd), ncols(ifd), N)
 
     @showprogress desc="Loading:" enabled=verbose for (idx, ifd) in enumerate(ifds)
         read!(cache, tf, ifd)
-        data[:, :, idx] .= (istiled(ifd) ? tile(cache, ifd) : cache')
+        data[:, :, idx] .= transform(cache, ifd)
+    end
+
+    return data
+ end
+
+function load(tf::TiffFile, ifds::AbstractVector{<:IFD}, N, homogeneous::Val{false}; verbose = true)
+    data = Vector{AbstractMatrix}()
+
+    @showprogress desc="Loading:" enabled=verbose for (idx, ifd) in enumerate(ifds)
+        cache = getcache(ifd)
+        read!(cache, tf, ifd)
+        push!(data, transform(cache, ifd))
     end
 
     return data
 end
 
+transform(cache, ifd) = istiled(ifd) ? tile(cache, ifd) : cache'
+
 function tile(cache, ifd)
     rows = nrows(ifd)
     cols = ncols(ifd)

src/types/strided.jl

@@ -1,15 +1,7 @@
-struct StridedTaggedImage{T, O} <: AbstractTIFF{T, 1} where {O <: Unsigned}
-    data::Vector{AbstractArray{T, 2}}
-
+struct StridedTaggedImage{O <: Unsigned, AA <: AbstractMatrix} <: AbstractTIFF{Any, 1}
+    data::Vector{AA}
     ifds::Vector{IFD{O}}
 end
 
-function StridedTaggedImage(data::AbstractArray{T, 2}, ifd::IFD{O}) where {T, O}
-    StridedTaggedImage([data], IFD{O}[ifd])
-end
-
-Base.size(t::StridedTaggedImage) = length(t.data)
-Base.getindex(img::StridedTaggedImage, i) = getindex(img.data, i)
-
-
-
+Base.size(t::StridedTaggedImage) = size(t.data)
+Base.getindex(img::StridedTaggedImage, i) = getindex(img.data, i)

test/runtests.jl

@@ -320,3 +320,20 @@ end
     im = get_example("earthlab.tif")
     @test size(TiffImages.load(im)) == (2400, 2400) # no error
 end
+
+@testset "Ragged" begin
+    original = TiffImages.load(get_example("shapes_uncompressed.tif"))
+    half = TiffImages.load(get_example("shapes_uncompressed_half.tif"))
+    palette = TiffImages.load(get_example("shapes_lzw_palette.tif"))
+    multisize = TiffImages.load(get_example("shapes_multi_size.tif"))
+    multicolor = TiffImages.load(get_example("shapes_multi_color.tif"))
+
+    @test original == multisize[1]
+    @test half == multisize[2]
+
+    @test original == multicolor[1]
+    @test palette == multicolor[2]
+    @test original == multicolor[3]
+    @test sum(convert.(Float64, reinterpret(N0f8, original)) .- convert.(Float64, reinterpret(N4f12, multicolor[4]))) ./ (216*128) < 0.0001
+    @test original == multicolor[5]
+end