Merge pull request #43 from tlnagy/tn/refactor-layout-approach

Refactor layout approach

Project.toml

@@ -1,7 +1,7 @@
 name = "TiffImages"
 uuid = "731e570b-9d59-4bfa-96dc-6df516fadf69"
 authors = ["Tamas Nagy <[email protected]>"]
-version = "0.3.0"
+version = "0.3.1"
 
 [deps]
 ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"

src/files.jl

@@ -20,7 +20,7 @@ mutable struct TiffFile{O <: Unsigned, S <: Stream}
 end
 
 TiffFile{O}(s::Stream) where O <: Unsigned = TiffFile{O, typeof(s)}("", s, -1, false)
-TiffFile{O}(io::IO) where O <: Unsigned    = TiffFile{O}(Stream(format"TIFF", io))
+TiffFile{O}(io::IO) where O <: Unsigned    = TiffFile{O}(getstream(format"TIFF", io))
 TiffFile{O}() where O <: Unsigned          = TiffFile{O}(IOBuffer())
 
 function Base.read(io::Stream, ::Type{TiffFile})
@@ -33,7 +33,7 @@ function Base.read(io::Stream, ::Type{TiffFile})
     TiffFile{offset_size, typeof(io)}(filepath, io, first_offset, bs)
 end
 
-Base.read(io::IOStream, t::Type{TiffFile}) = read(Stream(format"TIFF", io, extract_filename(io)), t)
+Base.read(io::IOStream, t::Type{TiffFile}) = read(getstream(format"TIFF", io, extract_filename(io)), t)
 
 function Base.write(file::TiffFile{O}) where O
     seekstart(file.io)

src/ifds.jl

@@ -110,106 +110,46 @@ function Base.iterate(file::TiffFile, state::Tuple{Union{IFD{O}, Nothing}, Int})
     return (curr_ifd, (next_ifd, next_ifd_offset))
 end
 
-struct IFDLayout
-    nsamples::Int
-    nrows::Int
-    ncols::Int
-    nbytes::Int
-    readtype::DataType
-    rawtype::DataType
-    mappedtype::DataType
-    compression::CompressionType
-    interpretation::PhotometricInterpretations
-end
-
-function output(ifd::IFD)
-    nrows = Int(ifd[IMAGELENGTH].data)::Int
-    ncols = Int(ifd[IMAGEWIDTH].data)::Int
-
-    samplesperpixel = Int(ifd[SAMPLESPERPIXEL].data)::Int
-    sampleformats = fill(UInt16(0x01), samplesperpixel)
-    if SAMPLEFORMAT in ifd
-        sampleformats = ifd[SAMPLEFORMAT].data  # can a type be specified for this?
-    end
-
-    interpretation = Int(ifd[PHOTOMETRIC].data)::Int
-
-    strip_nbytes = ifd[STRIPBYTECOUNTS].data
-    nbytes = Int(sum(strip_nbytes))::Int
-    bitsperpixel = ifd[BITSPERSAMPLE].data::Union{Int,UInt16,Vector{UInt16}}
-    rawtypes = Base.IdSet{Any}()
-    mappedtypes = Base.IdSet{Any}()
-    for i in 1:samplesperpixel
-        rawtype = rawtype_mapping[(SampleFormats(sampleformats[i]), bitsperpixel[i])]
-        push!(rawtypes, rawtype)
-        if rawtype <: Unsigned
-            push!(mappedtypes, Normed{rawtype, sizeof(rawtype)*8})
-        elseif rawtype <: Signed
-            push!(mappedtypes, Fixed{rawtype, sizeof(rawtype)*8-1})
-        else
-            push!(mappedtypes, rawtype)
-        end
-    end
-    (length(rawtypes) > 1) && error("Variable per-pixel storage types are not yet supported")
-    rawtype = first(rawtypes)::DataType
-    readtype = rawtype
-
-    compression = COMPRESSION_NONE
-    if COMPRESSION in ifd
-        compression = CompressionType(ifd[COMPRESSION].data)
-    end
-
-    if compression != COMPRESSION_NONE
-        # recalculate nbytes if the data is compressed since the inflated data
-        # is most likely larger than the bytes on disk
-        nbytes = nrows*ncols*samplesperpixel*sizeof(rawtype)
-        readtype = UInt8
-    end
-
-    IFDLayout(
-        samplesperpixel, nrows, ncols,
-        nbytes,
-        readtype,
-        rawtype,
-        first(mappedtypes)::DataType,
-        compression,
-        PhotometricInterpretations(interpretation))
-end
-
 function Base.read!(target::AbstractArray{T, N}, tf::TiffFile, ifd::IFD) where {T, N}
-    layout = output(ifd)
-
     strip_offsets = ifd[STRIPOFFSETS].data
 
     if PLANARCONFIG in ifd
         planarconfig = ifd[PLANARCONFIG].data
         (planarconfig != 1) && error("Images with data stored in planar format not yet supported")
     end
 
+    rows = nrows(ifd)
+    cols = ncols(ifd)
+    compression = COMPRESSION_NONE
+    try 
+        compression = CompressionType(ifd[COMPRESSION].data)
+    catch
+    end
+
     if !iscontiguous(ifd)
-        rowsperstrip = layout.nrows
+        rowsperstrip = rows
         (ROWSPERSTRIP in ifd) && (rowsperstrip = ifd[ROWSPERSTRIP].data)
-        nstrips = ceil(Int, layout.nrows / rowsperstrip)
+        nstrips = ceil(Int, rows / rowsperstrip)
 
         strip_nbytes = ifd[STRIPBYTECOUNTS].data
 
-        if layout.compression != COMPRESSION_NONE
+        if compression != COMPRESSION_NONE
             # strip_nbytes is the number of bytes pre-inflation so we need to
             # calculate the expected size once decompressed and update the values
-            strip_nbytes = fill(rowsperstrip*layout.ncols, length(strip_nbytes)::Int)
-            strip_nbytes[end] = (layout.nrows - (rowsperstrip * (nstrips-1))) * layout.ncols
+            strip_nbytes = fill(rowsperstrip*cols, length(strip_nbytes)::Int)
+            strip_nbytes[end] = (rows - (rowsperstrip * (nstrips-1))) * cols
         end
 
         startbyte = 1
         for i in 1:nstrips
             seek(tf, strip_offsets[i]::Core.BuiltinInts)
             nbytes = Int(strip_nbytes[i]::Core.BuiltinInts / sizeof(T))
-            read!(tf, view(target, startbyte:(startbyte+nbytes-1)), layout.compression)
+            read!(tf, view(target, startbyte:(startbyte+nbytes-1)), compression)
             startbyte += nbytes
         end
     else
         seek(tf, strip_offsets[1]::Core.BuiltinInts)
-        read!(tf, target, layout.compression)
+        read!(tf, target, compression)
     end
 end
 

src/load.jl

@@ -6,26 +6,12 @@ end
 
 load(io::IOStream; verbose=true, mmap = false) = load(read(io, TiffFile); verbose=verbose, mmap=mmap)
 function load(tf::TiffFile; verbose=true, mmap = false)
-    isdense = true
     ifds = IFD{offset(tf)}[]
 
-    layout = nothing
     nplanes = 0
-
     for ifd in tf
         load!(tf, ifd)
         push!(ifds, ifd)
-
-        new_layout = output(ifd)
-
-        # if we detect variance in the format of the IFD data then we can't
-        # represent the image as a dense array
-        if layout != nothing && layout != new_layout
-            isdense = false
-            @info "Not dense"
-        end
-        layout = new_layout
-
         nplanes += 1
     end
 

src/types/dense.jl

@@ -79,7 +79,7 @@ function _constructifd(data::AbstractArray{T, 2}, ::Type{O}) where {T <: Coloran
     ifd
 end
 
-Base.write(io::IOStream, img::DenseTaggedImage) = write(Stream(format"TIFF", io, extract_filename(io)), img)
+Base.write(io::IOStream, img::DenseTaggedImage) = write(getstream(format"TIFF", io, extract_filename(io)), img)
 
 function Base.write(io::Stream, img::DenseTaggedImage)
     O = offset(img)
@@ -116,6 +116,6 @@ save(io::IO, data::DenseTaggedImage) where {IO <: Union{IOStream, Stream}} = wri
 save(io::IO, data) where {IO <: Union{IOStream, Stream}} = save(io, DenseTaggedImage(data))
 function save(filepath::String, data)
     open(filepath, "w") do io
-        save(Stream(format"TIFF", io, filepath), data)
+        save(getstream(format"TIFF", io, filepath), data)
     end
 end

src/types/mmap.jl

@@ -55,7 +55,7 @@ function Base.getindex(A::DiskTaggedImage{T, O, AA}, i1::Int, i2::Int, i::Int) w
     # if the file isn't open, lets open a handle and update it
     if !isopen(A.file.io)
         path = A.file.filepath
-        A.file.io = Stream(format"TIFF", open(path), path)
+        A.file.io = getstream(format"TIFF", open(path), path)
     end
 
     read!(A.cache, A.file, ifd)

src/utils.jl

@@ -172,4 +172,20 @@ const rawtype_mapping = Dict{Tuple{TiffImages.SampleFormats, UInt16}, DataType}(
     (SAMPLEFORMAT_IEEEFP, 64) => Float64,
 )
 
-Base.bswap(c::Colorant{T, N}) where {T, N} = mapc(bswap, c)
+Base.bswap(c::Colorant{T, N}) where {T, N} = mapc(bswap, c)
+
+function getstream(fmt, io, name)
+    # adapted from https://github.com/JuliaStats/RDatasets.jl/pull/119/
+    if isdefined(FileIO, :action)
+        # FileIO >= 1.6
+        return Stream{fmt}(io, name)
+    else
+        # FileIO < 1.6
+        return Stream(fmt, io, name)
+    end
+end
+
+getstream(fmt, io::IOBuffer) = getstream(fmt, io, "")
+getstream(fmt, io::IOStream) = getstream(fmt, io, extract_filename(io))
+# assume OMETIFF if no format given
+getstream(io) = getstream(format"TIFF", io)

test/Project.toml

@@ -2,6 +2,7 @@
 ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
 ColorVectorSpace = "c3611d14-8923-5661-9e6a-0046d554d3a4"
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
+Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
 FixedPointNumbers = "53c48c17-4a7d-5ca2-90c5-79b7896eea93"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

test/runtests.jl

@@ -9,7 +9,14 @@ using TiffImages
 DocMeta.setdocmeta!(TiffImages, :DocTestSetup, :(using TiffImages); recursive=true)
 doctest(TiffImages)
 
-get_example(name) = download("https://github.com/tlnagy/exampletiffs/blob/master/$name?raw=true")
+_wrap(name) = "https://github.com/tlnagy/exampletiffs/blob/master/$name?raw=true"
+
+if VERSION >= v"1.6.0"
+    using Downloads
+    get_example(x) = Downloads.download(_wrap(x))
+else
+    get_example(x) = download(_wrap(x))
+end
 
 @testset "Tag loading" begin
     include("tags.jl")
@@ -111,6 +118,16 @@ end
     end
 end
 
+@testset "Mmap" begin
+    filepath = get_example("julia.tif")
+    img = TiffImages.load(filepath, mmap=true)
+    @test size(img) == (300, 500, 1)
+    @test all(img[3, 1:50] .== RGB{N0f8}(1, 1, 1))
+    # force close the stream behind the file to see if it's properly reopened 
+    close(img.data.file.io)
+    @test all(img[3, 1:50] .== RGB{N0f8}(1, 1, 1))
+end
+
 @testset "Writing" begin
     include("writer.jl")
 end

test/writer.jl

@@ -88,16 +88,6 @@ end
     TiffImages.load!(tf, read_ifd)
 
     @test all(ifd .== read_ifd)
-
-    expected = TiffImages.IFDLayout(1, 512, 512, 262144,
-                                    UInt8, UInt8, FixedPointNumbers.Normed{UInt8,8},
-                                    TiffImages.COMPRESSION_NONE,
-                                    TiffImages.PHOTOMETRIC_MINISBLACK)
-    @test TiffImages.output(ifd) == expected
-
-    delete!(ifd, TiffImages.COMPRESSION)
-
-    @test TiffImages.output(ifd) == expected
 end
 
 @testset "Simple 2D image" begin