From e864affaa71f24f5be352ca4b39ce3eead2f7450 Mon Sep 17 00:00:00 2001 From: Tamas Nagy Date: Fri, 16 Apr 2021 16:49:14 -0700 Subject: [PATCH 1/3] fix deprecations on Julia 1.6 and FileIO 1.6 --- src/files.jl | 4 ++-- src/types/dense.jl | 4 ++-- src/types/mmap.jl | 2 +- src/utils.jl | 18 +++++++++++++++++- test/Project.toml | 1 + test/runtests.jl | 9 ++++++++- 6 files changed, 31 insertions(+), 7 deletions(-) diff --git a/src/files.jl b/src/files.jl index 77fd2106..42eb013a 100644 --- a/src/files.jl +++ b/src/files.jl @@ -20,7 +20,7 @@ mutable struct TiffFile{O <: Unsigned, S <: Stream} end TiffFile{O}(s::Stream) where O <: Unsigned = TiffFile{O, typeof(s)}("", s, -1, false) -TiffFile{O}(io::IO) where O <: Unsigned = TiffFile{O}(Stream(format"TIFF", io)) +TiffFile{O}(io::IO) where O <: Unsigned = TiffFile{O}(getstream(format"TIFF", io)) TiffFile{O}() where O <: Unsigned = TiffFile{O}(IOBuffer()) function Base.read(io::Stream, ::Type{TiffFile}) @@ -33,7 +33,7 @@ function Base.read(io::Stream, ::Type{TiffFile}) TiffFile{offset_size, typeof(io)}(filepath, io, first_offset, bs) end -Base.read(io::IOStream, t::Type{TiffFile}) = read(Stream(format"TIFF", io, extract_filename(io)), t) +Base.read(io::IOStream, t::Type{TiffFile}) = read(getstream(format"TIFF", io, extract_filename(io)), t) function Base.write(file::TiffFile{O}) where O seekstart(file.io) diff --git a/src/types/dense.jl b/src/types/dense.jl index db215035..8240a563 100644 --- a/src/types/dense.jl +++ b/src/types/dense.jl @@ -79,7 +79,7 @@ function _constructifd(data::AbstractArray{T, 2}, ::Type{O}) where {T <: Coloran ifd end -Base.write(io::IOStream, img::DenseTaggedImage) = write(Stream(format"TIFF", io, extract_filename(io)), img) +Base.write(io::IOStream, img::DenseTaggedImage) = write(getstream(format"TIFF", io, extract_filename(io)), img) function Base.write(io::Stream, img::DenseTaggedImage) O = offset(img) @@ -116,6 +116,6 @@ save(io::IO, data::DenseTaggedImage) where {IO <: Union{IOStream, Stream}} = wri save(io::IO, data) where {IO <: Union{IOStream, Stream}} = save(io, DenseTaggedImage(data)) function save(filepath::String, data) open(filepath, "w") do io - save(Stream(format"TIFF", io, filepath), data) + save(getstream(format"TIFF", io, filepath), data) end end diff --git a/src/types/mmap.jl b/src/types/mmap.jl index c9b8c368..a0e38836 100644 --- a/src/types/mmap.jl +++ b/src/types/mmap.jl @@ -55,7 +55,7 @@ function Base.getindex(A::DiskTaggedImage{T, O, AA}, i1::Int, i2::Int, i::Int) w # if the file isn't open, lets open a handle and update it if !isopen(A.file.io) path = A.file.filepath - A.file.io = Stream(format"TIFF", open(path), path) + A.file.io = getstream(format"TIFF", open(path), path) end read!(A.cache, A.file, ifd) diff --git a/src/utils.jl b/src/utils.jl index 5a0ca8e5..2ea82de2 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -172,4 +172,20 @@ const rawtype_mapping = Dict{Tuple{TiffImages.SampleFormats, UInt16}, DataType}( (SAMPLEFORMAT_IEEEFP, 64) => Float64, ) -Base.bswap(c::Colorant{T, N}) where {T, N} = mapc(bswap, c) \ No newline at end of file +Base.bswap(c::Colorant{T, N}) where {T, N} = mapc(bswap, c) + +function getstream(fmt, io, name) + # adapted from https://github.com/JuliaStats/RDatasets.jl/pull/119/ + if isdefined(FileIO, :action) + # FileIO >= 1.6 + return Stream{fmt}(io, name) + else + # FileIO < 1.6 + return Stream(fmt, io, name) + end +end + +getstream(fmt, io::IOBuffer) = getstream(fmt, io, "") +getstream(fmt, io::IOStream) = getstream(fmt, io, extract_filename(io)) +# assume OMETIFF if no format given +getstream(io) = getstream(format"TIFF", io) \ No newline at end of file diff --git a/test/Project.toml b/test/Project.toml index 82462fc5..7e5a7020 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -2,6 +2,7 @@ ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" ColorVectorSpace = "c3611d14-8923-5661-9e6a-0046d554d3a4" Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" FixedPointNumbers = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" diff --git a/test/runtests.jl b/test/runtests.jl index 5f778883..61a72208 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -9,7 +9,14 @@ using TiffImages DocMeta.setdocmeta!(TiffImages, :DocTestSetup, :(using TiffImages); recursive=true) doctest(TiffImages) -get_example(name) = download("https://github.com/tlnagy/exampletiffs/blob/master/$name?raw=true") +_wrap(name) = "https://github.com/tlnagy/exampletiffs/blob/master/$name?raw=true" + +if VERSION >= v"1.6.0" + using Downloads + get_example(x) = Downloads.download(_wrap(x)) +else + get_example(x) = download(_wrap(x)) +end @testset "Tag loading" begin include("tags.jl") From 839fa69813ecb2b01f7d08de45c3d889969ee6ad Mon Sep 17 00:00:00 2001 From: Tamas Nagy Date: Fri, 16 Apr 2021 16:50:46 -0700 Subject: [PATCH 2/3] remove old layout code the sparse TIFF handling is pretty limited anyway so ripping it out for now should be fine --- Project.toml | 2 +- src/ifds.jl | 90 +++++++++----------------------------------------- src/load.jl | 13 -------- test/writer.jl | 10 ------ 4 files changed, 16 insertions(+), 99 deletions(-) diff --git a/Project.toml b/Project.toml index f200de58..2348e208 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "TiffImages" uuid = "731e570b-9d59-4bfa-96dc-6df516fadf69" authors = ["Tamas Nagy "] -version = "0.3.0" +version = "0.3.1" [deps] ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" diff --git a/src/ifds.jl b/src/ifds.jl index 91cd872f..41efb7bf 100644 --- a/src/ifds.jl +++ b/src/ifds.jl @@ -110,75 +110,7 @@ function Base.iterate(file::TiffFile, state::Tuple{Union{IFD{O}, Nothing}, Int}) return (curr_ifd, (next_ifd, next_ifd_offset)) end -struct IFDLayout - nsamples::Int - nrows::Int - ncols::Int - nbytes::Int - readtype::DataType - rawtype::DataType - mappedtype::DataType - compression::CompressionType - interpretation::PhotometricInterpretations -end - -function output(ifd::IFD) - nrows = Int(ifd[IMAGELENGTH].data)::Int - ncols = Int(ifd[IMAGEWIDTH].data)::Int - - samplesperpixel = Int(ifd[SAMPLESPERPIXEL].data)::Int - sampleformats = fill(UInt16(0x01), samplesperpixel) - if SAMPLEFORMAT in ifd - sampleformats = ifd[SAMPLEFORMAT].data # can a type be specified for this? - end - - interpretation = Int(ifd[PHOTOMETRIC].data)::Int - - strip_nbytes = ifd[STRIPBYTECOUNTS].data - nbytes = Int(sum(strip_nbytes))::Int - bitsperpixel = ifd[BITSPERSAMPLE].data::Union{Int,UInt16,Vector{UInt16}} - rawtypes = Base.IdSet{Any}() - mappedtypes = Base.IdSet{Any}() - for i in 1:samplesperpixel - rawtype = rawtype_mapping[(SampleFormats(sampleformats[i]), bitsperpixel[i])] - push!(rawtypes, rawtype) - if rawtype <: Unsigned - push!(mappedtypes, Normed{rawtype, sizeof(rawtype)*8}) - elseif rawtype <: Signed - push!(mappedtypes, Fixed{rawtype, sizeof(rawtype)*8-1}) - else - push!(mappedtypes, rawtype) - end - end - (length(rawtypes) > 1) && error("Variable per-pixel storage types are not yet supported") - rawtype = first(rawtypes)::DataType - readtype = rawtype - - compression = COMPRESSION_NONE - if COMPRESSION in ifd - compression = CompressionType(ifd[COMPRESSION].data) - end - - if compression != COMPRESSION_NONE - # recalculate nbytes if the data is compressed since the inflated data - # is most likely larger than the bytes on disk - nbytes = nrows*ncols*samplesperpixel*sizeof(rawtype) - readtype = UInt8 - end - - IFDLayout( - samplesperpixel, nrows, ncols, - nbytes, - readtype, - rawtype, - first(mappedtypes)::DataType, - compression, - PhotometricInterpretations(interpretation)) -end - function Base.read!(target::AbstractArray{T, N}, tf::TiffFile, ifd::IFD) where {T, N} - layout = output(ifd) - strip_offsets = ifd[STRIPOFFSETS].data if PLANARCONFIG in ifd @@ -186,30 +118,38 @@ function Base.read!(target::AbstractArray{T, N}, tf::TiffFile, ifd::IFD) where { (planarconfig != 1) && error("Images with data stored in planar format not yet supported") end + rows = nrows(ifd) + cols = ncols(ifd) + compression = COMPRESSION_NONE + try + compression = CompressionType(ifd[COMPRESSION].data) + catch + end + if !iscontiguous(ifd) - rowsperstrip = layout.nrows + rowsperstrip = rows (ROWSPERSTRIP in ifd) && (rowsperstrip = ifd[ROWSPERSTRIP].data) - nstrips = ceil(Int, layout.nrows / rowsperstrip) + nstrips = ceil(Int, rows / rowsperstrip) strip_nbytes = ifd[STRIPBYTECOUNTS].data - if layout.compression != COMPRESSION_NONE + if compression != COMPRESSION_NONE # strip_nbytes is the number of bytes pre-inflation so we need to # calculate the expected size once decompressed and update the values - strip_nbytes = fill(rowsperstrip*layout.ncols, length(strip_nbytes)::Int) - strip_nbytes[end] = (layout.nrows - (rowsperstrip * (nstrips-1))) * layout.ncols + strip_nbytes = fill(rowsperstrip*cols, length(strip_nbytes)::Int) + strip_nbytes[end] = (rows - (rowsperstrip * (nstrips-1))) * cols end startbyte = 1 for i in 1:nstrips seek(tf, strip_offsets[i]::Core.BuiltinInts) nbytes = Int(strip_nbytes[i]::Core.BuiltinInts / sizeof(T)) - read!(tf, view(target, startbyte:(startbyte+nbytes-1)), layout.compression) + read!(tf, view(target, startbyte:(startbyte+nbytes-1)), compression) startbyte += nbytes end else seek(tf, strip_offsets[1]::Core.BuiltinInts) - read!(tf, target, layout.compression) + read!(tf, target, compression) end end diff --git a/src/load.jl b/src/load.jl index c47fdf13..eceb1a5d 100644 --- a/src/load.jl +++ b/src/load.jl @@ -9,23 +9,10 @@ function load(tf::TiffFile; verbose=true, mmap = false) isdense = true ifds = IFD{offset(tf)}[] - layout = nothing nplanes = 0 - for ifd in tf load!(tf, ifd) push!(ifds, ifd) - - new_layout = output(ifd) - - # if we detect variance in the format of the IFD data then we can't - # represent the image as a dense array - if layout != nothing && layout != new_layout - isdense = false - @info "Not dense" - end - layout = new_layout - nplanes += 1 end diff --git a/test/writer.jl b/test/writer.jl index 4a03b044..5720450e 100644 --- a/test/writer.jl +++ b/test/writer.jl @@ -88,16 +88,6 @@ end TiffImages.load!(tf, read_ifd) @test all(ifd .== read_ifd) - - expected = TiffImages.IFDLayout(1, 512, 512, 262144, - UInt8, UInt8, FixedPointNumbers.Normed{UInt8,8}, - TiffImages.COMPRESSION_NONE, - TiffImages.PHOTOMETRIC_MINISBLACK) - @test TiffImages.output(ifd) == expected - - delete!(ifd, TiffImages.COMPRESSION) - - @test TiffImages.output(ifd) == expected end @testset "Simple 2D image" begin From 90d8481c40a7ffa24086f2e0644e0c3cdc752586 Mon Sep 17 00:00:00 2001 From: Tamas Nagy Date: Fri, 16 Apr 2021 17:12:19 -0700 Subject: [PATCH 3/3] add basic mmap tests --- src/load.jl | 1 - test/runtests.jl | 10 ++++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/load.jl b/src/load.jl index eceb1a5d..e78a8815 100644 --- a/src/load.jl +++ b/src/load.jl @@ -6,7 +6,6 @@ end load(io::IOStream; verbose=true, mmap = false) = load(read(io, TiffFile); verbose=verbose, mmap=mmap) function load(tf::TiffFile; verbose=true, mmap = false) - isdense = true ifds = IFD{offset(tf)}[] nplanes = 0 diff --git a/test/runtests.jl b/test/runtests.jl index 61a72208..8f79ef67 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -118,6 +118,16 @@ end end end +@testset "Mmap" begin + filepath = get_example("julia.tif") + img = TiffImages.load(filepath, mmap=true) + @test size(img) == (300, 500, 1) + @test all(img[3, 1:50] .== RGB{N0f8}(1, 1, 1)) + # force close the stream behind the file to see if it's properly reopened + close(img.data.file.io) + @test all(img[3, 1:50] .== RGB{N0f8}(1, 1, 1)) +end + @testset "Writing" begin include("writer.jl") end