Merge pull request #20 from tlnagy/tn/add-mmap

Basic load-only mmap support

.gitignore

@@ -2,4 +2,4 @@
 Manifest.toml
 /dev/
 docs/build/
-docs/src/generated/
+docs/src/examples/

README.md

@@ -1,12 +1,34 @@
 # 💎 TiffImages.jl
 
-| **Documentation**                 | **Build Status**                                              |
-|:----------------------------------|:--------------------------------------------------------------|
-| [![][docs-dev-img]][docs-dev-url] | [![][status-img]][status-url] [![][travis-img]][travis-url] [![][codecov-img]][codecov-url] |
+| **Stable release** | **Documentation**                 | **Build Status**                                              |
+|:------------------------------------------------------|:-------------------------------------------------------------------------|:--------------------------------------------------------------|
+| ![](https://juliahub.com/docs/TiffImages/version.svg) | [![][docs-stable-img]][docs-stable-url][![][docs-dev-img]][docs-dev-url] | [![][status-img]][status-url] [![][travis-img]][travis-url] [![][codecov-img]][codecov-url] |
 
-This package aims to be a fast, minimal, and correct TIFF reader and writer written in Julia.
+This package aims to be a fast, minimal, and correct TIFF reader and writer
+written in Julia.
 
-This is a WIP. Be warned. Here be🐉.
+## Features
+
+- Fast reading and writing of many common TIFFs
+- Extensible core for other TIFF packages to build on
+- Native integration with `Colors.jl` and the Julia Array ecosystem
+- Memory-mapping for loading images too large to fit in memory
+- Support for BigTIFFs for large images
+
+## Installation
+
+`TiffImages.jl` is available through Julia's general repository. You can install
+it by running the following commands in the Julia REPL:
+
+```julia
+using Pkg
+Pkg.install("TiffImages")
+```
+
+Please see the documentation above for usage details and examples
+
+[docs-stable-img]: https://img.shields.io/badge/docs-stable-blue.svg
+[docs-stable-url]: https://tamasnagy.com/TiffImages.jl/stable
 
 [docs-dev-img]: https://img.shields.io/badge/docs-dev-blue.svg
 [docs-dev-url]: https://tamasnagy.com/TiffImages.jl/dev
@@ -17,5 +39,5 @@ This is a WIP. Be warned. Here be🐉.
 [codecov-img]: https://codecov.io/gh/tlnagy/TiffImages.jl/branch/master/graph/badge.svg
 [codecov-url]: https://codecov.io/gh/tlnagy/TiffImages.jl
 
-[status-img]: https://www.repostatus.org/badges/latest/wip.svg
-[status-url]: https://www.repostatus.org/#wip
+[status-img]: https://www.repostatus.org/badges/latest/active.svg
+[status-url]: https://www.repostatus.org/#active

docs/make.jl

@@ -6,7 +6,7 @@ using Images
 
 EXAMPLE_DIR = joinpath(@__DIR__, "..", "examples")
 EXAMPLES = filter(x->endswith(x, ".jl"), joinpath.(EXAMPLE_DIR, readdir(EXAMPLE_DIR)))
-OUTPUT = joinpath(@__DIR__, "src", "generated")
+OUTPUT = joinpath(@__DIR__, "src", "examples")
 
 for ex in EXAMPLES
     Literate.markdown(ex, OUTPUT, documenter = true)
@@ -23,7 +23,8 @@ makedocs(
     pages = [
         "Home" => "index.md",
         "Examples" => [
-            "Writing TIFFs" => joinpath("generated", "writing.md")
+            "Reading TIFFs" => joinpath("examples", "reading.md")
+            "Writing TIFFs" => joinpath("examples", "writing.md")
         ],
         "Library" => [
             "Public" => joinpath("lib", "public.md"),

docs/src/index.md

@@ -18,9 +18,17 @@ do as much lazily and flexibly as possible.
 
 TiffImages.jl supports:
 
-- [x] The TIFF 6.0 baseline spec
-- [x] Thorough testing
-- [x] HDR images stored as 32bit or 64bit floats
-- [x] BigTIFFs
-- [ ] Out-of-memory support (WIP)
-- [ ] Streaming from disk (WIP)
+- The TIFF 6.0 baseline spec
+- Thorough testing
+- HDR images stored as 32bit or 64bit floats
+- BigTIFFs
+- Memory-mapped loading
+
+## Usage
+
+Check out the examples to see how to use `TiffImages.jl`
+
+```@contents
+Pages = ["examples/reading.md", "examples/writing.md"]
+Depth = 1
+```

docs/src/lib/extend/index.md

@@ -4,5 +4,7 @@ If you want to extend `TiffImages.jl` to add support for more features or change
 TIFF data is loaded, you have come to right place.
 
 ```@docs
+TiffImages.TiffFile
+TiffImages.IFD
 TiffImages.Tag
 ```

examples/reading.jl

@@ -0,0 +1,59 @@
+# # Reading TIFFs
+
+# Loading most TIFFs should just work, see [Writing TIFFs](@ref) for more
+# advanced manipulation of TIFF objects. But we'll quickly run through a common
+# use cases.
+
+#md # ```@contents
+#md # Pages = ["reading.md"]
+#md # Depth = 5
+#md # ```
+
+get_example(name) = download("https://github.com/tlnagy/exampletiffs/blob/master/$name?raw=true") #hide
+filepath = get_example("spring.tif")                                                              #hide
+
+# ### Basic loading
+
+# At it's most basic, we can just point `TiffImages.jl` to the filepath of an
+# image and it will attempt to load it. Here, we're loading `spring.tif` from
+# the [`tlnagy/exampletiffs`](https://github.com/tlnagy/exampletiffs) repo
+
+using TiffImages
+img = TiffImages.load(filepath)
+
+# If you're a graphical environment, you can load the `Images.jl` repo to get
+# a nice graphical representation of your image. If you're in the REPL, I highly
+# recommend the `ImageInTerminal.jl` package for some visual feedback.
+
+# Continuing on, `img` here behaves exactly as you would expect a Julian array
+# to despite the funky type signature
+
+typeof(img)
+
+# Everything should behave as expected
+
+eltype(img)
+#---------
+
+# Accessing and setting data should work as expected
+img[160:180, 50]
+#---------
+img[160:180, 50] .= 1.0
+img
+
+
+# ### Memory-mapped files
+
+# `TiffImages.jl` also supports memory-mapping so that you can load images that
+# are larger than the available memory.
+
+img = TiffImages.load(filepath, mmap=true)
+img[:, :, 1]
+
+# Naturally, this only really makes sense for large images, not the single pane
+# image here. Currently, memory-mapped files are readonly so trying to set a
+# value will throw an error
+
+# ```julia
+# img[1, 1, 1] = 0 # this won't work
+# ```

src/TiffImages.jl

@@ -22,6 +22,7 @@ include("ifds.jl")
 include("layout.jl")
 include(joinpath("types", "common.jl"))
 include(joinpath("types", "dense.jl"))
+include(joinpath("types", "mmap.jl"))
 include("load.jl")
 
 end # module

src/files.jl

@@ -1,7 +1,9 @@
 """
-    TiffFile(io) -> TiffFile
+    $(TYPEDEF) -> TiffFile
 
 Wrap `io` with helper parameters to keep track of file attributes.
+
+$(FIELDS)
 """
 mutable struct TiffFile{O <: Unsigned}
     """The relative path to this file"""

src/ifds.jl

@@ -1,3 +1,9 @@
+"""
+    $(TYPEDEF)
+
+An image file directory is a sorted collection of the tags representing this
+plane in the TIFF file.
+"""
 struct IFD{O <: Unsigned}
     tags::OrderedDict{UInt16, Tag{O}}
 end
@@ -123,7 +129,7 @@ function output(ifd::IFD)
         end
     end
     (length(rawtypes) > 1) && error("Variable per-pixel storage types are not yet supported")
-    rawtype = first(rawtypes)    
+    rawtype = first(rawtypes)
     readtype = rawtype
 
     compression = CompressionType(first(ifd[COMPRESSION].data))
@@ -140,7 +146,7 @@ function output(ifd::IFD)
         nbytes,
         readtype,
         rawtype,
-        first(mappedtypes), 
+        first(mappedtypes),
         compression,
         PhotometricInterpretations(interpretation))
 end
@@ -206,7 +212,7 @@ function Base.write(tf::TiffFile{O}, ifd::IFD{O}) where {O <: Unsigned}
     for (tag, poses) in remotedata
         data_pos = position(tf.io)
         write(tf, tag.data)
-        push!(poses, data_pos)      
+        push!(poses, data_pos)
     end
 
     for (tag, poses) in remotedata
@@ -216,6 +222,6 @@ function Base.write(tf::TiffFile{O}, ifd::IFD{O}) where {O <: Unsigned}
     end
 
     seek(tf, ifd_end_pos)
-    
+
     return ifd_end_pos
 end

src/layout.jl

@@ -13,7 +13,7 @@ function interpretation(ifd::IFD)
     interp = PhotometricInterpretations(first(ifd[PHOTOMETRIC].data))
     extras = EXTRASAMPLE_UNSPECIFIED
     if EXTRASAMPLES in ifd
-        try 
+        try
             extras = ExtraSamples(first(ifd[EXTRASAMPLES].data))
         catch
             extras = EXTRASAMPLE_ASSOCALPHA
@@ -23,7 +23,7 @@ function interpretation(ifd::IFD)
 end
 
 # dummy color type for palette colored images to dispatch on
-struct Palette{T} <: Colorant{T, 1} 
+struct Palette{T} <: Colorant{T, 1}
     i::T
 end
 Base.reinterpret(::Type{Palette{T}}, arr::A) where {T, N, S, A <: AbstractArray{S, N}} = arr
@@ -71,4 +71,15 @@ function rawtype(ifd::IFD)
     rawtype_mapping[SampleFormats(first(sampleformats)), first(bitsperpixel)]
 end
 
+"""
+    $(SIGNATURES)
+
+Allocate a cache for this IFD with correct type and size.
+"""
+getcache(ifd::IFD) = getcache(ifd, Val(rawtype(ifd)))
+getcache(ifd::IFD, ::Val{Bool}) = BitArray(undef, ncols(ifd), nrows(ifd))
+function getcache(ifd::IFD, ::Val{T}) where {T}
+    colortype, extras = interpretation(ifd)
 
+    Array{colortype{_mappedtype(T)}}(undef, ncols(ifd), nrows(ifd))
+end

src/load.jl

@@ -1,10 +1,10 @@
-function load(filepath::String; verbose=true)
+function load(filepath::String; verbose=true, mmap = false)
     open(filepath) do io
-        load(io; verbose=verbose)
+        load(io; verbose=verbose, mmap=mmap)
     end
 end
 
-function load(io::IOStream; verbose=true)
+function load(io::IOStream; verbose=true, mmap = false)
     tf = read(io, TiffFile)
 
     isdense = true
@@ -30,8 +30,12 @@ function load(io::IOStream; verbose=true)
         nplanes += 1
     end
 
-    loaded = load(tf, layout, ifds, Val(nplanes); verbose=verbose)
-
+    if mmap
+        loaded = DiskTaggedImage(tf, ifds)
+    else
+        loaded = load(tf, ifds, Val(nplanes); verbose=verbose)
+    end
+
     if eltype(loaded) <: Palette
         ifd = ifds[1]
         raw = rawtype(ifd)
@@ -50,32 +54,18 @@ function load(io::IOStream; verbose=true)
     return DenseTaggedImage(data, ifds)
 end
 
-function load(tf::TiffFile, layout::IFDLayout, ifds, ::Val{1}; verbose = true)
+function load(tf::TiffFile, ifds, ::Val{1}; verbose = true)
     ifd = ifds[1]
-
-    colortype, extras = interpretation(ifd)
-
-    if layout.rawtype == Bool
-        cache = BitArray(undef, ncols(ifd), nrows(ifd))
-    else
-        cache = Array{colortype{layout.mappedtype}}(undef, ncols(ifd), nrows(ifd))
-    end
-
+    cache = getcache(ifd)
     read!(cache, tf, ifd)
 
     return Array(cache')
 end
 
-function load(tf::TiffFile, layout::IFDLayout, ifds, ::Val{N}; verbose = true) where {N}
+function load(tf::TiffFile, ifds, ::Val{N}; verbose = true) where {N}
     ifd = ifds[1]
 
-    colortype, extras = interpretation(ifd)
-
-    if layout.rawtype == Bool
-        cache = BitArray(undef, ncols(ifd), nrows(ifd))
-    else
-        cache = Array{colortype{layout.mappedtype}}(undef, ncols(ifd), nrows(ifd))
-    end
+    cache = getcache(ifd)
 
     data = similar(cache, nrows(ifd), ncols(ifd), N)
 
@@ -84,6 +74,6 @@ function load(tf::TiffFile, layout::IFDLayout, ifds, ::Val{N}; verbose = true) w
         read!(cache, tf, ifd)
         data[:, :, idx] .= cache'
     end
-    
+
     return data
 end