From dac4b9d4ac904a809eb82d5b0fbe612ce3657dd8 Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Sat, 25 Mar 2023 02:36:00 -0400 Subject: [PATCH] feat: use distributed for paralell processing --- Manifest.toml | 32 ++++++++++++++++---------------- Project.toml | 1 + src/IceFloeTracker.jl | 6 ++++-- src/pipeline/preprocess.jl | 4 ++-- test/Manifest.toml | 31 +++++++++++++++++++++++++++---- test/Project.toml | 1 + test/runtests.jl | 1 + test/test-pipeline.jl | 2 ++ 8 files changed, 54 insertions(+), 24 deletions(-) diff --git a/Manifest.toml b/Manifest.toml index 56c1e96f..882f08f5 100644 --- a/Manifest.toml +++ b/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.8.5" manifest_format = "2.0" -project_hash = "ece8674ce0cb697ff8b97320835d14c18533acb1" +project_hash = "a79e9aeadb335a5108825546c21f209cbd6b755d" [[deps.AbstractFFTs]] deps = ["ChainRulesCore", "LinearAlgebra"] @@ -39,9 +39,9 @@ version = "0.2.0" [[deps.ArrayInterface]] deps = ["Adapt", "LinearAlgebra", "Requires", "SnoopPrecompile", "SparseArrays", "SuiteSparse"] -git-tree-sha1 = "a89acc90c551067cd84119ff018619a1a76c6277" +git-tree-sha1 = "af2fd6a5d299078a6d7827ce3356b68d2de5470b" uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" -version = "7.2.1" +version = "7.3.1" [[deps.ArrayInterfaceCore]] deps = ["LinearAlgebra", "SnoopPrecompile", "SparseArrays", "SuiteSparse"] @@ -127,9 +127,9 @@ version = "0.1.12" [[deps.Clustering]] deps = ["Distances", "LinearAlgebra", "NearestNeighbors", "Printf", "Random", "SparseArrays", "Statistics", "StatsBase"] -git-tree-sha1 = "64df3da1d2a26f4de23871cd1b6482bb68092bd5" +git-tree-sha1 = "7ebbd653f74504447f1c33b91cd706a69a1b189f" uuid = "aaaa29a8-35af-508c-8bc3-b662a17a0fe5" -version = "0.14.3" +version = "0.14.4" [[deps.ColorSchemes]] deps = ["ColorTypes", "ColorVectorSpace", "Colors", "FixedPointNumbers", "Random", "SnoopPrecompile"] @@ -691,9 +691,9 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" [[deps.LoopVectorization]] deps = ["ArrayInterface", "ArrayInterfaceCore", "CPUSummary", "ChainRulesCore", "CloseOpenIntervals", "DocStringExtensions", "ForwardDiff", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "SIMDTypes", "SLEEFPirates", "SnoopPrecompile", "SpecialFunctions", "Static", "StaticArrayInterface", "ThreadingUtilities", "UnPack", "VectorizationBase"] -git-tree-sha1 = "2acf6874142d05d5d1ad49e8d3786b8cd800936d" +git-tree-sha1 = "6d878b416680b7eaf5fae71990028c982e7c8b38" uuid = "bdcacae8-1622-11e9-2a5c-532679323890" -version = "0.12.152" +version = "0.12.153" [[deps.MKL_jll]] deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "Pkg"] @@ -709,9 +709,9 @@ version = "0.5.10" [[deps.MakieCore]] deps = ["Observables"] -git-tree-sha1 = "2c3fc86d52dfbada1a2e5e150e50f06c30ef149c" +git-tree-sha1 = "9926529455a331ed73c19ff06d16906737a876ed" uuid = "20f20a25-4f0e-4fdf-b5d1-57303727442b" -version = "0.6.2" +version = "0.6.3" [[deps.ManualMemory]] git-tree-sha1 = "bcaef4fc7a0cfe2cba636d84cda54b5e4e4ca3cd" @@ -1156,9 +1156,9 @@ version = "1.3.0" [[deps.StaticArrays]] deps = ["LinearAlgebra", "Random", "StaticArraysCore", "Statistics"] -git-tree-sha1 = "6aa098ef1012364f2ede6b17bf358c7f1fbe90d4" +git-tree-sha1 = "b8d897fe7fa688e93aef573711cb207c08c9e11e" uuid = "90137ffa-7385-5640-81b9-e52037218182" -version = "1.5.17" +version = "1.5.19" [[deps.StaticArraysCore]] git-tree-sha1 = "6b7ba252635a5eff6a0b0664a41ee140a1c9e72a" @@ -1272,9 +1272,9 @@ uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" [[deps.VectorizationBase]] deps = ["ArrayInterface", "CPUSummary", "HostCPUFeatures", "IfElse", "LayoutPointers", "Libdl", "LinearAlgebra", "SIMDTypes", "Static", "StaticArrayInterface"] -git-tree-sha1 = "a2d69abfc8b9b71a5db110f9757d0cd4a27877c0" +git-tree-sha1 = "a07b279148653974edd945c8c8e09af156bcb46e" uuid = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f" -version = "0.21.61" +version = "0.21.62" [[deps.VersionParsing]] git-tree-sha1 = "58d6e80b4ee071f5efd07fda82cb9fbe17200868" @@ -1299,10 +1299,10 @@ uuid = "3161d3a3-bdf6-5164-811a-617609db77b4" version = "1.5.4+0" [[deps.ZygoteRules]] -deps = ["MacroTools"] -git-tree-sha1 = "8c1a8e4dfacb1fd631745552c8db35d0deb09ea0" +deps = ["ChainRulesCore", "MacroTools"] +git-tree-sha1 = "977aed5d006b840e2e40c0b48984f7463109046d" uuid = "700de1a5-db45-46bc-99cf-38207098b444" -version = "0.2.2" +version = "0.2.3" [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl", "OpenBLAS_jll"] diff --git a/Project.toml b/Project.toml index ab616183..9e4f3f0f 100644 --- a/Project.toml +++ b/Project.toml @@ -12,6 +12,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" DataStructures = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" Folds = "41a02a25-b8f0-4f67-bc48-60067656b558" Git = "d7ba0133-e1db-5d97-8f8c-041e4b3a1eb2" diff --git a/src/IceFloeTracker.jl b/src/IceFloeTracker.jl index 28f2f668..b45513a0 100644 --- a/src/IceFloeTracker.jl +++ b/src/IceFloeTracker.jl @@ -17,7 +17,8 @@ using RegisterQD using StaticArrays using OffsetArrays: centered using Serialization: serialize, deserialize -using Folds +using Distributed +# addprocs(1) export readdlm, padnhood, bridge, branch, @persist, load, cloudmask, create_cloudmask, deserialize, serialize, check_landmask_path, create_landmask, RGB, Gray, float64, imsharpen, label_components, regionprops_table, loadimg @@ -62,7 +63,8 @@ This module contains the wrapper functions called by CLI. """ module Pipeline using IceFloeTracker -using IceFloeTracker: Folds, DataFrame, RGB, Gray, load, float64, imsharpen +using IceFloeTracker: DataFrame, RGB, Gray, load, float64, imsharpen +using Distributed: pmap, @everywhere include("pipeline/landmask.jl") include("pipeline/preprocess.jl") include("pipeline/feature-extraction.jl") diff --git a/src/pipeline/preprocess.jl b/src/pipeline/preprocess.jl index b36d5c8e..d327a266 100644 --- a/src/pipeline/preprocess.jl +++ b/src/pipeline/preprocess.jl @@ -172,8 +172,8 @@ function preprocess(truecolor_image::T, reflectance_image::T, landmask_imgs::Nam # Process watershed in parallel using Folds @info "Building watersheds" - # container_for_watersheds = [landmask_imgs.non_dilated, similar(landmask_imgs.non_dilated)] - watersheds_segB = Folds.map(IceFloeTracker.watershed_ice_floes, [segB.not_ice_bit, segB.ice_intersect]) + + watersheds_segB = pmap(IceFloeTracker.watershed_ice_floes, [segB.not_ice_bit, segB.ice_intersect]) # reuse the memory allocated for the first watershed watersheds_segB[1] .= IceFloeTracker.watershed_product(watersheds_segB...) diff --git a/test/Manifest.toml b/test/Manifest.toml index 786b9d00..cf3f4316 100644 --- a/test/Manifest.toml +++ b/test/Manifest.toml @@ -1,7 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.7.2" +julia_version = "1.8.5" manifest_format = "2.0" +project_hash = "fb6cf52084391b6c0b6564e1a1263b4b84e82dc8" [[deps.AbstractFFTs]] deps = ["ChainRulesCore", "LinearAlgebra"] @@ -17,6 +18,7 @@ version = "3.4.0" [[deps.ArgTools]] uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" [[deps.ArnoldiMethod]] deps = ["LinearAlgebra", "Random", "StaticArrays"] @@ -104,6 +106,7 @@ version = "3.45.0" [[deps.CompilerSupportLibraries_jll]] deps = ["Artifacts", "Libdl"] uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.0.1+0" [[deps.ComputationalResources]] git-tree-sha1 = "52cb3ec90e8a8bea0e62e275ba577ad0f74821f7" @@ -179,8 +182,9 @@ uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" version = "0.9.1" [[deps.Downloads]] -deps = ["ArgTools", "LibCURL", "NetworkOptions"] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" [[deps.DualNumbers]] deps = ["Calculus", "NaNMath", "SpecialFunctions"] @@ -212,6 +216,9 @@ git-tree-sha1 = "94f5101b96d2d968ace56f7f2db19d0a5f592e28" uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" version = "1.15.0" +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + [[deps.FixedPointNumbers]] deps = ["Statistics"] git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc" @@ -230,9 +237,9 @@ uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" [[deps.Ghostscript_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] -git-tree-sha1 = "78e2c69783c9753a91cdae88a8d432be85a2ab5e" +git-tree-sha1 = "43ba3d3c82c18d88471cfd2924931658838c9d8f" uuid = "61579ee1-b43e-5ca0-a5da-69d92c66a64b" -version = "9.55.0+0" +version = "9.55.0+4" [[deps.Graphics]] deps = ["Colors", "LinearAlgebra", "NaNMath"] @@ -460,10 +467,12 @@ version = "0.3.1" [[deps.LibCURL]] deps = ["LibCURL_jll", "MozillaCACerts_jll"] uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" [[deps.LibCURL_jll]] deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" [[deps.LibGit2]] deps = ["Base64", "NetworkOptions", "Printf", "SHA"] @@ -472,6 +481,7 @@ uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" [[deps.LibSSH2_jll]] deps = ["Artifacts", "Libdl", "MbedTLS_jll"] uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" [[deps.Libdl]] uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" @@ -519,6 +529,7 @@ uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" [[deps.MbedTLS_jll]] deps = ["Artifacts", "Libdl"] uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.0+0" [[deps.MetaGraphs]] deps = ["Graphs", "JLD2", "Random"] @@ -543,6 +554,7 @@ version = "0.3.3" [[deps.MozillaCACerts_jll]] uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.2.1" [[deps.NaNMath]] deps = ["OpenLibm_jll"] @@ -564,6 +576,7 @@ version = "1.0.2" [[deps.NetworkOptions]] uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" [[deps.OffsetArrays]] deps = ["Adapt"] @@ -574,6 +587,7 @@ version = "1.12.7" [[deps.OpenBLAS_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.20+0" [[deps.OpenEXR]] deps = ["Colors", "FileIO", "OpenEXR_jll"] @@ -590,6 +604,7 @@ version = "3.1.1+0" [[deps.OpenLibm_jll]] deps = ["Artifacts", "Libdl"] uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.1+0" [[deps.OpenSpecFun_jll]] deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] @@ -629,6 +644,7 @@ version = "2.3.2" [[deps.Pkg]] deps = ["Artifacts", "Dates", "Downloads", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.8.0" [[deps.PkgVersion]] deps = ["Pkg"] @@ -726,6 +742,7 @@ version = "1.3.1" [[deps.SHA]] uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" [[deps.Serialization]] uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" @@ -807,6 +824,7 @@ version = "0.33.21" [[deps.TOML]] deps = ["Dates"] uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.0" [[deps.TableTraits]] deps = ["IteratorInterfaceExtensions"] @@ -823,6 +841,7 @@ version = "1.7.0" [[deps.Tar]] deps = ["ArgTools", "SHA"] uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.1" [[deps.TensorCore]] deps = ["LinearAlgebra"] @@ -878,6 +897,7 @@ version = "0.5.5" [[deps.Zlib_jll]] deps = ["Libdl"] uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.12+3" [[deps.Zstd_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] @@ -888,6 +908,7 @@ version = "1.5.2+0" [[deps.libblastrampoline_jll]] deps = ["Artifacts", "Libdl", "OpenBLAS_jll"] uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.1.1+0" [[deps.libpng_jll]] deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] @@ -904,7 +925,9 @@ version = "1.8.6+1" [[deps.nghttp2_jll]] deps = ["Artifacts", "Libdl"] uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" [[deps.p7zip_jll]] deps = ["Artifacts", "Libdl"] uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" diff --git a/test/Project.toml b/test/Project.toml index 101cf906..1494c441 100644 --- a/test/Project.toml +++ b/test/Project.toml @@ -2,6 +2,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab" +Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" ImageTransformations = "02fcd773-0e25-5acc-982a-7f6622650795" Images = "916415d5-f1e6-5110-898d-aaa5f9f070e0" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" diff --git a/test/runtests.jl b/test/runtests.jl index d90ed588..23a7540b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -8,6 +8,7 @@ using Random using ImageTransformations: imrotate include("test_error_rate.jl") include("config.jl") +using Distributed # Setting things up (see config.jl) diff --git a/test/test-pipeline.jl b/test/test-pipeline.jl index 6285d0df..9a0ce7b1 100644 --- a/test/test-pipeline.jl +++ b/test/test-pipeline.jl @@ -1,3 +1,5 @@ +nprocs() == 1 && addprocs(1) +@everywhere using IceFloeTracker: watershed_ice_floes @testset verbose = true "pipeline" begin using IceFloeTracker.Pipeline println("-------------------------------------------------")