From cdf232857d736baefdf46ba3ea37c53b6accd5f0 Mon Sep 17 00:00:00 2001 From: gaelforget Date: Tue, 5 Sep 2023 08:37:03 -0400 Subject: [PATCH 1/5] update notebook (w Pkg.activate) --- docs/src/notebook.jl | 557 ++----------------------------------------- 1 file changed, 24 insertions(+), 533 deletions(-) diff --git a/docs/src/notebook.jl b/docs/src/notebook.jl index a5dc1cd..9e1b854 100644 --- a/docs/src/notebook.jl +++ b/docs/src/notebook.jl @@ -1,5 +1,5 @@ ### A Pluto.jl notebook ### -# v0.19.25 +# v0.19.27 using Markdown using InteractiveUtils @@ -16,6 +16,7 @@ end # ╔═╡ 8e7742bc-ed2e-11ec-2bbe-adbcf21330e7 begin + using Pkg; Pkg.activate() using Dataverse, PlutoUI, Downloads using DataFrames, PrettyTables "Packages ready" @@ -58,9 +59,13 @@ end # ╔═╡ 2240395b-4112-4e8a-a233-26fde9dffbc4 begin + df=Dataverse.file_list(DOI) + + #alternative : + #🏁 #dataset = NativeApi.get_dataset(DOI) #df=pyDataverse.dataset_file_list(DOI) - df=Dataverse.file_list(DOI) + "Done scanning dataset files" end @@ -79,8 +84,23 @@ md"""## Explore a Dataverse A Dataverse may contain multiple Datasets. Here each index of `tree` is one Dataset. Below we select one of them. """ +# ╔═╡ e34e0d5b-7e43-4acd-9cf4-5916cd1b5404 +md""" Download file ? $(@bind dl Select([true,false],default=true))""" + +# ╔═╡ e448e0ce-4991-4a75-b611-570aa64439f3 +md"""## Appendix""" + +# ╔═╡ d216f95e-fc57-4b89-b796-be08b3f137d2 +begin + (DataAccessApi,NativeApi)=pyDataverse.APIs(do_install=true) + 🏁=true + "pyDataverse APIs are ready to use" +end + + # ╔═╡ 756bec67-c27e-49c1-9573-9521746cf856 begin + 🏁 tree=pyDataverse.dataverse_file_list() "Done scanning dataverse files" end @@ -108,24 +128,20 @@ begin nfi=length(tree[num].filename) file_select = @bind file Select(tree[num].filename) ii=findall(tree[num].filename.==file) - pidURL=tree[num].pidURL[ii][1] md"""## Select & Download File Select file : $(file_select) """ end -# ╔═╡ e34e0d5b-7e43-4acd-9cf4-5916cd1b5404 -md""" Download file ? $(@bind dl Select([true,false],default=true))""" - # ╔═╡ ffe06b1c-0e0e-44df-93d5-c33d88bf7e7e begin - filePATH=joinpath(tempdir(),file) - dl ? Downloads.download(pidURL,filePATH) : nothing + dl ? Dataverse.file_download(DOI,file) : nothing end # ╔═╡ 1a5a27c9-7f72-49b7-a406-2576c11f0f3c begin + filePATH=joinpath(tempdir(),file) if isfile(filePATH) stat(filePATH) else @@ -133,529 +149,6 @@ begin end end -# ╔═╡ e448e0ce-4991-4a75-b611-570aa64439f3 -md"""## Appendix""" - -# ╔═╡ d216f95e-fc57-4b89-b796-be08b3f137d2 -begin - (DataAccessApi,NativeApi)=pyDataverse.APIs(do_install=true) - "pyDataverse APIs ready" -end - -# ╔═╡ 00000000-0000-0000-0000-000000000001 -PLUTO_PROJECT_TOML_CONTENTS = """ -[deps] -DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -Dataverse = "9c0b9be8-e31e-490f-90fe-77697562404d" -Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8" -PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" - -[compat] -DataFrames = "~1.5.0" -Dataverse = "~0.2.0" -PlutoUI = "~0.7.50" -PrettyTables = "~2.2.3" -""" - -# ╔═╡ 00000000-0000-0000-0000-000000000002 -PLUTO_MANIFEST_TOML_CONTENTS = """ -# This file is machine-generated - editing it directly is not advised - -julia_version = "1.9.0-rc2" -manifest_format = "2.0" -project_hash = "f7f2e37c2a958c9e93193113041d46649f362d1b" - -[[deps.AbstractPlutoDingetjes]] -deps = ["Pkg"] -git-tree-sha1 = "8eaf9f1b4921132a4cff3f36a1d9ba923b14a481" -uuid = "6e696c72-6542-2067-7265-42206c756150" -version = "1.1.4" - -[[deps.ArgTools]] -uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" -version = "1.1.1" - -[[deps.Artifacts]] -uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" - -[[deps.Base64]] -uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" - -[[deps.BitFlags]] -git-tree-sha1 = "43b1a4a8f797c1cddadf60499a8a077d4af2cd2d" -uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" -version = "0.1.7" - -[[deps.CSV]] -deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "SentinelArrays", "SnoopPrecompile", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] -git-tree-sha1 = "c700cce799b51c9045473de751e9319bdd1c6e94" -uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" -version = "0.10.9" - -[[deps.CodecZlib]] -deps = ["TranscodingStreams", "Zlib_jll"] -git-tree-sha1 = "9c209fb7536406834aa938fb149964b985de6c83" -uuid = "944b1d66-785c-5afd-91f1-9de20f533193" -version = "0.7.1" - -[[deps.ColorTypes]] -deps = ["FixedPointNumbers", "Random"] -git-tree-sha1 = "eb7f0f8307f71fac7c606984ea5fb2817275d6e4" -uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" -version = "0.11.4" - -[[deps.Compat]] -deps = ["UUIDs"] -git-tree-sha1 = "7a60c856b9fa189eb34f5f8a6f6b5529b7942957" -uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" -version = "4.6.1" -weakdeps = ["Dates", "LinearAlgebra"] - - [deps.Compat.extensions] - CompatLinearAlgebraExt = "LinearAlgebra" - -[[deps.CompilerSupportLibraries_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" -version = "1.0.2+0" - -[[deps.ConcurrentUtilities]] -deps = ["Serialization", "Sockets"] -git-tree-sha1 = "b306df2650947e9eb100ec125ff8c65ca2053d30" -uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" -version = "2.1.1" - -[[deps.Conda]] -deps = ["Downloads", "JSON", "VersionParsing"] -git-tree-sha1 = "e32a90da027ca45d84678b826fffd3110bb3fc90" -uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d" -version = "1.8.0" - -[[deps.Crayons]] -git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" -uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" -version = "4.1.1" - -[[deps.DataAPI]] -git-tree-sha1 = "e8119c1a33d267e16108be441a287a6981ba1630" -uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" -version = "1.14.0" - -[[deps.DataFrames]] -deps = ["Compat", "DataAPI", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrettyTables", "Printf", "REPL", "Random", "Reexport", "SentinelArrays", "SnoopPrecompile", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] -git-tree-sha1 = "aa51303df86f8626a962fccb878430cdb0a97eee" -uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" -version = "1.5.0" - -[[deps.DataStructures]] -deps = ["Compat", "InteractiveUtils", "OrderedCollections"] -git-tree-sha1 = "d1fff3a548102f48987a52a2e0d114fa97d730f0" -uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" -version = "0.18.13" - -[[deps.DataValueInterfaces]] -git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" -uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" -version = "1.0.0" - -[[deps.Dataverse]] -deps = ["CSV", "Conda", "DataFrames", "Downloads", "HTTP", "JSON", "PyCall"] -git-tree-sha1 = "21fd97553d03212049286c74f3d63b0121cac6bb" -uuid = "9c0b9be8-e31e-490f-90fe-77697562404d" -version = "0.2.0" - -[[deps.Dates]] -deps = ["Printf"] -uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" - -[[deps.Downloads]] -deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] -uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" -version = "1.6.0" - -[[deps.FilePathsBase]] -deps = ["Compat", "Dates", "Mmap", "Printf", "Test", "UUIDs"] -git-tree-sha1 = "e27c4ebe80e8699540f2d6c805cc12203b614f12" -uuid = "48062228-2e41-5def-b9a4-89aafe57970f" -version = "0.9.20" - -[[deps.FileWatching]] -uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" - -[[deps.FixedPointNumbers]] -deps = ["Statistics"] -git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc" -uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" -version = "0.8.4" - -[[deps.Formatting]] -deps = ["Printf"] -git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8" -uuid = "59287772-0a20-5a39-b81b-1366585eb4c0" -version = "0.4.2" - -[[deps.Future]] -deps = ["Random"] -uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" - -[[deps.HTTP]] -deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] -git-tree-sha1 = "69182f9a2d6add3736b7a06ab6416aafdeec2196" -uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" -version = "1.8.0" - -[[deps.Hyperscript]] -deps = ["Test"] -git-tree-sha1 = "8d511d5b81240fc8e6802386302675bdf47737b9" -uuid = "47d2ed2b-36de-50cf-bf87-49c2cf4b8b91" -version = "0.0.4" - -[[deps.HypertextLiteral]] -deps = ["Tricks"] -git-tree-sha1 = "c47c5fa4c5308f27ccaac35504858d8914e102f9" -uuid = "ac1192a8-f4b3-4bfe-ba22-af5b92cd3ab2" -version = "0.9.4" - -[[deps.IOCapture]] -deps = ["Logging", "Random"] -git-tree-sha1 = "f7be53659ab06ddc986428d3a9dcc95f6fa6705a" -uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" -version = "0.2.2" - -[[deps.InlineStrings]] -deps = ["Parsers"] -git-tree-sha1 = "9cc2baf75c6d09f9da536ddf58eb2f29dedaf461" -uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" -version = "1.4.0" - -[[deps.InteractiveUtils]] -deps = ["Markdown"] -uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" - -[[deps.InvertedIndices]] -git-tree-sha1 = "0dc7b50b8d436461be01300fd8cd45aa0274b038" -uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" -version = "1.3.0" - -[[deps.IteratorInterfaceExtensions]] -git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" -uuid = "82899510-4779-5014-852e-03e436cf321d" -version = "1.0.0" - -[[deps.JLLWrappers]] -deps = ["Preferences"] -git-tree-sha1 = "abc9885a7ca2052a736a600f7fa66209f96506e1" -uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" -version = "1.4.1" - -[[deps.JSON]] -deps = ["Dates", "Mmap", "Parsers", "Unicode"] -git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" -uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" -version = "0.21.4" - -[[deps.LaTeXStrings]] -git-tree-sha1 = "f2355693d6778a178ade15952b7ac47a4ff97996" -uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" -version = "1.3.0" - -[[deps.LibCURL]] -deps = ["LibCURL_jll", "MozillaCACerts_jll"] -uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" -version = "0.6.3" - -[[deps.LibCURL_jll]] -deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] -uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" -version = "7.84.0+0" - -[[deps.LibGit2]] -deps = ["Base64", "NetworkOptions", "Printf", "SHA"] -uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" - -[[deps.LibSSH2_jll]] -deps = ["Artifacts", "Libdl", "MbedTLS_jll"] -uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" -version = "1.10.2+0" - -[[deps.Libdl]] -uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" - -[[deps.LinearAlgebra]] -deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] -uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" - -[[deps.Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[deps.LoggingExtras]] -deps = ["Dates", "Logging"] -git-tree-sha1 = "cedb76b37bc5a6c702ade66be44f831fa23c681e" -uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" -version = "1.0.0" - -[[deps.MIMEs]] -git-tree-sha1 = "65f28ad4b594aebe22157d6fac869786a255b7eb" -uuid = "6c6e2e6c-3030-632d-7369-2d6c69616d65" -version = "0.1.4" - -[[deps.MacroTools]] -deps = ["Markdown", "Random"] -git-tree-sha1 = "42324d08725e200c23d4dfb549e0d5d89dede2d2" -uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" -version = "0.5.10" - -[[deps.Markdown]] -deps = ["Base64"] -uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" - -[[deps.MbedTLS]] -deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "Random", "Sockets"] -git-tree-sha1 = "03a9b9718f5682ecb107ac9f7308991db4ce395b" -uuid = "739be429-bea8-5141-9913-cc70e7f3736d" -version = "1.1.7" - -[[deps.MbedTLS_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" -version = "2.28.2+0" - -[[deps.Missings]] -deps = ["DataAPI"] -git-tree-sha1 = "f66bdc5de519e8f8ae43bdc598782d35a25b1272" -uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" -version = "1.1.0" - -[[deps.Mmap]] -uuid = "a63ad114-7e13-5084-954f-fe012c677804" - -[[deps.MozillaCACerts_jll]] -uuid = "14a3606d-f60d-562e-9121-12d972cd8159" -version = "2022.10.11" - -[[deps.NetworkOptions]] -uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" -version = "1.2.0" - -[[deps.OpenBLAS_jll]] -deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] -uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" -version = "0.3.21+4" - -[[deps.OpenSSL]] -deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"] -git-tree-sha1 = "7fb975217aea8f1bb360cf1dde70bad2530622d2" -uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c" -version = "1.4.0" - -[[deps.OpenSSL_jll]] -deps = ["Artifacts", "JLLWrappers", "Libdl"] -git-tree-sha1 = "6cc6366a14dbe47e5fc8f3cbe2816b1185ef5fc4" -uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" -version = "3.0.8+0" - -[[deps.OrderedCollections]] -git-tree-sha1 = "d321bf2de576bf25ec4d3e4360faca399afca282" -uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" -version = "1.6.0" - -[[deps.Parsers]] -deps = ["Dates", "SnoopPrecompile"] -git-tree-sha1 = "478ac6c952fddd4399e71d4779797c538d0ff2bf" -uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" -version = "2.5.8" - -[[deps.Pkg]] -deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] -uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" -version = "1.9.0" - -[[deps.PlutoUI]] -deps = ["AbstractPlutoDingetjes", "Base64", "ColorTypes", "Dates", "FixedPointNumbers", "Hyperscript", "HypertextLiteral", "IOCapture", "InteractiveUtils", "JSON", "Logging", "MIMEs", "Markdown", "Random", "Reexport", "URIs", "UUIDs"] -git-tree-sha1 = "5bb5129fdd62a2bbbe17c2756932259acf467386" -uuid = "7f904dfe-b85e-4ff6-b463-dae2292396a8" -version = "0.7.50" - -[[deps.PooledArrays]] -deps = ["DataAPI", "Future"] -git-tree-sha1 = "a6062fe4063cdafe78f4a0a81cfffb89721b30e7" -uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" -version = "1.4.2" - -[[deps.Preferences]] -deps = ["TOML"] -git-tree-sha1 = "47e5f437cc0e7ef2ce8406ce1e7e24d44915f88d" -uuid = "21216c6a-2e73-6563-6e65-726566657250" -version = "1.3.0" - -[[deps.PrettyTables]] -deps = ["Crayons", "Formatting", "LaTeXStrings", "Markdown", "Reexport", "StringManipulation", "Tables"] -git-tree-sha1 = "548793c7859e28ef026dba514752275ee871169f" -uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" -version = "2.2.3" - -[[deps.Printf]] -deps = ["Unicode"] -uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" - -[[deps.PyCall]] -deps = ["Conda", "Dates", "Libdl", "LinearAlgebra", "MacroTools", "Serialization", "VersionParsing"] -git-tree-sha1 = "62f417f6ad727987c755549e9cd88c46578da562" -uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" -version = "1.95.1" - -[[deps.REPL]] -deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] -uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" - -[[deps.Random]] -deps = ["SHA", "Serialization"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[deps.Reexport]] -git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" -uuid = "189a3867-3050-52da-a836-e630ba90ab69" -version = "1.2.2" - -[[deps.SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" - -[[deps.SentinelArrays]] -deps = ["Dates", "Random"] -git-tree-sha1 = "77d3c4726515dca71f6d80fbb5e251088defe305" -uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" -version = "1.3.18" - -[[deps.Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[deps.SimpleBufferStream]] -git-tree-sha1 = "874e8867b33a00e784c8a7e4b60afe9e037b74e1" -uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" -version = "1.1.0" - -[[deps.SnoopPrecompile]] -deps = ["Preferences"] -git-tree-sha1 = "e760a70afdcd461cf01a575947738d359234665c" -uuid = "66db9d55-30c0-4569-8b51-7e840670fc0c" -version = "1.0.3" - -[[deps.Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" - -[[deps.SortingAlgorithms]] -deps = ["DataStructures"] -git-tree-sha1 = "a4ada03f999bd01b3a25dcaa30b2d929fe537e00" -uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" -version = "1.1.0" - -[[deps.SparseArrays]] -deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] -uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" - -[[deps.Statistics]] -deps = ["LinearAlgebra", "SparseArrays"] -uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" -version = "1.9.0" - -[[deps.StringManipulation]] -git-tree-sha1 = "46da2434b41f41ac3594ee9816ce5541c6096123" -uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" -version = "0.3.0" - -[[deps.SuiteSparse_jll]] -deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] -uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" -version = "5.10.1+6" - -[[deps.TOML]] -deps = ["Dates"] -uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" -version = "1.0.3" - -[[deps.TableTraits]] -deps = ["IteratorInterfaceExtensions"] -git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" -uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" -version = "1.0.1" - -[[deps.Tables]] -deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits", "Test"] -git-tree-sha1 = "1544b926975372da01227b382066ab70e574a3ec" -uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" -version = "1.10.1" - -[[deps.Tar]] -deps = ["ArgTools", "SHA"] -uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" -version = "1.10.0" - -[[deps.Test]] -deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] -uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" - -[[deps.TranscodingStreams]] -deps = ["Random", "Test"] -git-tree-sha1 = "0b829474fed270a4b0ab07117dce9b9a2fa7581a" -uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" -version = "0.9.12" - -[[deps.Tricks]] -git-tree-sha1 = "aadb748be58b492045b4f56166b5188aa63ce549" -uuid = "410a4b4d-49e4-4fbc-ab6d-cb71b17b3775" -version = "0.1.7" - -[[deps.URIs]] -git-tree-sha1 = "074f993b0ca030848b897beff716d93aca60f06a" -uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" -version = "1.4.2" - -[[deps.UUIDs]] -deps = ["Random", "SHA"] -uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" - -[[deps.Unicode]] -uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" - -[[deps.VersionParsing]] -git-tree-sha1 = "58d6e80b4ee071f5efd07fda82cb9fbe17200868" -uuid = "81def892-9a0e-5fdd-b105-ffc91e053289" -version = "1.3.0" - -[[deps.WeakRefStrings]] -deps = ["DataAPI", "InlineStrings", "Parsers"] -git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23" -uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" -version = "1.4.2" - -[[deps.WorkerUtilities]] -git-tree-sha1 = "cd1659ba0d57b71a464a29e64dbc67cfe83d54e7" -uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60" -version = "1.6.1" - -[[deps.Zlib_jll]] -deps = ["Libdl"] -uuid = "83775a58-1f1d-513f-b197-d71354ab007a" -version = "1.2.13+0" - -[[deps.libblastrampoline_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" -version = "5.4.0+0" - -[[deps.nghttp2_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" -version = "1.48.0+0" - -[[deps.p7zip_jll]] -deps = ["Artifacts", "Libdl"] -uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" -version = "17.4.0+0" -""" - # ╔═╡ Cell order: # ╟─52c81a38-0b73-4585-8833-40a834c12312 # ╟─bff0a085-9636-4b42-a3bb-67ec705529a9 @@ -674,5 +167,3 @@ version = "17.4.0+0" # ╟─e448e0ce-4991-4a75-b611-570aa64439f3 # ╟─8e7742bc-ed2e-11ec-2bbe-adbcf21330e7 # ╟─d216f95e-fc57-4b89-b796-be08b3f137d2 -# ╟─00000000-0000-0000-0000-000000000001 -# ╟─00000000-0000-0000-0000-000000000002 From 02ec145469b68d1d0cb7d5f65a84e09b1dd96650 Mon Sep 17 00:00:00 2001 From: gaelforget Date: Tue, 5 Sep 2023 10:51:39 -0400 Subject: [PATCH 2/5] update notebook --- .gitignore | 2 + docs/src/notebook.jl | 602 ++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 567 insertions(+), 37 deletions(-) diff --git a/.gitignore b/.gitignore index 685b89d..193d4d1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +dev/ +.ipynb_checkpoints/ *.jl.*.cov *.jl.cov *.jl.mem diff --git a/docs/src/notebook.jl b/docs/src/notebook.jl index 9e1b854..9f070cf 100644 --- a/docs/src/notebook.jl +++ b/docs/src/notebook.jl @@ -16,7 +16,6 @@ end # ╔═╡ 8e7742bc-ed2e-11ec-2bbe-adbcf21330e7 begin - using Pkg; Pkg.activate() using Dataverse, PlutoUI, Downloads using DataFrames, PrettyTables "Packages ready" @@ -78,23 +77,61 @@ let end +# ╔═╡ dc40c42c-6d5a-4110-a8b5-7b00568c4312 +begin + nfi=length(df.filename) + file_select = @bind file Select(df.filename) + ii=findall(df.filename.==file) + md"""## Select & Download File + + Select file : $(file_select) + """ +end + +# ╔═╡ e34e0d5b-7e43-4acd-9cf4-5916cd1b5404 +md""" Download file ? $(@bind dl Select([true,false],default=false))""" + +# ╔═╡ ffe06b1c-0e0e-44df-93d5-c33d88bf7e7e +begin + file_dl=joinpath(tempdir(),file) + dl&&!isfile(file_dl) ? Dataverse.file_download(DOI,file) : nothing +end + +# ╔═╡ 1a5a27c9-7f72-49b7-a406-2576c11f0f3c +begin + filePATH=joinpath(tempdir(),file) + if isfile(filePATH) + stat(filePATH) + else + println("file not found") + end +end + + # ╔═╡ 061ea815-66ef-4e06-987f-fbd1f7e837d7 md"""## Explore a Dataverse A Dataverse may contain multiple Datasets. Here each index of `tree` is one Dataset. Below we select one of them. -""" -# ╔═╡ e34e0d5b-7e43-4acd-9cf4-5916cd1b5404 -md""" Download file ? $(@bind dl Select([true,false],default=true))""" +!!! note + This feature is only available from `NativeApi` currently. +""" # ╔═╡ e448e0ce-4991-4a75-b611-570aa64439f3 md"""## Appendix""" +# ╔═╡ fe3a6735-00a8-4640-8f4b-f937720b7752 +md"""#### Other APIs + +!!! note + `DataAccessApi` and `NativeApi` are obtained via `pyDataverse` +""" + # ╔═╡ d216f95e-fc57-4b89-b796-be08b3f137d2 begin (DataAccessApi,NativeApi)=pyDataverse.APIs(do_install=true) 🏁=true - "pyDataverse APIs are ready to use" + "DataAccessApi,NativeApi are ready to use" end @@ -105,9 +142,6 @@ begin "Done scanning dataverse files" end -# ╔═╡ c3c4448f-63bb-481f-a28a-dc4ad0311519 -tree - # ╔═╡ ecfc53da-38b4-478c-8559-41746d3cd8d0 begin nf=length(tree) @@ -115,6 +149,9 @@ begin md"""Select a dataset : $(num_select)""" end +# ╔═╡ 98222492-2a6c-4f54-a40d-6f16df747991 +tree + # ╔═╡ 135e1777-6a29-4736-9b6d-60f88dbd405a let with_terminal() do @@ -123,31 +160,519 @@ let end end -# ╔═╡ dc40c42c-6d5a-4110-a8b5-7b00568c4312 -begin - nfi=length(tree[num].filename) - file_select = @bind file Select(tree[num].filename) - ii=findall(tree[num].filename.==file) - md"""## Select & Download File - - Select file : $(file_select) - """ -end - -# ╔═╡ ffe06b1c-0e0e-44df-93d5-c33d88bf7e7e -begin - dl ? Dataverse.file_download(DOI,file) : nothing -end +# ╔═╡ 00000000-0000-0000-0000-000000000001 +PLUTO_PROJECT_TOML_CONTENTS = """ +[deps] +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +Dataverse = "9c0b9be8-e31e-490f-90fe-77697562404d" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +PlutoUI = "7f904dfe-b85e-4ff6-b463-dae2292396a8" +PrettyTables = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" + +[compat] +DataFrames = "~1.6.1" +Dataverse = "~0.2.1" +PlutoUI = "~0.7.52" +PrettyTables = "~2.2.7" +""" -# ╔═╡ 1a5a27c9-7f72-49b7-a406-2576c11f0f3c -begin - filePATH=joinpath(tempdir(),file) - if isfile(filePATH) - stat(filePATH) - else - println("file not found") - end -end +# ╔═╡ 00000000-0000-0000-0000-000000000002 +PLUTO_MANIFEST_TOML_CONTENTS = """ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.9.3" +manifest_format = "2.0" +project_hash = "c49cabf7d2b2146df2031acb82ec24ebea9b6bec" + +[[deps.AbstractPlutoDingetjes]] +deps = ["Pkg"] +git-tree-sha1 = "91bd53c39b9cbfb5ef4b015e8b582d344532bd0a" +uuid = "6e696c72-6542-2067-7265-42206c756150" +version = "1.2.0" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[deps.BitFlags]] +git-tree-sha1 = "43b1a4a8f797c1cddadf60499a8a077d4af2cd2d" +uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" +version = "0.1.7" + +[[deps.CSV]] +deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] +git-tree-sha1 = "44dbf560808d49041989b8a96cae4cffbeb7966a" +uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +version = "0.10.11" + +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "02aa26a4cf76381be7f66e020a3eddeb27b0a092" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.2" + +[[deps.ColorTypes]] +deps = ["FixedPointNumbers", "Random"] +git-tree-sha1 = "eb7f0f8307f71fac7c606984ea5fb2817275d6e4" +uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" +version = "0.11.4" + +[[deps.Compat]] +deps = ["UUIDs"] +git-tree-sha1 = "e460f044ca8b99be31d35fe54fc33a5c33dd8ed7" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.9.0" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.0.5+0" + +[[deps.ConcurrentUtilities]] +deps = ["Serialization", "Sockets"] +git-tree-sha1 = "5372dbbf8f0bdb8c700db5367132925c0771ef7e" +uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" +version = "2.2.1" + +[[deps.Conda]] +deps = ["Downloads", "JSON", "VersionParsing"] +git-tree-sha1 = "8c86e48c0db1564a1d49548d3515ced5d604c408" +uuid = "8f4d0f93-b110-5947-807f-2305c1781a2d" +version = "1.9.1" + +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.DataAPI]] +git-tree-sha1 = "8da84edb865b0b5b0100c0666a9bc9a0b71c553c" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.15.0" + +[[deps.DataFrames]] +deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "REPL", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] +git-tree-sha1 = "04c738083f29f86e62c8afc341f0967d8717bdb8" +uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +version = "1.6.1" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "3dbd312d370723b6bb43ba9d02fc36abade4518d" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.15" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.Dataverse]] +deps = ["CSV", "Conda", "DataFrames", "Downloads", "HTTP", "JSON", "PyCall"] +git-tree-sha1 = "c3a709a2e010f8607719d272c948e0a90ca7062b" +uuid = "9c0b9be8-e31e-490f-90fe-77697562404d" +version = "0.2.1" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.ExceptionUnwrapping]] +deps = ["Test"] +git-tree-sha1 = "e90caa41f5a86296e014e148ee061bd6c3edec96" +uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4" +version = "0.1.9" + +[[deps.FilePathsBase]] +deps = ["Compat", "Dates", "Mmap", "Printf", "Test", "UUIDs"] +git-tree-sha1 = "e27c4ebe80e8699540f2d6c805cc12203b614f12" +uuid = "48062228-2e41-5def-b9a4-89aafe57970f" +version = "0.9.20" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + +[[deps.FixedPointNumbers]] +deps = ["Statistics"] +git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc" +uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" +version = "0.8.4" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" + +[[deps.HTTP]] +deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] +git-tree-sha1 = "cb56ccdd481c0dd7f975ad2b3b62d9eda088f7e2" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "1.9.14" + +[[deps.Hyperscript]] +deps = ["Test"] +git-tree-sha1 = "8d511d5b81240fc8e6802386302675bdf47737b9" +uuid = "47d2ed2b-36de-50cf-bf87-49c2cf4b8b91" +version = "0.0.4" + +[[deps.HypertextLiteral]] +deps = ["Tricks"] +git-tree-sha1 = "c47c5fa4c5308f27ccaac35504858d8914e102f9" +uuid = "ac1192a8-f4b3-4bfe-ba22-af5b92cd3ab2" +version = "0.9.4" + +[[deps.IOCapture]] +deps = ["Logging", "Random"] +git-tree-sha1 = "d75853a0bdbfb1ac815478bacd89cd27b550ace6" +uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" +version = "0.2.3" + +[[deps.InlineStrings]] +deps = ["Parsers"] +git-tree-sha1 = "9cc2baf75c6d09f9da536ddf58eb2f29dedaf461" +uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" +version = "1.4.0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.InvertedIndices]] +git-tree-sha1 = "0dc7b50b8d436461be01300fd8cd45aa0274b038" +uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" +version = "1.3.0" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "7e5d6779a1e09a36db2a7b6cff50942a0a7d0fca" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.5.0" + +[[deps.JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.4" + +[[deps.LaTeXStrings]] +git-tree-sha1 = "f2355693d6778a178ade15952b7ac47a4ff97996" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.3.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.3" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "7.84.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.10.2+0" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.LoggingExtras]] +deps = ["Dates", "Logging"] +git-tree-sha1 = "0d097476b6c381ab7906460ef1ef1638fbce1d91" +uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" +version = "1.0.2" + +[[deps.MIMEs]] +git-tree-sha1 = "65f28ad4b594aebe22157d6fac869786a255b7eb" +uuid = "6c6e2e6c-3030-632d-7369-2d6c69616d65" +version = "0.1.4" + +[[deps.MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "9ee1618cbf5240e6d4e0371d6f24065083f60c48" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.11" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.MbedTLS]] +deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "Random", "Sockets"] +git-tree-sha1 = "03a9b9718f5682ecb107ac9f7308991db4ce395b" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "1.1.7" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+0" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "f66bdc5de519e8f8ae43bdc598782d35a25b1272" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.1.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2022.10.11" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.21+4" + +[[deps.OpenSSL]] +deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"] +git-tree-sha1 = "51901a49222b09e3743c65b8847687ae5fc78eb2" +uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c" +version = "1.4.1" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "e78db7bd5c26fc5a6911b50a47ee302219157ea8" +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.0.10+0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "2e73fe17cac3c62ad1aebe70d44c963c3cfdc3e3" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.2" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "716e24b21538abc91f6205fd1d8363f39b442851" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.7.2" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.9.2" + +[[deps.PlutoUI]] +deps = ["AbstractPlutoDingetjes", "Base64", "ColorTypes", "Dates", "FixedPointNumbers", "Hyperscript", "HypertextLiteral", "IOCapture", "InteractiveUtils", "JSON", "Logging", "MIMEs", "Markdown", "Random", "Reexport", "URIs", "UUIDs"] +git-tree-sha1 = "e47cd150dbe0443c3a3651bc5b9cbd5576ab75b7" +uuid = "7f904dfe-b85e-4ff6-b463-dae2292396a8" +version = "0.7.52" + +[[deps.PooledArrays]] +deps = ["DataAPI", "Future"] +git-tree-sha1 = "a6062fe4063cdafe78f4a0a81cfffb89721b30e7" +uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" +version = "1.4.2" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "03b4c25b43cb84cee5c90aa9b5ea0a78fd848d2f" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.2.0" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "7eb1686b4f04b82f96ed7a4ea5890a4f0c7a09f1" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.0" + +[[deps.PrettyTables]] +deps = ["Crayons", "LaTeXStrings", "Markdown", "Printf", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "ee094908d720185ddbdc58dbe0c1cbe35453ec7a" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "2.2.7" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.PyCall]] +deps = ["Conda", "Dates", "Libdl", "LinearAlgebra", "MacroTools", "Serialization", "VersionParsing"] +git-tree-sha1 = "43d304ac6f0354755f1d60730ece8c499980f7ba" +uuid = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" +version = "1.96.1" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Random]] +deps = ["SHA", "Serialization"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.SentinelArrays]] +deps = ["Dates", "Random"] +git-tree-sha1 = "04bdff0b09c65ff3e06a05e3eb7b120223da3d39" +uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" +version = "1.4.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.SimpleBufferStream]] +git-tree-sha1 = "874e8867b33a00e784c8a7e4b60afe9e037b74e1" +uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" +version = "1.1.0" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "c60ec5c62180f27efea3ba2908480f8055e17cee" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.1.1" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" + +[[deps.Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.9.0" + +[[deps.StringManipulation]] +git-tree-sha1 = "46da2434b41f41ac3594ee9816ce5541c6096123" +uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" +version = "0.3.0" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "Pkg", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "5.10.1+6" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits", "Test"] +git-tree-sha1 = "1544b926975372da01227b382066ab70e574a3ec" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.10.1" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.TranscodingStreams]] +deps = ["Random", "Test"] +git-tree-sha1 = "9a6ae7ed916312b41236fcef7e0af564ef934769" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.9.13" + +[[deps.Tricks]] +git-tree-sha1 = "aadb748be58b492045b4f56166b5188aa63ce549" +uuid = "410a4b4d-49e4-4fbc-ab6d-cb71b17b3775" +version = "0.1.7" + +[[deps.URIs]] +git-tree-sha1 = "b7a5e99f24892b6824a954199a45e9ffcc1c70f0" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.5.0" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[deps.VersionParsing]] +git-tree-sha1 = "58d6e80b4ee071f5efd07fda82cb9fbe17200868" +uuid = "81def892-9a0e-5fdd-b105-ffc91e053289" +version = "1.3.0" + +[[deps.WeakRefStrings]] +deps = ["DataAPI", "InlineStrings", "Parsers"] +git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23" +uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" +version = "1.4.2" + +[[deps.WorkerUtilities]] +git-tree-sha1 = "cd1659ba0d57b71a464a29e64dbc67cfe83d54e7" +uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60" +version = "1.6.1" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+0" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.8.0+0" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.48.0+0" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+0" +""" # ╔═╡ Cell order: # ╟─52c81a38-0b73-4585-8833-40a834c12312 @@ -155,15 +680,18 @@ end # ╟─e90fc026-b134-4694-87db-b5fc8fd1dcb2 # ╟─eb402ef0-96fb-41d0-93e4-7fe5a69e8465 # ╟─2240395b-4112-4e8a-a233-26fde9dffbc4 -# ╟─061ea815-66ef-4e06-987f-fbd1f7e837d7 -# ╟─756bec67-c27e-49c1-9573-9521746cf856 -# ╟─c3c4448f-63bb-481f-a28a-dc4ad0311519 -# ╟─ecfc53da-38b4-478c-8559-41746d3cd8d0 -# ╟─135e1777-6a29-4736-9b6d-60f88dbd405a # ╟─dc40c42c-6d5a-4110-a8b5-7b00568c4312 # ╟─e34e0d5b-7e43-4acd-9cf4-5916cd1b5404 # ╟─ffe06b1c-0e0e-44df-93d5-c33d88bf7e7e # ╟─1a5a27c9-7f72-49b7-a406-2576c11f0f3c +# ╟─061ea815-66ef-4e06-987f-fbd1f7e837d7 +# ╟─756bec67-c27e-49c1-9573-9521746cf856 +# ╟─ecfc53da-38b4-478c-8559-41746d3cd8d0 +# ╟─98222492-2a6c-4f54-a40d-6f16df747991 +# ╟─135e1777-6a29-4736-9b6d-60f88dbd405a # ╟─e448e0ce-4991-4a75-b611-570aa64439f3 # ╟─8e7742bc-ed2e-11ec-2bbe-adbcf21330e7 +# ╟─fe3a6735-00a8-4640-8f4b-f937720b7752 # ╟─d216f95e-fc57-4b89-b796-be08b3f137d2 +# ╟─00000000-0000-0000-0000-000000000001 +# ╟─00000000-0000-0000-0000-000000000002 From d3be365d8dc6c4d178e89367f499f49de27bef85 Mon Sep 17 00:00:00 2001 From: gaelforget Date: Tue, 5 Sep 2023 11:57:48 -0400 Subject: [PATCH 3/5] add dataverse_scan + fix for file_download --- src/Dataverse.jl | 2 +- src/downloads.jl | 8 ++++++-- src/restDataverse.jl | 46 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 52 insertions(+), 4 deletions(-) diff --git a/src/Dataverse.jl b/src/Dataverse.jl index 80735b9..6b2198c 100644 --- a/src/Dataverse.jl +++ b/src/Dataverse.jl @@ -1,7 +1,7 @@ module Dataverse include("restDataverse.jl") -import Dataverse.restDataverse: file_list +import Dataverse.restDataverse: file_list, dataverse_scan include("pyDataverse.jl") export pyDataverse diff --git a/src/downloads.jl b/src/downloads.jl index dc225ac..7013f01 100644 --- a/src/downloads.jl +++ b/src/downloads.jl @@ -32,14 +32,18 @@ Dataverse.file_download(lst,lst.name[2],tempdir()) function file_download(lists::NamedTuple,nam::String,pth=tempdir()) ii = findall([occursin("$nam", lists.name[i]) for i=1:length(lists.ID)]) for i in ii - nam1=Downloads.download(lists.URL[i]) if length(ii)>1 !isdir(joinpath(pth,nam)) ? mkdir(joinpath(pth,nam)) : nothing nam2=joinpath(pth,nam,lists.name[i]) - mv(nam1,nam2) else nam2=joinpath(pth,lists.name[i]) + end + if !isfile(nam2) + nam1=Downloads.download(lists.URL[i]) mv(nam1,nam2) + println("Downloading file : $(nam2)") + else + println("Skipping file that's already there : $(nam2)") end end end diff --git a/src/restDataverse.jl b/src/restDataverse.jl index e6fde6c..5a786bb 100644 --- a/src/restDataverse.jl +++ b/src/restDataverse.jl @@ -31,6 +31,48 @@ function file_list(nam::Symbol) file_list(DOI[nam]) end +""" + dataverse_scan(nam::Symbol=:ECCOv4r2) + +Use HTTP, JSON, and DataFrames to list contents in a dataverse. + +Returns header (Dict), dataverses (DataFrame), and datasets (DataFrame). + +``` +(header,dataverses,datasets)=Dataverse.dataverse_scan() +Dataverse.file_list(datasets[1,:persistentUrl]) +``` +""" +function dataverse_scan(nam::Symbol=:ECCOv4r2) + #header + r=HTTP.get("https://dataverse.harvard.edu/api/dataverses/$(nam)") + header=JSON.parse(String(r.body))["data"] + #contents + r=HTTP.get("https://dataverse.harvard.edu/api/dataverses/$(nam)/contents") + tmp=JSON.parse(String(r.body))["data"] + type=[f["type"] for f in tmp] + #1. dataverses + ii=findall(type.=="dataverse") + if !isempty(ii) + id=[f["id"] for f in tmp[ii]] + title=[f["title"] for f in tmp[ii]] + dataverses=DataFrame(id=id,type=type[ii],title=title) + else + dataverses=DataFrame(id=[],type=[],title=[]) + end + #2. datasets + ii=findall(type.=="dataset") + if !isempty(ii) + id=[f["id"] for f in tmp[ii]] + persistentUrl=[f["persistentUrl"] for f in tmp[ii]] + datasets=DataFrame(id=id,type=type[ii],persistentUrl=persistentUrl) + else + dataverses=DataFrame(id=[],type=[],persistentUrl=[]) + end + # + return header,dataverses,datasets +end + """ files_to_DataFrame(files) @@ -41,7 +83,9 @@ function files_to_DataFrame(files) filename=[files[ff]["dataFile"]["filename"] for ff in 1:nf] filesize=[files[ff]["dataFile"]["filesize"] for ff in 1:nf] id=[files[ff]["dataFile"]["id"] for ff in 1:nf] - DataFrame(filename=filename,filesize=filesize,id=id) + tmp="https://dataverse.harvard.edu/api/access/datafile/" + url=[tmp*"$(id[nf])" for ff in 1:nf] + DataFrame(filename=filename,filesize=filesize,id=id,url=url) end end From 6892fd254a212eeac01c1ce98b301880af5e260f Mon Sep 17 00:00:00 2001 From: gaelforget Date: Tue, 5 Sep 2023 12:11:36 -0400 Subject: [PATCH 4/5] rm url_list, add url in DataFrame instead --- docs/src/index.md | 2 +- src/Dataverse.jl | 2 +- src/downloads.jl | 33 ++++++++++----------------------- test/runtests.jl | 4 ++-- 4 files changed, 14 insertions(+), 27 deletions(-) diff --git a/docs/src/index.md b/docs/src/index.md index 77f83b4..664da3c 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -15,7 +15,7 @@ This package is about interfaces to the [Dataverse](https://dataverse.org) data ```@docs file_list -url_list file_download +dataverse_scan ``` diff --git a/src/Dataverse.jl b/src/Dataverse.jl index 6b2198c..e7e5880 100644 --- a/src/Dataverse.jl +++ b/src/Dataverse.jl @@ -7,6 +7,6 @@ include("pyDataverse.jl") export pyDataverse include("downloads.jl") -import Dataverse.downloads: file_download, url_list +import Dataverse.downloads: file_download end diff --git a/src/downloads.jl b/src/downloads.jl index 7013f01..cef543f 100644 --- a/src/downloads.jl +++ b/src/downloads.jl @@ -16,30 +16,28 @@ Dataverse.file_download(DOI,filename) """ function file_download(DOI::String,nam::String,pth=tempdir()) df=file_list(DOI) - lst=url_list(df) file_download(lst,nam,pth) end """ - file_download(lst::NamedTuple,nam::String,pth::String) + file_download(list::DataFrame,nam::String,pth::String) ``` -lst0=Dataverse.file_list("doi:10.7910/DVN/RNXA2A") -lst=Dataverse.url_list(lst0) -Dataverse.file_download(lst,lst.name[2],tempdir()) +lst=Dataverse.file_list("doi:10.7910/DVN/RNXA2A") +Dataverse.file_download(lst,lst.filename[2],tempdir()) ``` """ -function file_download(lists::NamedTuple,nam::String,pth=tempdir()) - ii = findall([occursin("$nam", lists.name[i]) for i=1:length(lists.ID)]) +function file_download(list::DataFrame,nam::String,pth=tempdir()) + ii = findall([occursin("$nam", list.filename[i]) for i=1:length(list.id)]) for i in ii if length(ii)>1 !isdir(joinpath(pth,nam)) ? mkdir(joinpath(pth,nam)) : nothing - nam2=joinpath(pth,nam,lists.name[i]) + nam2=joinpath(pth,nam,list.filename[i]) else - nam2=joinpath(pth,lists.name[i]) + nam2=joinpath(pth,list.filename[i]) end if !isfile(nam2) - nam1=Downloads.download(lists.URL[i]) + nam1=Downloads.download(list.url[i]) mv(nam1,nam2) println("Downloading file : $(nam2)") else @@ -48,20 +46,9 @@ function file_download(lists::NamedTuple,nam::String,pth=tempdir()) end end -""" - url_list(lst::DataFrame) - -Add download URL (using df.id) and return as `NamedTuple`. -""" -url_list(df::DataFrame) = begin - tmp="https://dataverse.harvard.edu/api/access/datafile/" - URL=[tmp*"$(df.id[j])" for j=1:length(df.id)] - (ID=df.id,name=df.filename,URL=URL) -end - ## -OCCA_file_list()=url_list(file_list(:OCCA_clim)) -ECCO_file_list()=url_list(file_list(:ECCO_clim)) +OCCA_file_list()=file_list(:OCCA_clim) +ECCO_file_list()=file_list(:ECCO_clim) end diff --git a/test/runtests.jl b/test/runtests.jl index 172ae82..c0ef2d4 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -8,8 +8,8 @@ pyDataverse.APIs(do_install=true) lst=Dataverse.downloads.OCCA_file_list() pth=joinpath(tempdir(),string(UUIDs.uuid4())) mkdir(pth) - Dataverse.file_download(lst,lst.name[1],pth) - @test isfile(joinpath(pth,lst.name[1])) + Dataverse.file_download(lst,lst.filename[1],pth) + @test isfile(joinpath(pth,lst.filename[1])) tmp=pyDataverse.demo("download") @test isfile(tmp[1]) From e7702162fc27b267b7d56c56f048b95b08ad5472 Mon Sep 17 00:00:00 2001 From: gaelforget Date: Tue, 5 Sep 2023 12:12:10 -0400 Subject: [PATCH 5/5] bump version --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 851a6d5..d830599 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "Dataverse" uuid = "9c0b9be8-e31e-490f-90fe-77697562404d" authors = ["gaelforget and contributors"] -version = "0.2.1" +version = "0.2.2" [deps] CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"