From fac4515ccdd46981524018fa16602c2bb5f47df1 Mon Sep 17 00:00:00 2001 From: David Widmann Date: Fri, 15 Jan 2021 16:44:09 +0000 Subject: [PATCH] Fixes for Julia 1.6 (#212) This PR fixes https://github.com/TuringLang/Turing.jl/issues/1525#issuecomment-760900398 and fixes the model definition and some warning on Julia 1.6. It also includes a line number node of the call site instead of a reference to `src/compiler.jl` in the model definition for easier debugging. On Julia 1.6, initially I got warnings about method redefinitions (models of the same name) that all referred to line 344 in `src/compiler.jl` instead of the place where the model was defined by the user. Turing tests are disabled currently since they crash (segmentation fault). Might be related to the fact that Libtask is not compatible with Julia 1.6 yet (therefore also an older version of Turing and Libtask would be used on Julia 1.6). Co-authored-by: David Widmann --- .../workflows/{DynamicPPL-CI.yml => CI.yml} | 8 ++-- .github/workflows/JuliaNightly.yml | 33 +++++++++++++++ Project.toml | 4 +- README.md | 5 ++- src/compiler.jl | 20 ++++++---- src/utils.jl | 30 +------------- test/compiler.jl | 40 +++++++++---------- test/runtests.jl | 26 ++++++------ test/utils.jl | 3 +- 9 files changed, 92 insertions(+), 77 deletions(-) rename .github/workflows/{DynamicPPL-CI.yml => CI.yml} (93%) create mode 100644 .github/workflows/JuliaNightly.yml diff --git a/.github/workflows/DynamicPPL-CI.yml b/.github/workflows/CI.yml similarity index 93% rename from .github/workflows/DynamicPPL-CI.yml rename to .github/workflows/CI.yml index c571fd972..292b2a7c8 100644 --- a/.github/workflows/DynamicPPL-CI.yml +++ b/.github/workflows/CI.yml @@ -1,4 +1,4 @@ -name: DynamicPPL-CI +name: CI on: push: @@ -13,13 +13,11 @@ on: jobs: test: runs-on: ${{ matrix.os }} - continue-on-error: ${{ matrix.version == 'nightly' }} strategy: matrix: version: - - '1.3' - - '1' - # - 'nightly' + - '1.3' # minimum supported version + - '1' # current stable version os: - ubuntu-latest - macOS-latest diff --git a/.github/workflows/JuliaNightly.yml b/.github/workflows/JuliaNightly.yml new file mode 100644 index 000000000..a7aa85194 --- /dev/null +++ b/.github/workflows/JuliaNightly.yml @@ -0,0 +1,33 @@ +name: JuliaNightly + +on: + push: + branches: + # This is where pull requests from "bors r+" are built. + - staging + # This is where pull requests from "bors try" are built. + - trying + # Build the master branch. + - master + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: julia-actions/setup-julia@v1 + with: + version: 'nightly' + arch: x64 + - uses: actions/cache@v1 + env: + cache-name: cache-artifacts + with: + path: ~/.julia/artifacts + key: ${{ runner.os }}-test-${{ env.cache-name }}-${{ hashFiles('**/Project.toml') }} + restore-keys: | + ${{ runner.os }}-test-${{ env.cache-name }}- + ${{ runner.os }}-test- + ${{ runner.os }}- + - uses: julia-actions/julia-buildpkg@latest + - uses: julia-actions/julia-runtest@latest diff --git a/Project.toml b/Project.toml index e60e32ad8..2c14cd316 100644 --- a/Project.toml +++ b/Project.toml @@ -1,6 +1,6 @@ name = "DynamicPPL" uuid = "366bfd00-2699-11ea-058f-f148b4cae6d8" -version = "0.10.5" +version = "0.10.6" [deps] AbstractMCMC = "80f14c24-f653-4e6a-9b94-39d6b0f70001" @@ -16,6 +16,6 @@ AbstractMCMC = "2" Bijectors = "0.5.2, 0.6, 0.7, 0.8" ChainRulesCore = "0.9.7" Distributions = "0.23.8, 0.24" -MacroTools = "0.5.1" +MacroTools = "0.5.6" NaturalSort = "1" julia = "1.3" diff --git a/README.md b/README.md index 64c97ad17..2172cf391 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ # DynamicPPL.jl -[![Build Status](https://travis-ci.com/TuringLang/DynamicPPL.jl.svg?branch=master)](https://travis-ci.com/TuringLang/DynamicPPL.jl) -[![Build Status](https://github.com/TuringLang/DynamicPPL.jl/workflows/DynamicPPL-CI/badge.svg)](https://github.com/TuringLang/DynamicPPL.jl/actions?query=workflow%3ADynamicPPL-CI+branch%3Amaster) +[![Build Status](https://github.com/TuringLang/DynamicPPL.jl/workflows/CI/badge.svg?branch=master)](https://github.com/TuringLang/DynamicPPL.jl/actions?query=workflow%3ACI+branch%3Amaster) +[![Build Status](https://github.com/TuringLang/DynamicPPL.jl/workflows/JuliaNightly/badge.svg?branch=master)](https://github.com/TuringLang/DynamicPPL.jl/actions?query=workflow%3AJuliaNightly+branch%3Amaster) +[![Coverage Status](https://coveralls.io/repos/github/TuringLang/DynamicPPL.jl/badge.svg?branch=master)](https://coveralls.io/github/TuringLang/DynamicPPL.jl?branch=master) [![Codecov](https://codecov.io/gh/TuringLang/DynamicPPL.jl/branch/master/graph/badge.svg)](https://codecov.io/gh/TuringLang/DynamicPPL.jl) [![ColPrac: Contributor's Guide on Collaborative Practices for Community Packages](https://img.shields.io/badge/ColPrac-Contributor's%20Guide-blueviolet)](https://colprac.sciml.ai/) [![Bors enabled](https://bors.tech/images/badge_small.svg)](https://app.bors.tech/repositories/24589) diff --git a/src/compiler.jl b/src/compiler.jl index 13ef0373b..73ae707c5 100644 --- a/src/compiler.jl +++ b/src/compiler.jl @@ -62,10 +62,12 @@ end To generate a `Model`, call `model(xvalue)` or `model(xvalue, yvalue)`. """ macro model(expr, warn=true) - esc(model(expr, warn)) + # include `LineNumberNode` with information about the call site in the + # generated function for easier debugging and interpretation of error messages + esc(model(expr, __source__, warn)) end -function model(expr, warn) +function model(expr, linenumbernode, warn) modelinfo = build_model_info(expr) # Generate main body @@ -73,7 +75,7 @@ function model(expr, warn) modelinfo[:modeldef][:body], modelinfo[:allargs_syms], warn ) - return build_output(modelinfo) + return build_output(modelinfo, linenumbernode) end """ @@ -301,11 +303,11 @@ hasmissing(T::Type{<:AbstractArray{>:Missing}}) = true hasmissing(T::Type) = false """ - build_output(modelinfo) + build_output(modelinfo, linenumbernode) Builds the output expression. """ -function build_output(modelinfo) +function build_output(modelinfo, linenumbernode) ## Build the anonymous evaluator from the user-provided model definition. # Remove the name. @@ -340,8 +342,12 @@ function build_output(modelinfo) # We use a name for the anonymous evaluator that does not conflict with other variables. modeldef = modelinfo[:modeldef] @gensym evaluator - modeldef[:body] = quote - $evaluator = $(combinedef_anonymous(evaluatordef)) + # We use `MacroTools.@q begin ... end` instead of regular `quote ... end` to ensure + # that no new `LineNumberNode`s are added apart from the reference `linenumbernode` + # to the call site + modeldef[:body] = MacroTools.@q begin + $(linenumbernode) + $evaluator = $(MacroTools.combinedef(evaluatordef)) return $(DynamicPPL.Model)( $(QuoteNode(modeldef[:name])), $evaluator, diff --git a/src/utils.jl b/src/utils.jl index 65373745e..a2d209fa2 100644 --- a/src/utils.jl +++ b/src/utils.jl @@ -2,34 +2,6 @@ struct NoDefault end const NO_DEFAULT = NoDefault() -# FIXME: This is copied from MacroTools and should be removed when a MacroTools release with -# support for anonymous functions is available (> 0.5.5). -function combinedef_anonymous(dict::Dict) - rtype = get(dict, :rtype, nothing) - params = get(dict, :params, []) - wparams = get(dict, :whereparams, []) - body = MacroTools.block(dict[:body]) - - if isempty(dict[:kwargs]) - arg = :($(dict[:args]...),) - else - arg = Expr(:tuple, Expr(:parameters, dict[:kwargs]...), dict[:args]...) - end - if isempty(wparams) - if rtype==nothing - MacroTools.@q($arg -> $body) - else - MacroTools.@q(($arg::$rtype) -> $body) - end - else - if rtype === nothing - MacroTools.@q(($arg where {$(wparams...)}) -> $body) - else - MacroTools.@q(($arg::$rtype where {$(wparams...)}) -> $body) - end - end -end - """ @addlogprob!(ex) @@ -52,6 +24,8 @@ function getargs_dottilde(expr::Expr) return MacroTools.@match expr begin (.~)(L_, R_) => (L, R) (~).(L_, R_) => (L, R) + # Julia 1.6: see https://github.com/TuringLang/Turing.jl/issues/1525 + (L_ .~ R_) => (L, R) x_ => nothing end end diff --git a/test/compiler.jl b/test/compiler.jl index a08601bf4..ee5e1e126 100644 --- a/test/compiler.jl +++ b/test/compiler.jl @@ -20,15 +20,15 @@ end testmodel_comp(1.0, 1.2) # check if drawing from the prior works - @model function testmodel0(x = missing) + @model function testmodel01(x = missing) x ~ Normal() return x end - f0_mm = testmodel0() + f0_mm = testmodel01() @test mean(f0_mm() for _ in 1:1000) ≈ 0. atol=0.1 # Test #544 - @model function testmodel0(x = missing) + @model function testmodel02(x = missing) if x === missing x = Vector{Float64}(undef, 2) end @@ -36,14 +36,14 @@ end x[2] ~ Normal() return x end - f0_mm = testmodel0() + f0_mm = testmodel02() @test all(x -> isapprox(x, 0; atol = 0.1), mean(f0_mm() for _ in 1:1000)) - @model function testmodel01(x = missing) + @model function testmodel03(x = missing) x ~ Bernoulli(0.5) return x end - f01_mm = testmodel01() + f01_mm = testmodel03() @test mean(f01_mm() for _ in 1:1000) ≈ 0.5 atol=0.1 # test if we get the correct return values @@ -133,22 +133,22 @@ end @test_throws ArgumentError btest() # Test missing input arguments - @model function testmodel(x) - x ~ Bernoulli(0.5) + @model function testmodel_missing1(x) + x ~ Bernoulli(0.5) return x end - @test_throws MethodError testmodel() + @test_throws MethodError testmodel_missing1() # Test missing initialization for vector observation turned parameter - @model function testmodel(x) - x[1] ~ Bernoulli(0.5) + @model function testmodel_missing2(x) + x[1] ~ Bernoulli(0.5) return x end - @test_throws MethodError testmodel(missing)() + @test_throws MethodError testmodel_missing2(missing)() # Test use of internal names - @model function testmodel(x) - x[1] ~  Bernoulli(0.5) + @model function testmodel_missing3(x) + x[1] ~ Bernoulli(0.5) global varinfo_ = _varinfo global sampler_ = _sampler global model_ = _model @@ -157,7 +157,7 @@ end global lp = getlogp(_varinfo) return x end - model = testmodel([1.0]) + model = testmodel_missing3([1.0]) varinfo = VarInfo(model) @test getlogp(varinfo) == lp @test varinfo_ isa AbstractVarInfo @@ -167,8 +167,8 @@ end @test rng_ isa Random.AbstractRNG # disable warnings - @model function testmodel(x) - x[1] ~  Bernoulli(0.5) + @model function testmodel_missing4(x) + x[1] ~ Bernoulli(0.5) global varinfo_ = _varinfo global sampler_ = _sampler global model_ = _model @@ -178,17 +178,17 @@ end return x end false lpold = lp - model = testmodel([1.0]) + model = testmodel_missing4([1.0]) varinfo = VarInfo(model) @test getlogp(varinfo) == lp == lpold # test DPPL#61 - @model function testmodel(z) + @model function testmodel_missing5(z) m ~ Normal() z[1:end] ~ MvNormal(fill(m, length(z)), 1.0) return m end - model = testmodel(rand(10)) + model = testmodel_missing5(rand(10)) @test all(z -> isapprox(z, 0; atol = 0.2), mean(model() for _ in 1:1000)) # test Turing#1464 diff --git a/test/runtests.jl b/test/runtests.jl index 0fec3a2ba..8ebc10868 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -43,18 +43,20 @@ include("test_util.jl") include(joinpath("compat", "ad.jl")) end - @testset "turing" begin - # activate separate test environment - Pkg.activate(DIRECTORY_Turing_tests) - Pkg.develop(PackageSpec(path=DIRECTORY_DynamicPPL)) - Pkg.instantiate() - - # make sure that the new environment is considered `using` and `import` statements - # (not added automatically on Julia 1.3, see e.g. PR #209) - if !(joinpath(DIRECTORY_Turing_tests, "Project.toml") in Base.load_path()) - pushfirst!(LOAD_PATH, DIRECTORY_Turing_tests) + @static if VERSION <= v"1.5.3" + @testset "turing" begin + # activate separate test environment + Pkg.activate(DIRECTORY_Turing_tests) + Pkg.develop(PackageSpec(path=DIRECTORY_DynamicPPL)) + Pkg.instantiate() + + # make sure that the new environment is considered `using` and `import` statements + # (not added automatically on Julia 1.3, see e.g. PR #209) + if !(joinpath(DIRECTORY_Turing_tests, "Project.toml") in Base.load_path()) + pushfirst!(LOAD_PATH, DIRECTORY_Turing_tests) + end + + include(joinpath("turing", "runtests.jl")) end - - include(joinpath("turing", "runtests.jl")) end end diff --git a/test/utils.jl b/test/utils.jl index 8fd6e2ddd..a4a1923ce 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -22,6 +22,7 @@ @test getargs_dottilde(:(x ~ Normal(μ, σ))) === nothing @test getargs_dottilde(:((.~)(x, Normal(μ, σ)))) == (:x, :(Normal(μ, σ))) @test getargs_dottilde(:((~).(x, Normal(μ, σ)))) == (:x, :(Normal(μ, σ))) + @test getargs_dottilde(:(x .~ Normal(μ, σ))) == (:x, :(Normal(μ, σ))) @test getargs_dottilde(:(@. x ~ Normal(μ, σ))) === nothing @test getargs_dottilde(:(@. x ~ Normal(μ, $(Expr(:$, :(sqrt(v))))))) === nothing @test getargs_dottilde(:(@~ Normal.(μ, σ))) === nothing @@ -41,4 +42,4 @@ @test getargs_tilde(:(@. x ~ Normal(μ, $(Expr(:$, :(sqrt(v))))))) === nothing @test getargs_tilde(:(@~ Normal.(μ, σ))) === nothing end -end \ No newline at end of file +end