Resurected Tapir code

base/boot.jl

@@ -155,6 +155,20 @@
 #    name::Symbol
 #end
 
+#struct DetachNode
+#    syncregion
+#    label::Int
+#end
+
+#struct ReattachNode
+#    syncregion
+#    label::Int
+#end
+
+#struct SyncNode
+#    syncregion
+#end
+
 #mutable struct Task
 #    parent::Task
 #    storage::Any
@@ -427,6 +441,9 @@ eval(Core, quote
     ReturnNode(@nospecialize val) = $(Expr(:new, :ReturnNode, :val))
     ReturnNode() = $(Expr(:new, :ReturnNode)) # unassigned val indicates unreachable
     GotoIfNot(@nospecialize(cond), dest::Int) = $(Expr(:new, :GotoIfNot, :cond, :dest))
+    SyncNode(token) = $(Expr(:new, :SyncNode, :token))
+    DetachNode(token, bb::Int) = $(Expr(:new, :DetachNode, :token, :bb,))
+    ReattachNode(token, bb::Int) = $(Expr(:new, :ReattachNode, :token, :bb))
     LineNumberNode(l::Int) = $(Expr(:new, :LineNumberNode, :l, nothing))
     function LineNumberNode(l::Int, @nospecialize(f))
         isa(f, String) && (f = Symbol(f))
@@ -525,11 +542,13 @@ module IR
 export CodeInfo, MethodInstance, CodeInstance, GotoNode, GotoIfNot, ReturnNode,
     NewvarNode, SSAValue, SlotNumber, Argument,
     PiNode, PhiNode, PhiCNode, UpsilonNode, LineInfoNode,
+    DetachNode, ReattachNode, SyncNode,
     Const, PartialStruct, InterConditional
 
 using Core: CodeInfo, MethodInstance, CodeInstance, GotoNode, GotoIfNot, ReturnNode,
     NewvarNode, SSAValue, SlotNumber, Argument,
     PiNode, PhiNode, PhiCNode, UpsilonNode, LineInfoNode,
+    DetachNode, ReattachNode, SyncNode,
     Const, PartialStruct, InterConditional
 
 end # module IR

base/compiler/abstractinterpretation.jl

@@ -3081,6 +3081,27 @@ function typeinf_local(interp::AbstractInterpreter, frame::InferenceState)
                         end
                         @goto fallthrough
                     end
+                elseif isa(stmt, DetachNode)
+                    # A detach node has two edges we need to add
+                    # the reattach edge to the work queue
+                    succs = bbs[currbb].succs
+                    @assert length(succs) == 2
+                    detachbb = succs[1] == currbb + 1 ? succs[2] : succs[1]
+                    if update_bbstate!(𝕃ᵢ, frame, detachbb, currstate)
+                        handle_control_backedge!(interp, frame, currpc, stmt.label)
+                        push!(W, detachbb)
+                    end
+                    ssavaluetypes[currpc] = Any
+                    add_curr_ssaflag!(frame, IR_FLAG_NOTHROW)
+                    @goto fallthrough
+                elseif isa(stmt, ReattachNode)
+                    succs = bbs[currbb].succs
+                    @assert length(succs) == 1
+                    nextbb = succs[1]
+                    ssavaluetypes[currpc] = Any
+                    handle_control_backedge!(interp, frame, currpc, stmt.label)
+                    add_curr_ssaflag!(frame, IR_FLAG_NOTHROW)
+                    @goto branch
                 elseif isa(stmt, ReturnNode)
                     rt = abstract_eval_value(interp, stmt.val, currstate, frame)
                     if update_bestguess!(interp, frame, currstate, rt)

base/compiler/optimize.jl

@@ -287,6 +287,9 @@ function stmt_effect_flags(𝕃ₒ::AbstractLattice, @nospecialize(stmt), @nospe
     isa(stmt, ReturnNode) && return (true, false, true)
     isa(stmt, GotoNode) && return (true, false, true)
     isa(stmt, GotoIfNot) && return (true, false, ⊑(𝕃ₒ, argextype(stmt.cond, src), Bool))
+    isa(stmt, SyncNode) && return (true, false, true)
+    isa(stmt, DetachNode) && return (true, false, true)
+    isa(stmt, ReattachNode) && return (true, false, true)
     if isa(stmt, GlobalRef)
         nothrow = isdefined(stmt.mod, stmt.name)
         consistent = nothrow && isconst(stmt.mod, stmt.name)
@@ -1185,6 +1188,39 @@ function renumber_ir_elements!(body::Vector{Any}, ssachangemap::Vector{Int}, lab
                     body[i] = ReturnNode(SSAValue(val.id + ssachangemap[val.id]))
                 end
             end
+        elseif isa(el, DetachNode)
+            if labelchangemap[el.label] == typemin(Int)
+                body[i] = nothing
+            else
+                label = el.label + labelchangemap[el.label]
+                syncregion = el.syncregion
+                if isa(syncregion, SSAValue)
+                    syncregion = SSAValue(syncregion.id + ssachangemap[syncregion.id])
+                    body[i] = DetachNode(syncregion, label)
+                else
+                    body[i] = DetachNode(syncregion, label)
+                end
+            end
+        elseif isa(el, ReattachNode)
+            if labelchangemap[el.label] == typemin(Int)
+                body[i] = nothing
+                # TODO: Make this always valid by using fallthrough in ReattachNode
+            else
+                label = el.label + labelchangemap[el.label]
+                syncregion = el.syncregion
+                if isa(syncregion, SSAValue)
+                    syncregion = SSAValue(syncregion.id + ssachangemap[syncregion.id])
+                    body[i] = ReattachNode(syncregion, label)
+                else
+                    body[i] = ReattachNode(syncregion, label)
+                end
+            end
+        elseif isa(el, SyncNode)
+            syncregion = el.syncregion
+            if isa(syncregion, SSAValue)
+                syncregion = SSAValue(syncregion.id + ssachangemap[syncregion.id])
+            end
+            body[i] = SyncNode(syncregion)
         elseif isa(el, SSAValue)
             body[i] = SSAValue(el.id + ssachangemap[el.id])
         elseif isa(el, PhiNode)

base/compiler/ssair/inlining.jl

@@ -461,6 +461,10 @@ function ir_inline_item!(compact::IncrementalCompact, idx::Int, argexprs::Vector
                 end
             elseif isa(stmt′, GotoNode)
                 stmt′ = GotoNode(stmt′.label + bb_offset)
+            elseif isa(stmt′, DetachNode)
+                stmt′ = DetachNode(stmt′.syncregion, stmt′.label + bb_offset)
+            elseif isa(stmt′, ReattachNode)
+                stmt′ = ReattachNode(stmt′.syncregion, stmt′.label + bb_offset)
             elseif isa(stmt′, Expr) && stmt′.head === :enter
                 stmt′ = Expr(:enter, stmt′.args[1]::Int + bb_offset)
             elseif isa(stmt′, GotoIfNot)
@@ -715,6 +719,10 @@ function batch_inline!(ir::IRCode, todo::Vector{Pair{Int,Any}}, propagate_inboun
                 end
             elseif isa(stmt, GotoNode)
                 compact[idx] = GotoNode(state.bb_rename[stmt.label])
+            elseif isa(stmt, DetachNode)
+                compact[idx] = DetachNode(stmt.syncregion, state.bb_rename[stmt.label])
+            elseif isa(stmt, ReattachNode)
+                compact[idx] = ReattachNode(stmt.syncregion, state.bb_rename[stmt.label])
             elseif isa(stmt, Expr) && stmt.head === :enter
                 compact[idx] = Expr(:enter, state.bb_rename[stmt.args[1]::Int])
             elseif isa(stmt, GotoIfNot)

base/compiler/ssair/ir.jl

@@ -2,7 +2,13 @@
 
 Core.PhiNode() = Core.PhiNode(Int32[], Any[])
 
-isterminator(@nospecialize(stmt)) = isa(stmt, GotoNode) || isa(stmt, GotoIfNot) || isa(stmt, ReturnNode)
+isterminator(@nospecialize(stmt)) =
+    isa(stmt, GotoNode) ||
+    isa(stmt, GotoIfNot) ||
+    isa(stmt, ReturnNode) ||
+    isa(stmt, DetachNode) ||
+    isa(stmt, ReattachNode) ||
+    isa(stmt, SyncNode)
 
 struct CFG
     blocks::Vector{BasicBlock}
@@ -60,6 +66,13 @@ block_for_inst(cfg::CFG, inst::Int) = block_for_inst(cfg.index, inst)
             # This is a fake dest to force the next stmt to start a bb
             idx < length(stmts) && push!(jump_dests, idx+1)
             push!(jump_dests, stmt.label)
+        elseif isa(stmt, DetachNode)
+            push!(jump_dests, idx+1)
+            push!(jump_dests, stmt.label)
+        elseif isa(stmt, ReattachNode)
+            push!(jump_dests, stmt.label)
+        elseif isa(stmt, SyncNode)
+            push!(jump_dests, idx+1)
         elseif isa(stmt, Expr)
             if stmt.head === :leave
                 # :leave terminates a BB
@@ -109,12 +122,17 @@ function compute_basic_blocks(stmts::Vector{Any})
             # return never has any successors
             continue
         end
-        if isa(terminator, GotoNode)
+        if isa(terminator, GotoNode) || isa(terminator, ReattachNode)
             block′ = block_for_inst(basic_block_index, terminator.label)
             push!(blocks[block′].preds, num)
             push!(b.succs, block′)
             continue
         end
+        if isa(terminator, DetachNode)
+            block′ = block_for_inst(basic_block_index, terminator.label)
+            push!(blocks[block′].preds, num)
+            push!(b.succs, block′)
+        end
         # Conditional Branch
         if isa(terminator, GotoIfNot)
             block′ = block_for_inst(basic_block_index, terminator.dest)
@@ -442,6 +460,9 @@ struct UndefToken end; const UNDEF_TOKEN = UndefToken()
     elseif isa(stmt, GotoIfNot)
         op == 1 || return OOB_TOKEN
         return stmt.cond
+    elseif isa(stmt, DetachNode) || isa(stmt, ReattachNode) || isa(stmt, SyncNode)
+        op == 1 || return OOBToken()
+        return stmt.syncregion
     elseif isa(stmt, ReturnNode)
         isdefined(stmt, :val) || return OOB_TOKEN
         op == 1 || return OOB_TOKEN
@@ -478,6 +499,7 @@ function is_relevant_expr(e::Expr)
                       :foreigncall, :isdefined, :copyast,
                       :throw_undef_if_not,
                       :cfunction, :method, :pop_exception,
+                      :syncregion,
                       :new_opaque_closure)
 end
 
@@ -499,6 +521,15 @@ end
     elseif isa(stmt, GotoIfNot)
         op == 1 || throw(BoundsError())
         stmt = GotoIfNot(v, stmt.dest)
+    elseif isa(stmt, DetachNode)
+        op == 1 || throw(BoundsError())
+        stmt = DetachNode(v, stmt.label)
+    elseif isa(stmt, ReattachNode)
+        op == 1 || throw(BoundsError())
+        stmt = ReattachNode(v, stmt.label)
+    elseif isa(stmt, SyncNode)
+        op == 1 || throw(BoundsError())
+        stmt = SyncNode(v)
     elseif isa(stmt, ReturnNode)
         op == 1 || throw(BoundsError())
         stmt = typeof(stmt)(v)
@@ -533,7 +564,8 @@ end
 function userefs(@nospecialize(x))
     relevant = (isa(x, Expr) && is_relevant_expr(x)) ||
         isa(x, GotoIfNot) || isa(x, ReturnNode) || isa(x, SSAValue) || isa(x, NewSSAValue) ||
-        isa(x, PiNode) || isa(x, PhiNode) || isa(x, PhiCNode) || isa(x, UpsilonNode)
+        isa(x, PiNode) || isa(x, PhiNode) || isa(x, PhiCNode) || isa(x, UpsilonNode) ||
+        isa(x, DetachNode) || isa(x, ReattachNode) || isa(x, SyncNode)
     return UseRefIterator(x, relevant)
 end
 
@@ -1316,6 +1348,20 @@ function process_node!(compact::IncrementalCompact, result_idx::Int, inst::Instr
         ssa_rename[idx] = SSAValue(result_idx)
         result[result_idx][:stmt] = stmt
         result_idx += 1
+    elseif isa(stmt, DetachNode) || isa(stmt, ReattachNode) || isa(stmt, SyncNode)
+        stmt = renumber_ssa2!(stmt, ssa_rename, used_ssas, new_new_used_ssas, late_fixup, result_idx, do_rename_ssa, mark_refined!)
+        if cfg_transforms_enabled
+            if isa(stmt, DetachNode)
+                stmt = DetachNode(
+                    stmt.syncregion,
+                    bb_rename_succ[stmt.label],
+                )
+            elseif isa(stmt, ReattachNode)
+                stmt = ReattachNode(stmt.syncregion, bb_rename_succ[stmt.label])
+            end
+        end
+        result[result_idx][:inst] = stmt
+        result_idx += 1
     elseif isa(stmt, GotoIfNot) && cfg_transforms_enabled
         stmt = renumber_ssa2!(stmt, ssa_rename, used_ssas, new_new_used_ssas, late_fixup, result_idx, do_rename_ssa, mark_refined!)::GotoIfNot
         result[result_idx][:stmt] = stmt

base/compiler/ssair/legacy.jl

@@ -21,6 +21,10 @@ function inflate_ir!(ci::CodeInfo, sptypes::Vector{VarState}, argtypes::Vector{A
         # Translate statement edges to bb_edges
         if isa(stmt, GotoNode)
             code[i] = GotoNode(block_for_inst(cfg, stmt.label))
+        elseif isa(stmt, DetachNode)
+            code[i] = DetachNode(stmt.syncregion, block_for_inst(cfg, stmt.label))
+        elseif isa(stmt, ReattachNode)
+            code[i] = ReattachNode(stmt.syncregion, block_for_inst(cfg, stmt.label))
         elseif isa(stmt, GotoIfNot)
             code[i] = GotoIfNot(stmt.cond, block_for_inst(cfg, stmt.dest))
         elseif isa(stmt, PhiNode)
@@ -89,6 +93,10 @@ function replace_code_newstyle!(ci::CodeInfo, ir::IRCode)
         stmt = code[i]
         if isa(stmt, GotoNode)
             code[i] = GotoNode(first(ir.cfg.blocks[stmt.label].stmts))
+        elseif isa(stmt, DetachNode)
+            code[i] = DetachNode(stmt.syncregion, first(ir.cfg.blocks[stmt.label].stmts))
+        elseif isa(stmt, ReattachNode)
+            code[i] = ReattachNode(stmt.syncregion, first(ir.cfg.blocks[stmt.label].stmts))
         elseif isa(stmt, GotoIfNot)
             code[i] = GotoIfNot(stmt.cond, first(ir.cfg.blocks[stmt.dest].stmts))
         elseif isa(stmt, PhiNode)

base/compiler/ssair/show.jl

@@ -71,6 +71,12 @@ function print_stmt(io::IO, idx::Int, @nospecialize(stmt), used::BitSet, maxleng
         print(io, "\$(Expr(:enter, #", (stmt::Expr).args[1]::Int, "))")
     elseif stmt isa GotoNode
         print(io, "goto #", stmt.label)
+    elseif stmt isa DetachNode
+        print(io, "detach within ", stmt.syncregion, ", reattach to #", stmt.label)
+    elseif stmt isa ReattachNode
+        print(io, "reattach within ", stmt.syncregion, ", #", stmt.label)
+    elseif stmt isa SyncNode
+        print(io, "sync within ", stmt.syncregion)
     elseif stmt isa PhiNode
         show_unquoted_phinode(io, stmt, indent, "#")
     elseif stmt isa GotoIfNot
@@ -154,11 +160,12 @@ end
 
 function should_print_ssa_type(@nospecialize node)
     if isa(node, Expr)
-        return !(node.head in (:gc_preserve_begin, :gc_preserve_end, :meta, :enter, :leave))
+        return !(node.head in (:gc_preserve_begin, :gc_preserve_end, :meta, :enter, :leave, :syncregion))
     end
     return !isa(node, PiNode)   && !isa(node, GotoIfNot) &&
            !isa(node, GotoNode) && !isa(node, ReturnNode) &&
-           !isa(node, QuoteNode)
+           !isa(node, DetachNode) && !isa(node, ReattachNode) &&
+           !isa(node, SyncNode) && !isa(node, QuoteNode)
 end
 
 function default_expr_type_printer(io::IO; @nospecialize(type), used::Bool, show_type::Bool=true, _...)
@@ -573,6 +580,10 @@ function statement_indices_to_labels(stmt, cfg::CFG)
         stmt = GotoIfNot(stmt.cond, block_for_inst(cfg, stmt.dest))
     elseif stmt isa GotoNode
         stmt = GotoNode(block_for_inst(cfg, stmt.label))
+    elseif isa(stmt, DetachNode)
+        stmt = DetachNode(stmt.syncregion, block_for_inst(cfg, stmt.label))
+    elseif isa(stmt, ReattachNode)
+        stmt = ReattachNode(stmt.syncregion, block_for_inst(cfg, stmt.label))
     elseif stmt isa PhiNode
         e = stmt.edges
         stmt = PhiNode(Int32[block_for_inst(cfg, Int(e[i])) for i in 1:length(e)], stmt.values)

base/compiler/ssair/slot2ssa.jl

@@ -115,6 +115,23 @@ function fixemup!(@specialize(slot_filter), @specialize(rename_slot), ir::IRCode
         end
         return stmt
     end
+    if isa(stmt, Union{DetachNode, ReattachNode, SyncNode})
+        x = fixemup!(slot_filter, rename_slot, ir, ci, idx, stmt.syncregion)
+        if x === UNDEF_TOKEN
+            # `syncregion` can become undef at this point if the sub-CFG
+            # containing detaches etc. is already determined to be unreachable.
+            # Instead of removing the statement, we use goto nodes for
+            # preserving the CFG (i.e., serial projection).
+            isa(stmt, DetachNode) && return GotoIfNot(true, stmt.label)
+            isa(stmt, ReattachNode) && return GotoNode(stmt.label)
+            isa(stmt, SyncNode) && return nothing  # fallthrough
+        else
+            isa(stmt, DetachNode) && return DetachNode(x, stmt.label)
+            isa(stmt, ReattachNode) && return ReattachNode(x, stmt.label)
+            isa(stmt, SyncNode) && return SyncNode(x)
+        end
+        @assert false # unreachable
+    end
     if isexpr(stmt, :isdefined)
         val = stmt.args[1]
         if isa(val, SlotNumber)
@@ -179,7 +196,8 @@ function strip_trailing_junk!(ci::CodeInfo, cfg::CFG, code::Vector{Any}, info::V
     # If the last instruction is not a terminator, add one. This can
     # happen for implicit return on dead branches.
     term = code[end]
-    if !isa(term, GotoIfNot) && !isa(term, GotoNode) && !isa(term, ReturnNode)
+    if !isa(term, GotoIfNot) && !isa(term, GotoNode) && !isa(term, ReturnNode) &&
+       !isa(term, DetachNode) && !isa(term, ReattachNode) && !isa(term, SyncNode)
         push!(code, ReturnNode())
         push!(ssavaluetypes, Union{})
         push!(codelocs, 0)
@@ -399,7 +417,7 @@ function domsort_ssa!(ir::IRCode, domtree::DomTree)
         if node == -1 && !isempty(stack)
             node = pop!(stack)
         end
-        if node != next_node && !isa(terminator, Union{GotoNode, ReturnNode})
+        if node != next_node && !isa(terminator, Union{GotoNode, ReturnNode, DetachNode, ReattachNode, SyncNode})
             if isa(terminator, GotoIfNot)
                 # Need to break the critical edge
                 ncritbreaks += 1
@@ -467,6 +485,10 @@ function domsort_ssa!(ir::IRCode, domtree::DomTree)
                 node[:stmt], node[:type], node[:line] = GotoNode(bb_rename[bb + 1]), Any, 0
             end
             result[inst_range[end]][:stmt] = GotoIfNot(terminator.cond, bb_rename[terminator.dest])
+        elseif isa(terminator, DetachNode)
+            result[inst_range[end]][:stmt] = DetachNode(terminator.syncregion, bb_rename[terminator.label])
+        elseif isa(terminator, ReattachNode)
+            result[inst_range[end]][:stmt] = ReattachNode(terminator.syncregion, bb_rename[terminator.label])
         elseif !isa(terminator, ReturnNode)
             if isa(terminator, Expr)
                 if terminator.head === :enter
@@ -845,10 +867,11 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState,
             push!(worklist, (succ, item, copy(incoming_vals)))
         end
     end
+    code1 = copy(code)
     # Delete any instruction in unreachable blocks (except for terminators)
     for bb in setdiff(BitSet(1:length(cfg.blocks)), visited)
         for idx in cfg.blocks[bb].stmts
-            if isa(code[idx], Union{GotoNode, GotoIfNot, ReturnNode})
+            if isterminator(code[idx])
                 code[idx] = ReturnNode()
             else
                 code[idx] = nothing
@@ -866,6 +889,10 @@ function construct_ssa!(ci::CodeInfo, ir::IRCode, sv::OptimizationState,
         # Convert GotoNode/GotoIfNot/PhiNode to BB addressing
         if isa(stmt, GotoNode)
             new_code[idx] = GotoNode(block_for_inst(cfg, stmt.label))
+        elseif isa(stmt, DetachNode)
+            new_code[idx] = DetachNode(stmt.syncregion, block_for_inst(cfg, stmt.label))
+        elseif isa(stmt, ReattachNode)
+            new_code[idx] = ReattachNode(stmt.syncregion, block_for_inst(cfg, stmt.label))
         elseif isa(stmt, GotoIfNot)
             new_dest = block_for_inst(cfg, stmt.dest)
             if new_dest == bb+1