dryrun: keep track of loop variables from parent testsets (#28)

Partially fixes #27.

The full fix might be to make `resolve!` also keep track of these
loop variables; but this would involve storing lists of collected
iterators for each nested testset: once we evaluate an iterator
for filtering in `resolve!`, we have to store it in case
it has some random behavior (e.g. `rand(1:9, i)` where `i` is
a loop variable from a parent testset), in which case re-evaling
it at execution time would lead to inconsistency.
The testset tree would also have to be traversed depth-first by also
unfolding testset-for. This could be a nice improvement if the perfs
don't suffer (instead of storing parent strings subjects and looping
through them, we would store iterators and recurse into them).

Author: rfourquet

src/ReTest.jl

@@ -243,7 +243,13 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
     strings = empty!(ts.strings)
     desc = ts.desc
     ts.loopvalues = nothing # unnecessary ?
-    ts.loopiters = nothing
+    loopiters = ts.loopiters =
+        if ts.loops === nothing
+            nothing
+        else
+            Expr(:tuple, (arg.args[1] for arg in ts.loops)...)
+        end
+
     if 0 != ts.id != id && !warned[] && has(pat, Integer)
         # this can happen when nested testsets are added and Revise is active
         @warn "testset IDs have changed since last run"
@@ -337,7 +343,6 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
     else # we have a testset-for with description which needs interpolation, or
          # the iterator must be computed to get an iterator counter
         xs = ()
-        loopiters = Expr(:tuple, (arg.args[1] for arg in loops)...)
 
         try
             # we need to evaluate roughly the following:
@@ -359,7 +364,6 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
             xs = Core.eval(mod, xsgen)
             @assert xs isa Vector
             ts.loopvalues = xs
-            ts.loopiters = loopiters
         catch
             @assert xs == ()
             ts.descwidth = shown ? descwidth(missing) : 0
@@ -409,15 +413,20 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
     run, id
 end
 
-eval_desc(mod, ts, x) =
+eval_desc(mod, ts, x; stack=false) = # stack => x == iterstack in dryrun
     if ts.desc isa String
         ts.desc
     else
         try
-            Core.eval(mod, quote
-                      let $(ts.loopiters) = $x
-                          $(ts.desc)
-                      end
+            Core.eval(mod,
+                      if stack
+                          Expr(:let, x, ts.desc)
+                      else
+                          quote
+                              let $(ts.loopiters) = $x
+                                  $(ts.desc)
+                              end
+                          end
                       end)::String
         catch
             missing
@@ -442,7 +451,7 @@ function make_ts(ts::TestsetExpr, pat::Pattern, stats, chan)
         end
     else
         c = count(x -> x === nothing, (ts.loopvalues, ts.loopiters))
-        @assert c == 0 || c == 2
+        @assert c == 0 || c == 1
         if c == 0
             loops = [Expr(:(=), ts.loopiters, ts.loopvalues)]
         else
@@ -1448,7 +1457,7 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0,
                 # external calls:
                 ; maxidw::Int, marks::Bool, tag::Vector, clear::Bool,
                 # only recursive calls:
-                evaldesc=true, repeated=nothing, show::Bool=true)
+                evaldesc=true, repeated=nothing, show::Bool=true, iterstack=Expr(:block))
     @assert ts.run
     desc = ts.desc
 
@@ -1518,7 +1527,8 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0,
             for tsc in ts.children
                 tsc.run || continue
                 dryrun(mod, tsc, pat, align + 2, subject,
-                       maxidw=maxidw, marks=marks, tag=tag, clear=clear, show=true)
+                       maxidw=maxidw, marks=marks, tag=tag, clear=clear, show=true,
+                       iterstack=iterstack)
             end
             false, false, false # meaningless unused triple
         elseif marks
@@ -1536,7 +1546,7 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0,
                 tsc.run || continue
                 cp, cf, cu = dryrun(mod, tsc, pat, align + 2, subject,
                                     maxidw=maxidw, marks=marks, tag=tag, clear=clear,
-                                    show=false)
+                                    show=false, iterstack=iterstack)
                 passes |= cp
                 fails |= cf
                 unrun |= cu
@@ -1585,10 +1595,30 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0,
             ts.iter = iter # necessary when reachable is used
             dryrun(mod, beginend, pat, align, parentsubj; evaldesc=false,
                    repeated=repeated, maxidw=maxidw, marks=marks, tag=tag,
-                   clear=clear, show=show)
+                   clear=clear, show=show, iterstack=iterstack)
         end
 
         loopvalues = ts.loopvalues
+        if loopvalues === nothing
+            # we check whether we can now evaluate loopvalues via iterstack
+            try
+                # cf. resolve!
+                xssym = gensym()
+                xsgen = quote
+                    let $xssym = []
+                        $(Expr(:for, Expr(:block, ts.loops...),
+                               Expr(:call, Expr(:., :Base, QuoteNode(:push!)),
+                                    xssym, ts.loopiters)))
+                        $xssym
+                    end
+                end
+                loopvalues = Core.eval(mod, Expr(:let, iterstack, xsgen))
+                @assert loopvalues isa Vector
+            catch
+                @assert loopvalues == nothing
+            end
+        end
+
         if loopvalues === nothing
             # ts.desc is probably a String (cf. resolve!); if so, don't print repeated
             # identitical lines (caveat: if subjects of children would change randomly)
@@ -1615,7 +1645,8 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0,
         else
             passes, fails, unrun = false, false, false
             for (i, x) in enumerate(loopvalues)
-                descx = eval_desc(mod, ts, x)
+                push!(iterstack.args, Expr(:(=), ts.loopiters, x))
+                descx = eval_desc(mod, ts, iterstack, stack=true)
                 if descx === missing
                     # we would usually have `i == 1`, but not in some rare cases;
                     # once we find an uninterpolated description, we still assume
@@ -1628,6 +1659,7 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0,
                 else
                     lp, lf, lu = dryrun_beginend(descx, iter=i)
                 end
+                pop!(iterstack.args)
                 passes |= lp
                 fails |= lf
                 unrun |= lu

test/runtests.jl

@@ -635,6 +635,33 @@ end # MultiLoops
     check(MultiLoops, "1 1", [(1, 1)])
 end
 
+module LoopsVariablesDryrun
+# check that even with for-iterators which depend on previous loop variables,
+# dryrun mode is able to compute them and corresponding descriptions,
+# and filter accordingly
+
+using ReTest
+
+@testset "a$i" for i=1:2
+    @testset "b$j" for j=1:i
+        @test true
+    end
+end
+end
+
+@chapter Loops begin
+    # no match, so this it at least filtered for final testsets
+    check(LoopsVariablesDryrun, "a1/b3", dry=true, verbose=9, [], output="""
+1| a1
+1| a2
+""")
+    check(LoopsVariablesDryrun, "a2/b1", dry=true, verbose=9, [], output="""
+1| a1
+1| a2
+2|   b1
+""")
+end
+
 # * Anonym ...................................................................
 
 module Anonym