diff --git a/eventually.go b/eventually.go new file mode 100644 index 0000000..605f369 --- /dev/null +++ b/eventually.go @@ -0,0 +1,168 @@ +// Licensed under the MIT license, see LICENSE file for details. + +package quicktest + +import ( + "fmt" + "reflect" + "time" + + "github.com/rogpeppe/retry" +) + +// Eventually returns an EventuallyChecker, which expects a function with no +// arguments and one return value. It then calls the function repeatedly, +// passing its returned value to the provided Checker c, until it succeeds, +// or until time out according to the retry Strategy. +// +// The retry Strategy can be customized by calling WithStrategy() on the +// returned EventuallyChecker. By default, the check will be retried at a +// starting interval of 100ms and an exponential backoff with a factor of 2, +// timing out after about 5s. +// +// By default, the checker makes no stability check. For that, use +// EventuallyStable or augment the checker by calling WithStableStrategy() on +// it. +// +// Example calls: +// +// c.Assert(func() int64 { +// return atomic.LoadInt64(&foo) +// }, qt.Eventually(qt.Equals), int64(1234)) +// +// c.Assert(func() int64 { +// return atomic.LoadInt64(&foo) +// }, qt.Eventually(qt.Equals).WithStrategy(customStrategy), int64(1234)) +func Eventually(c Checker) *EventuallyChecker { + return &EventuallyChecker{ + checker: c, + retryStrategy: &retry.Strategy{ + Delay: 100 * time.Millisecond, + MaxDelay: 1 * time.Second, + MaxDuration: 5 * time.Second, + Factor: 2, + }, + } +} + +// EventuallyStable returns an EventuallyChecker that is like the one returned +// by Eventually, except it also provides a default retry strategy for stability +// check. +// +// The default stable retry strategy is to re-verify once after about 100ms +// since the intial successful check. +// +// Example calls: +// +// c.Assert(func() int64 { +// return atomic.LoadInt64(&foo) +// }, qt.EventuallyStable(qt.Equals), int64(1234)) +func EventuallyStable(c Checker) *EventuallyChecker { + return Eventually(c).WithStableStrategy(&retry.Strategy{ + Delay: 100 * time.Millisecond, + MaxDuration: 150 * time.Millisecond, + }) +} + +// EventuallyChecker is a Checker that allows providing retry strategies to +// retry the check over a period of time. +// It also allows providing a stable retry strategy to run a stability check, +// i.e. once the check is successful, it keeps checking that it stays that way. +type EventuallyChecker struct { + checker Checker + retryStrategy *retry.Strategy + stableRetryStrategy *retry.Strategy +} + +// WithStrategy allows specifying a custom retry strategy, specifying initial +// delay, delay between attempts, maximum duration before timing out, etc. +func (e *EventuallyChecker) WithStrategy(strategy *retry.Strategy) *EventuallyChecker { + return &EventuallyChecker{ + checker: e.checker, + retryStrategy: strategy, + stableRetryStrategy: e.stableRetryStrategy, + } +} + +// WithStableStrategy allows specifying a custom retry strategy for the +// stability check. If not provided, no stability check will be run. +func (e *EventuallyChecker) WithStableStrategy(strategy *retry.Strategy) *EventuallyChecker { + return &EventuallyChecker{ + checker: e.checker, + retryStrategy: e.retryStrategy, + stableRetryStrategy: strategy, + } +} + +// Check implements Checker.Check by calling the given got function repeatedly +// and calling the underlying Checker with the returned value, according to the +// retry Strategy, until the verification succeeds or the Strategy times out. +// If the Strategy times out, the Check will fail. +// +// If an additional stable retry Strategy is provided, it also calls the +// underlying Checker again after a succesfull check, according to the stable +// retry Strategy, until either the Strategy times out or the verification +// fails. If the stable Strategy times out, the Check will succeed. +func (e *EventuallyChecker) Check(got interface{}, args []interface{}, notef func(key string, value interface{})) error { + // Define local note function for underlying checker notes, so that we can + // save only the notes from the last call at the end of the execution. + notes := []note{} + localnotef := func(key string, value interface{}) { + notes = append(notes, note{key, value}) + } + defer func() { + for _, n := range append(notes, note{"got", got}) { + notef(n.key, n.value) + } + }() + + // Validate that the given got parameter is a function with no parameters + // and one return value. + f := reflect.ValueOf(got) + if f.Kind() != reflect.Func { + return BadCheckf("first argument is not a function") + } + ftype := f.Type() + if ftype.NumIn() != 0 { + return BadCheckf("cannot use a function receiving arguments") + } + if ftype.NumOut() != 1 { + return BadCheckf("cannot use a function returning more than one value") + } + + // Run the checker according to the retry strategy, succeeding on the first + // successful check. + var err error + for i, hasNext := e.retryStrategy.Start(), true; hasNext; hasNext = i.Next(nil) { + notes = []note{} + got = f.Call(nil)[0].Interface() + err = e.checker.Check(got, args, localnotef) + if err == nil { + break + } + } + if err != nil { + return fmt.Errorf("tried for %v, %s", e.retryStrategy.MaxDuration, err.Error()) + } + + // If a stable strategy is provided, run the checker again according to it, + // failing on the first error. + if e.stableRetryStrategy != nil { + for i, hasNext := e.stableRetryStrategy.Start(), true; hasNext; hasNext = i.Next(nil) { + notes = []note{} + got = f.Call(nil)[0].Interface() + err := e.checker.Check(got, args, localnotef) + if err != nil { + return fmt.Errorf("less than %v after an initial success, %s", e.stableRetryStrategy.MaxDuration, err.Error()) + } + } + } + + return nil +} + +// ArgNames implements Checker.ArgNames by delegating the call to the underlying +// Checker. +func (e *EventuallyChecker) ArgNames() []string { + return e.checker.ArgNames() +} diff --git a/eventually_test.go b/eventually_test.go new file mode 100644 index 0000000..38896e3 --- /dev/null +++ b/eventually_test.go @@ -0,0 +1,213 @@ +// Licensed under the MIT license, see LICENSE file for details. + +package quicktest_test + +import ( + "testing" + "time" + + "github.com/rogpeppe/retry" + + qt "github.com/frankban/quicktest" +) + +var evChannel = make(chan int) + +var eventuallyTests = []struct { + about string + checker qt.Checker + got interface{} + args []interface{} + waitTime time.Duration + stableRetryStrategy *retry.Strategy + verbose bool + expectedCheckFailure string + expectedNegateFailure string +}{{ + about: "Expected value instantly", + checker: qt.Equals, + got: func() int { return 42 }, + args: []interface{}{42}, + expectedNegateFailure: ` +error: + unexpected success +got: + int(42) +want: + +`, +}, { + about: "Expected value after a while", + checker: qt.Equals, + got: func() int { + select { + case <-evChannel: + return 42 + default: + return 40 + } + }, + args: []interface{}{42}, + waitTime: 10 * time.Millisecond, + expectedNegateFailure: ` +error: + unexpected success +got: + int(42) +want: + +`, +}, { + about: "Expected value after max duration", + checker: qt.Equals, + got: func() int { + select { + case <-evChannel: + return 42 + default: + return 40 + } + }, + args: []interface{}{42}, + expectedCheckFailure: ` +error: + tried for 100ms, values are not equal +got: + int(40) +want: + int(42) +`}, { + about: "Expected value instantly but then unstable", + checker: qt.Equals, + got: func() int { + select { + case <-evChannel: + return 40 + default: + return 42 + } + }, + args: []interface{}{42}, + waitTime: 10 * time.Millisecond, + stableRetryStrategy: &retry.Strategy{ + Delay: 20 * time.Millisecond, + MaxDelay: 20 * time.Millisecond, + MaxDuration: 100 * time.Millisecond, + }, + expectedCheckFailure: ` +error: + less than 100ms after an initial success, values are not equal +got: + int(40) +want: + int(42) +`}, { + about: "Expected value instantly and then stable", + checker: qt.Equals, + got: func() int { + return 42 + }, + args: []interface{}{42}, + stableRetryStrategy: &retry.Strategy{ + Delay: 20 * time.Millisecond, + MaxDelay: 20 * time.Millisecond, + MaxDuration: 100 * time.Millisecond, + }, + expectedNegateFailure: ` +error: + unexpected success +got: + int(42) +want: + +`}, { + about: "Value instead of function", + checker: qt.Equals, + got: 42, + args: []interface{}{42}, + expectedCheckFailure: ` +error: + bad check: first argument is not a function +got: + int(42) +`, + expectedNegateFailure: ` +error: + bad check: first argument is not a function +got: + int(42) +`}, { + about: "Function with too many return values", + checker: qt.Equals, + got: func() (int, error) { + return 42, nil + }, + args: []interface{}{42}, + expectedCheckFailure: ` +error: + bad check: cannot use a function returning more than one value +got: + func() (int, error) {...} +`, + expectedNegateFailure: ` +error: + bad check: cannot use a function returning more than one value +got: + func() (int, error) {...} +`}, { + about: "Function with arguments", + checker: qt.Equals, + got: func(a int) int { + return a + }, + args: []interface{}{42}, + expectedCheckFailure: ` +error: + bad check: cannot use a function receiving arguments +got: + func(int) int {...} +`, + expectedNegateFailure: ` +error: + bad check: cannot use a function receiving arguments +got: + func(int) int {...} +`}} + +func TestEventually(t *testing.T) { + for _, test := range eventuallyTests { + checker := qt.WithVerbosity(test.checker, test.verbose) + if test.waitTime != 0 { + go func() { + time.Sleep(test.waitTime) + evChannel <- 1 + }() + } + t.Run(test.about, func(t *testing.T) { + tt := &testingT{} + c := qt.New(tt) + ok := c.Check(test.got, qt.Eventually(checker).WithStrategy(&retry.Strategy{ + Delay: 10 * time.Millisecond, + MaxDelay: 10 * time.Millisecond, + MaxDuration: 100 * time.Millisecond, + }).WithStableStrategy(test.stableRetryStrategy), test.args...) + checkResult(t, ok, tt.errorString(), test.expectedCheckFailure) + }) + if test.waitTime != 0 { + go func() { + time.Sleep(test.waitTime) + evChannel <- 1 + }() + } + t.Run("Not "+test.about, func(t *testing.T) { + tt := &testingT{} + c := qt.New(tt) + ok := c.Check(test.got, qt.Not(qt.Eventually(checker).WithStrategy(&retry.Strategy{ + Delay: 10 * time.Millisecond, + MaxDelay: 10 * time.Millisecond, + MaxDuration: 100 * time.Millisecond, + }).WithStableStrategy(test.stableRetryStrategy)), test.args...) + checkResult(t, ok, tt.errorString(), test.expectedNegateFailure) + }) + } +} diff --git a/go.mod b/go.mod index 313758b..4850bc7 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/frankban/quicktest require ( github.com/google/go-cmp v0.5.7 github.com/kr/pretty v0.3.0 + github.com/rogpeppe/retry v0.1.0 ) go 1.13 diff --git a/go.sum b/go.sum index b34b4ae..2a42ecc 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -10,6 +12,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/retry v0.1.0 h1:6km4oqeZcFrnhx+PCPg/YxV3fnTdROBNVlSl8Pe/ztU= +github.com/rogpeppe/retry v0.1.0/go.mod h1:/PtRtl9qXn+Pv5S4wN+Y5nusihQeI1PJ9U7KDcKzuvI= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/report.go b/report.go index c8a2265..55ae186 100644 --- a/report.go +++ b/report.go @@ -90,9 +90,12 @@ func writeError(w io.Writer, err error, p reportParams) { } // Write notes if present. + noteKeys := map[string]bool{} for _, n := range p.notes { printPair(n.key, n.value) + noteKeys[n.key] = true } + if IsBadCheck(err) || err == ErrSilent { // For errors in the checker invocation or for silent errors, do not // show output from args. @@ -101,6 +104,9 @@ func writeError(w io.Writer, err error, p reportParams) { // Write provided args. for i, arg := range append([]interface{}{p.got}, p.args...) { + if noteKeys[p.argNames[i]] { + continue + } printPair(p.argNames[i], arg) } }