2023-03-28 22:48:58 +00:00
|
|
|
// Copyright (c) HashiCorp, Inc.
|
|
|
|
// SPDX-License-Identifier: MPL-2.0
|
|
|
|
|
2017-04-29 16:33:17 +00:00
|
|
|
package retry
|
|
|
|
|
|
|
|
import (
|
|
|
|
"fmt"
|
2023-12-06 17:11:32 +00:00
|
|
|
"os"
|
2017-04-29 16:33:17 +00:00
|
|
|
)
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
var _ TestingTB = &R{}
|
2020-08-14 20:54:44 +00:00
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
type R struct {
|
|
|
|
wrapped TestingTB
|
|
|
|
retryer Retryer
|
2017-04-29 16:33:17 +00:00
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
done bool
|
|
|
|
fullOutput bool
|
|
|
|
immediateCleanup bool
|
2017-04-29 16:33:17 +00:00
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
attempts []*attempt
|
|
|
|
}
|
2023-06-16 20:29:50 +00:00
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) Cleanup(clean func()) {
|
|
|
|
if r.immediateCleanup {
|
|
|
|
a := r.getCurrentAttempt()
|
|
|
|
a.cleanups = append(a.cleanups, clean)
|
|
|
|
} else {
|
|
|
|
r.wrapped.Cleanup(clean)
|
|
|
|
}
|
2017-04-29 16:33:17 +00:00
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) Error(args ...any) {
|
|
|
|
r.Log(args...)
|
|
|
|
r.Fail()
|
2022-12-15 17:52:48 +00:00
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) Errorf(format string, args ...any) {
|
|
|
|
r.Logf(format, args...)
|
|
|
|
r.Fail()
|
2023-06-06 21:09:48 +00:00
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) Fail() {
|
|
|
|
r.getCurrentAttempt().failed = true
|
|
|
|
}
|
2021-04-21 21:04:38 +00:00
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) FailNow() {
|
|
|
|
r.Fail()
|
|
|
|
panic(attemptFailed{})
|
2023-06-16 20:29:50 +00:00
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) Failed() bool {
|
|
|
|
return r.getCurrentAttempt().failed
|
|
|
|
}
|
2023-06-16 20:29:50 +00:00
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) Fatal(args ...any) {
|
|
|
|
r.Log(args...)
|
|
|
|
r.FailNow()
|
|
|
|
}
|
2023-06-16 20:29:50 +00:00
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) Fatalf(format string, args ...any) {
|
|
|
|
r.Logf(format, args...)
|
|
|
|
r.FailNow()
|
2023-06-16 20:29:50 +00:00
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) Helper() {
|
|
|
|
// *testing.T will just record which functions are helpers by their addresses and
|
|
|
|
// it doesn't much matter where where we record that they are helpers
|
|
|
|
r.wrapped.Helper()
|
2017-04-29 16:33:17 +00:00
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) Log(args ...any) {
|
|
|
|
r.log(fmt.Sprintln(args...))
|
2017-04-29 16:33:17 +00:00
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) Logf(format string, args ...any) {
|
2017-04-29 16:33:17 +00:00
|
|
|
r.log(fmt.Sprintf(format, args...))
|
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
// Name will return the name of the underlying TestingT.
|
|
|
|
func (r *R) Name() string {
|
|
|
|
return r.wrapped.Name()
|
2017-04-29 16:33:17 +00:00
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
// Setenv will save the current value of the specified env var, set it to the
|
|
|
|
// specified value and then restore it to the original value in a cleanup function
|
|
|
|
// once the retry attempt has finished.
|
|
|
|
func (r *R) Setenv(key, value string) {
|
|
|
|
prevValue, ok := os.LookupEnv(key)
|
|
|
|
|
|
|
|
if err := os.Setenv(key, value); err != nil {
|
|
|
|
r.wrapped.Fatalf("cannot set environment variable: %v", err)
|
|
|
|
}
|
|
|
|
|
|
|
|
if ok {
|
|
|
|
r.Cleanup(func() {
|
|
|
|
os.Setenv(key, prevValue)
|
|
|
|
})
|
|
|
|
} else {
|
|
|
|
r.Cleanup(func() {
|
|
|
|
os.Unsetenv(key)
|
|
|
|
})
|
|
|
|
}
|
2018-10-19 16:04:07 +00:00
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
// TempDir will use the wrapped TestingT to create a temporary directory
|
|
|
|
// that will be cleaned up when ALL RETRYING has finished.
|
|
|
|
func (r *R) TempDir() string {
|
|
|
|
return r.wrapped.TempDir()
|
|
|
|
}
|
|
|
|
|
|
|
|
// Check will call r.Fatal(err) if err is not nil
|
2017-05-05 07:23:28 +00:00
|
|
|
func (r *R) Check(err error) {
|
|
|
|
if err != nil {
|
2023-12-06 17:11:32 +00:00
|
|
|
r.Fatal(err)
|
2017-05-05 07:23:28 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-04-23 21:01:05 +00:00
|
|
|
func (r *R) Stop(err error) {
|
|
|
|
r.log(err.Error())
|
|
|
|
r.done = true
|
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) failCurrentAttempt() {
|
|
|
|
r.getCurrentAttempt().failed = true
|
2017-04-29 16:33:17 +00:00
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) log(s string) {
|
|
|
|
a := r.getCurrentAttempt()
|
|
|
|
a.output = append(a.output, decorate(s))
|
2017-04-29 16:33:17 +00:00
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) getCurrentAttempt() *attempt {
|
|
|
|
if len(r.attempts) == 0 {
|
|
|
|
panic("no retry attempts have been started yet")
|
|
|
|
}
|
|
|
|
|
|
|
|
return r.attempts[len(r.attempts)-1]
|
2017-04-29 16:33:17 +00:00
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
// cleanupAttempt will perform all the register cleanup operations recorded
|
|
|
|
// during execution of the single round of the test function.
|
|
|
|
func (r *R) cleanupAttempt(a *attempt) {
|
|
|
|
// Make sure that if a cleanup function panics,
|
|
|
|
// we still run the remaining cleanup functions.
|
|
|
|
defer func() {
|
|
|
|
err := recover()
|
|
|
|
if err != nil {
|
|
|
|
r.Stop(fmt.Errorf("error when performing test cleanup: %v", err))
|
|
|
|
}
|
|
|
|
if len(a.cleanups) > 0 {
|
|
|
|
r.cleanupAttempt(a)
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
|
|
|
|
for len(a.cleanups) > 0 {
|
|
|
|
var cleanup func()
|
|
|
|
if len(a.cleanups) > 0 {
|
|
|
|
last := len(a.cleanups) - 1
|
|
|
|
cleanup = a.cleanups[last]
|
|
|
|
a.cleanups = a.cleanups[:last]
|
|
|
|
}
|
|
|
|
if cleanup != nil {
|
|
|
|
cleanup()
|
2017-04-29 16:33:17 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
// runAttempt will execute one round of the test function and handle cleanups and panic recovery
|
|
|
|
// of a failed attempt that should not stop retrying.
|
|
|
|
func (r *R) runAttempt(f func(r *R)) {
|
|
|
|
r.Helper()
|
|
|
|
|
|
|
|
a := &attempt{}
|
|
|
|
r.attempts = append(r.attempts, a)
|
2021-04-23 21:01:05 +00:00
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
defer r.cleanupAttempt(a)
|
|
|
|
defer func() {
|
|
|
|
if p := recover(); p != nil && p != (attemptFailed{}) {
|
|
|
|
panic(p)
|
2021-04-23 21:01:05 +00:00
|
|
|
}
|
2023-12-06 17:11:32 +00:00
|
|
|
}()
|
|
|
|
f(r)
|
|
|
|
}
|
|
|
|
|
|
|
|
func (r *R) run(f func(r *R)) {
|
|
|
|
r.Helper()
|
2021-04-23 21:01:05 +00:00
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
for r.retryer.Continue() {
|
|
|
|
r.runAttempt(f)
|
2021-04-23 21:01:05 +00:00
|
|
|
|
|
|
|
switch {
|
2023-12-06 17:11:32 +00:00
|
|
|
case r.done:
|
|
|
|
r.recordRetryFailure()
|
2021-04-23 21:01:05 +00:00
|
|
|
return
|
2023-12-06 17:11:32 +00:00
|
|
|
case !r.Failed():
|
|
|
|
// the current attempt did not fail so we can go ahead and return
|
2021-04-21 21:04:38 +00:00
|
|
|
return
|
2017-04-29 16:33:17 +00:00
|
|
|
}
|
|
|
|
}
|
2023-12-06 17:11:32 +00:00
|
|
|
|
|
|
|
// We cannot retry any more and no attempt has succeeded yet.
|
|
|
|
r.recordRetryFailure()
|
2017-04-29 16:33:17 +00:00
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
func (r *R) recordRetryFailure() {
|
|
|
|
r.Helper()
|
|
|
|
output := r.getCurrentAttempt().output
|
|
|
|
if r.fullOutput {
|
|
|
|
var combined []string
|
|
|
|
for _, attempt := range r.attempts {
|
|
|
|
combined = append(combined, attempt.output...)
|
|
|
|
}
|
|
|
|
output = combined
|
|
|
|
}
|
|
|
|
|
|
|
|
out := dedup(output)
|
|
|
|
if out != "" {
|
|
|
|
r.wrapped.Log(out)
|
|
|
|
}
|
|
|
|
r.wrapped.FailNow()
|
2018-02-08 01:29:30 +00:00
|
|
|
}
|
|
|
|
|
2023-12-06 17:11:32 +00:00
|
|
|
type attempt struct {
|
|
|
|
failed bool
|
|
|
|
output []string
|
|
|
|
cleanups []func()
|
2017-04-29 16:33:17 +00:00
|
|
|
}
|
2023-12-06 17:11:32 +00:00
|
|
|
|
|
|
|
// attemptFailed is a sentinel value to indicate that the func itself
|
|
|
|
// didn't panic, rather that `FailNow` was called.
|
|
|
|
type attemptFailed struct{}
|