Skip to content

Commit

Permalink
STEP-1207: Modified 'Should retry tests on failure?' test input (#176)
Browse files Browse the repository at this point in the history
* Updated should_retry_test_on_fail title and description

* Remove unnecessary check when RetryTestsOnFailure is enabled

* Update error message when RetryTestsOnFailure is used above Xcode 12

* Updated messages for automatic retry and should_retry_test_on_fail

* Added missing category for relaunch_tests_for_each_repetition

* Adjusted should_retry_test_on_fail description
  • Loading branch information
Bence1001 authored Jul 19, 2021
1 parent 5de2593 commit 4fb0772
Show file tree
Hide file tree
Showing 3 changed files with 14 additions and 10 deletions.
2 changes: 1 addition & 1 deletion main.go
Original file line number Diff line number Diff line change
Expand Up @@ -274,7 +274,7 @@ func (s Step) ProcessConfig() (Config, error) {
}

if input.RetryTestsOnFailure && xcodeMajorVersion > 12 {
return Config{}, errors.New("Should retry test on failure? (should_retry_test_on_fail) is not available above Xcode 12; use test_repetition_mode=retry_on_failure instead")
return Config{}, errors.New("Should retry tests on failure? (should_retry_test_on_fail) is not available above Xcode 12; use test_repetition_mode=retry_on_failure instead")
}

return Config{
Expand Down
14 changes: 9 additions & 5 deletions step.yml
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,8 @@ inputs:
title: "Test Plan"
summary: Run tests in a specific Test Plan associated with the Scheme.
description: |-
Run tests in a specific Test Plan associated with the Scheme.
Run tests in a specific Test Plan associated with the Scheme.
Leave this input empty to run the default Test Plan or Test Targets associated with the Scheme.
- simulator_device: iPhone 8 Plus
opts:
Expand Down Expand Up @@ -186,12 +187,13 @@ inputs:
summary: The maximum number of times a test will repeat based on Test Repetition Mode.
description: |-
The maximum number of times a test will repeat based on Test Repetition Mode.
Should be more than 1 if the Test Repetition Mode (`test_repetition_mode`) is other than `none`.
is_required: true
- relaunch_tests_for_each_repetition: "no"
opts:
title: Relaunch Tests for Each Repetition (Available in Xcode 13+)
category: Test Repetition
summary: If enabled, tests will launch in a new process for each repetition.
description: |-
If enabled, tests will launch in a new process for each repetition.
Expand Down Expand Up @@ -298,10 +300,12 @@ inputs:
- should_retry_test_on_fail: "no"
opts:
category: Debug
title: "Should retry test on failure?"
title: "Should retry tests on failure? (Not available in Xcode 13+)"
summary: If you set this input to `yes`, the Step will rerun your failed tests.
description: |-
If you set this input to `yes`, the Step will rerun your tests. With Xcode 13 and above, only your failed test cases will be rerun. With older Xcode versions, all test cases will be rerun.
Please note that this feature is only available from Xcode 13 and above, earlier versions support rerunning ALL test cases instead of a specific failed one.
If you set this input to `yes`, the Step will rerun ALL your tests once in the case of failed test/s. Note that ALL your tests will be rerun, not just the ones that failed.
This input is not available if you are using Xcode 13+. In that case, we recommend using the `retry_on_failure` Test Repetition Mode (test_repetition_mode).
value_options:
- "yes"
- "no"
Expand Down
8 changes: 4 additions & 4 deletions xcodebuild.go
Original file line number Diff line number Diff line change
Expand Up @@ -283,21 +283,21 @@ func handleTestRunError(prevRunParams testRunParams, prevRunResult testRunResult
if isStringFoundInOutput(errorPattern, prevRunResult.xcodebuildLog) {
log.Warnf("Automatic retry reason found in log: %s", errorPattern)
if prevRunParams.retryOnTestRunnerError {
log.Printf("retryOnTestRunnerError=true - retrying...")
log.Printf("Automatic retry is enabled - retrying...")

prevRunParams.buildTestParams.RetryTestsOnFailure = false
prevRunParams.retryOnTestRunnerError = false
return cleanOutputDirAndRerunTest(prevRunParams)
}

log.Errorf("retryOnTestRunnerError=false, no more retry, stopping the test!")
log.Errorf("Automatic retry is disabled, no more retry, stopping the test!")
return prevRunResult.xcodebuildLog, prevRunResult.exitCode, prevRunResult.err
}
}

if prevRunParams.xcodeMajorVersion < 13 && prevRunParams.buildTestParams.RetryTestsOnFailure {
if prevRunParams.buildTestParams.RetryTestsOnFailure {
log.Warnf("Test run failed")
log.Printf("retryTestsOnFailure=true - retrying...")
log.Printf("'Should retry tests on failure?' (should_retry_test_on_fail) is enabled - retrying...")

prevRunParams.buildTestParams.RetryTestsOnFailure = false
prevRunParams.retryOnTestRunnerError = false
Expand Down

0 comments on commit 4fb0772

Please sign in to comment.