From 7da2f7a9e626ad9bcc56152400607c82fa295b28 Mon Sep 17 00:00:00 2001 From: Gijs Weterings Date: Wed, 28 Oct 2026 08:57:59 -0700 Subject: [PATCH] Fail CI retry chains on the job that actually failed Summary: The E2E and Fantom retry chains reported their final failure through a separate `report` job. The `test` job always succeeded (its test step used `continue-on-error: true`), and `report` read its `status` output and exited 1 on the last attempt. This caused three problems: - The red check was a five-second status job, not the job that ran the tests, so finding the real failure meant opening a green job and looking for the step with a warning icon. - "Re-run failed jobs" re-ran only `report`. It read the previous attempt's cached `status` output and failed again without running any tests. - The Debug and Release matrix legs wrote the same `status` output, and GitHub keeps only the last value written to a given output name. If one flavor failed and the other passed and finished later, `status` came out empty, so no retry ran and the failure was dropped silently. This change: - Removes the `report` jobs. The test step runs as two copies of the same step, gated on `inputs.fail-on-error`: earlier attempts use `continue-on-error: true`, so they stay green and record the failure in the `status` output, and the final attempt runs without it, so the test step itself fails the job. The two copies share their `with:` block through a YAML anchor. (A single step with `continue-on-error: ${{ !inputs.fail-on-error }}` doesn't work: GitHub evaluated it as true on the final attempt, so the failure stayed hidden. Step `if:` conditions read `inputs` correctly.) - Has each matrix leg write a flavor-specific output (`debug_status` / `release_status`). GitHub combines outputs with different names across matrix legs. The reusable workflow's `status` output is `debug_status || release_status`, or `jobs.test.outputs.status` for the single-job Fantom workflow. - Also triggers the retry jobs in `test-all.yml` on `needs..result == 'failure'`. The iOS `report` jobs counted a failed `build` or `test` job as a failure, and this keeps that behavior for every workflow. The flavor is lowercased with `tr` rather than `${FLAVOR,,}` because the macOS runners' default `bash` may be 3.2. A flavor with no matching output fails the step with an error instead of writing an output nothing reads; a red `test` job still triggers the retry through `needs..result`. Changelog: [Internal] Differential Revision: D128043004 --- .github/workflows/e2e-android-rntester.yml | 51 ++++++++++--------- .github/workflows/e2e-android-templateapp.yml | 51 ++++++++++--------- .github/workflows/e2e-ios-rntester.yml | 50 +++++++++--------- .github/workflows/e2e-ios-templateapp.yml | 51 ++++++++++--------- .github/workflows/fantom-tests.yml | 31 ++++------- .github/workflows/test-all.yml | 20 ++++---- 6 files changed, 126 insertions(+), 128 deletions(-) diff --git a/.github/workflows/e2e-android-rntester.yml b/.github/workflows/e2e-android-rntester.yml index f3ff2fc0dc98..62fa4c4759f7 100644 --- a/.github/workflows/e2e-android-rntester.yml +++ b/.github/workflows/e2e-android-rntester.yml @@ -14,14 +14,17 @@ on: default: 0 outputs: status: - description: 'The result of the E2E tests (success or failure)' - value: ${{ jobs.report.outputs.status }} + description: "'failure' if the E2E tests failed, empty otherwise" + value: ${{ jobs.test.outputs.debug_status || jobs.test.outputs.release_status }} jobs: test: runs-on: 4-core-ubuntu + # Matrix legs write differently named outputs so they are combined + # instead of the last leg to finish overwriting the others. outputs: - status: ${{ steps.report-status.outputs.status }} + debug_status: ${{ steps.report-status.outputs.debug_status }} + release_status: ${{ steps.report-status.outputs.release_status }} strategy: fail-fast: false matrix: @@ -59,11 +62,11 @@ jobs: echo "should-run=$SHOULD_RUN" >> "$GITHUB_OUTPUT" - name: Run E2E Tests id: run-tests - if: steps.test-state.outputs.should-run == 'true' + if: ${{ steps.test-state.outputs.should-run == 'true' && !inputs.fail-on-error }} continue-on-error: true uses: ./.github/actions/maestro-android timeout-minutes: 90 - with: + with: &test-inputs app-path: ./packages/rn-tester/android/app/build/outputs/apk/${{ matrix.flavor }}/app-arm64-v8a-${{ matrix.flavor }}.apk app-id: com.facebook.react.uiapp maestro-flow: ./packages/rn-tester/.maestro @@ -75,6 +78,14 @@ jobs: # Running the debug ARM64 APK through native translation can delay # React commits long enough for Maestro to miss taps and assertions. exclude-tags: ${{ matrix.flavor == 'debug' && 'android-release-only' || '' }} + - name: Run E2E Tests + # Same step without continue-on-error, so the final attempt fails + # here. A step's continue-on-error expression doesn't see inputs. + id: run-tests-final + if: ${{ steps.test-state.outputs.should-run == 'true' && inputs.fail-on-error }} + uses: ./.github/actions/maestro-android + timeout-minutes: 90 + with: *test-inputs - name: Store per-flow test state if: always() uses: ./.github/actions/upload-artifact @@ -85,23 +96,15 @@ jobs: path: /tmp/maestro-android-state/results.json - name: Report status id: report-status - if: ${{ always() && steps.run-tests.outcome == 'failure' }} - run: echo "status=failure" >> $GITHUB_OUTPUT - - report: - needs: test - if: always() - runs-on: ubuntu-latest - outputs: - status: ${{ steps.check.outputs.status }} - steps: - - id: check + if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }} + env: + FLAVOR: ${{ matrix.flavor }} run: | - if [[ "${{ needs.test.outputs.status }}" == "failure" ]]; then - echo "status=failure" >> $GITHUB_OUTPUT - else - echo "status=success" >> $GITHUB_OUTPUT - fi - - name: Fail if needed - if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }} - run: exit 1 + flavor=$(echo "$FLAVOR" | tr "[:upper:]" "[:lower:]") + case "$flavor" in + debug | release) echo "${flavor}_status=failure" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::No job output for flavor '$FLAVOR'. Add a ${flavor}_status output to this job and to the workflow status output." + exit 1 + ;; + esac diff --git a/.github/workflows/e2e-android-templateapp.yml b/.github/workflows/e2e-android-templateapp.yml index eb9518af734b..5fdc3b517dee 100644 --- a/.github/workflows/e2e-android-templateapp.yml +++ b/.github/workflows/e2e-android-templateapp.yml @@ -14,14 +14,17 @@ on: default: 0 outputs: status: - description: 'The result of the E2E tests (success or failure)' - value: ${{ jobs.report.outputs.status }} + description: "'failure' if the E2E tests failed, empty otherwise" + value: ${{ jobs.test.outputs.debug_status || jobs.test.outputs.release_status }} jobs: test: runs-on: 4-core-ubuntu + # Matrix legs write differently named outputs so they are combined + # instead of the last leg to finish overwriting the others. outputs: - status: ${{ steps.report-status.outputs.status }} + debug_status: ${{ steps.report-status.outputs.debug_status }} + release_status: ${{ steps.report-status.outputs.release_status }} strategy: fail-fast: false matrix: @@ -81,11 +84,11 @@ jobs: echo "should-run=$SHOULD_RUN" >> "$GITHUB_OUTPUT" - name: Run E2E Tests id: run-tests - if: steps.test-state.outputs.should-run == 'true' + if: ${{ steps.test-state.outputs.should-run == 'true' && !inputs.fail-on-error }} continue-on-error: true uses: ./.github/actions/maestro-android timeout-minutes: 90 - with: + with: &test-inputs app-path: /tmp/android-templateapp/app-${{ matrix.flavor }}.apk app-id: com.rntestproject maestro-flow: ./scripts/e2e/.maestro/ @@ -94,6 +97,14 @@ jobs: emulator-api-level: '35' emulator-target: google_apis emulator-arch: x86_64 + - name: Run E2E Tests + # Same step without continue-on-error, so the final attempt fails + # here. A step's continue-on-error expression doesn't see inputs. + id: run-tests-final + if: ${{ steps.test-state.outputs.should-run == 'true' && inputs.fail-on-error }} + uses: ./.github/actions/maestro-android + timeout-minutes: 90 + with: *test-inputs - name: Store per-flow test state if: always() uses: ./.github/actions/upload-artifact @@ -104,23 +115,15 @@ jobs: path: /tmp/maestro-android-state/results.json - name: Report status id: report-status - if: ${{ always() && steps.run-tests.outcome == 'failure' }} - run: echo "status=failure" >> $GITHUB_OUTPUT - - report: - needs: test - if: always() - runs-on: ubuntu-latest - outputs: - status: ${{ steps.check.outputs.status }} - steps: - - id: check + if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }} + env: + FLAVOR: ${{ matrix.flavor }} run: | - if [[ "${{ needs.test.outputs.status }}" == "failure" ]]; then - echo "status=failure" >> $GITHUB_OUTPUT - else - echo "status=success" >> $GITHUB_OUTPUT - fi - - name: Fail if needed - if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }} - run: exit 1 + flavor=$(echo "$FLAVOR" | tr "[:upper:]" "[:lower:]") + case "$flavor" in + debug | release) echo "${flavor}_status=failure" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::No job output for flavor '$FLAVOR'. Add a ${flavor}_status output to this job and to the workflow status output." + exit 1 + ;; + esac diff --git a/.github/workflows/e2e-ios-rntester.yml b/.github/workflows/e2e-ios-rntester.yml index c841c409b9ea..b69ab633febf 100644 --- a/.github/workflows/e2e-ios-rntester.yml +++ b/.github/workflows/e2e-ios-rntester.yml @@ -11,14 +11,17 @@ on: default: false outputs: status: - description: 'The result of the E2E tests (success or failure)' - value: ${{ jobs.report.outputs.status }} + description: "'failure' if the E2E tests failed, empty otherwise" + value: ${{ jobs.test.outputs.debug_status || jobs.test.outputs.release_status }} jobs: test: runs-on: macos-26-large + # Matrix legs write differently named outputs so they are combined + # instead of the last leg to finish overwriting the others. outputs: - status: ${{ steps.report-status.outputs.status }} + debug_status: ${{ steps.report-status.outputs.debug_status }} + release_status: ${{ steps.report-status.outputs.release_status }} strategy: fail-fast: false matrix: @@ -37,35 +40,34 @@ jobs: run: ls -lR /tmp/RNTesterBuild - name: Run E2E Tests id: run-tests + if: ${{ !inputs.fail-on-error }} continue-on-error: true uses: ./.github/actions/maestro-ios - with: + with: &test-inputs app-path: '/tmp/RNTesterBuild/RNTester.app' app-id: com.meta.RNTester.localDevelopment maestro-flow: ./packages/rn-tester/.maestro/ flavor: ${{ matrix.flavor }} device-model: iPhone-17-Pro device-os: iOS-26-2 + - name: Run E2E Tests + # Same step without continue-on-error, so the final attempt fails + # here. A step's continue-on-error expression doesn't see inputs. + id: run-tests-final + if: ${{ inputs.fail-on-error }} + uses: ./.github/actions/maestro-ios + with: *test-inputs - name: Report status id: report-status - if: ${{ always() && steps.run-tests.outcome == 'failure' }} - run: echo "status=failure" >> $GITHUB_OUTPUT - - report: - needs: test - if: always() - runs-on: ubuntu-latest - outputs: - status: ${{ steps.check.outputs.status }} - steps: - - id: check + if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }} + env: + FLAVOR: ${{ matrix.flavor }} run: | - if [[ "${{ needs.test.result }}" != "success" || - "${{ needs.test.outputs.status }}" == "failure" ]]; then - echo "status=failure" >> $GITHUB_OUTPUT - else - echo "status=success" >> $GITHUB_OUTPUT - fi - - name: Fail if needed - if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }} - run: exit 1 + flavor=$(echo "$FLAVOR" | tr "[:upper:]" "[:lower:]") + case "$flavor" in + debug | release) echo "${flavor}_status=failure" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::No job output for flavor '$FLAVOR'. Add a ${flavor}_status output to this job and to the workflow status output." + exit 1 + ;; + esac diff --git a/.github/workflows/e2e-ios-templateapp.yml b/.github/workflows/e2e-ios-templateapp.yml index e91b5ba0f72b..283a5d6ee1a3 100644 --- a/.github/workflows/e2e-ios-templateapp.yml +++ b/.github/workflows/e2e-ios-templateapp.yml @@ -11,8 +11,8 @@ on: default: false outputs: status: - description: 'The result of the E2E tests (success or failure)' - value: ${{ jobs.report.outputs.status }} + description: "'failure' if the E2E tests failed, empty otherwise" + value: ${{ jobs.test.outputs.debug_status || jobs.test.outputs.release_status }} jobs: build: @@ -100,8 +100,11 @@ jobs: test: needs: build runs-on: macos-26-large + # Matrix legs write differently named outputs so they are combined + # instead of the last leg to finish overwriting the others. outputs: - status: ${{ steps.report-status.outputs.status }} + debug_status: ${{ steps.report-status.outputs.debug_status }} + release_status: ${{ steps.report-status.outputs.release_status }} strategy: fail-fast: false matrix: @@ -149,35 +152,33 @@ jobs: node ./scripts/e2e/init-project-e2e.js --projectName RNTestProject --currentBranch $BRANCH --directory /tmp/RNTestProject --pathToLocalReactNative $REACT_NATIVE_PKG - name: Run E2E Tests id: run-tests + if: ${{ !inputs.fail-on-error }} continue-on-error: true uses: ./.github/actions/maestro-ios - with: + with: &test-inputs app-path: '/tmp/RNTestProjectBuild/RNTestProject.app' app-id: org.reactjs.native.example.RNTestProject maestro-flow: ./scripts/e2e/.maestro/ flavor: ${{ matrix.flavor }} working-directory: /tmp/RNTestProject + - name: Run E2E Tests + # Same step without continue-on-error, so the final attempt fails + # here. A step's continue-on-error expression doesn't see inputs. + id: run-tests-final + if: ${{ inputs.fail-on-error }} + uses: ./.github/actions/maestro-ios + with: *test-inputs - name: Report status id: report-status - if: ${{ always() && steps.run-tests.outcome == 'failure' }} - run: echo "status=failure" >> $GITHUB_OUTPUT - - report: - needs: [build, test] - if: always() - runs-on: ubuntu-latest - outputs: - status: ${{ steps.check.outputs.status }} - steps: - - id: check + if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }} + env: + FLAVOR: ${{ matrix.flavor }} run: | - if [[ "${{ needs.build.result }}" != "success" || - "${{ needs.test.result }}" != "success" || - "${{ needs.test.outputs.status }}" == "failure" ]]; then - echo "status=failure" >> $GITHUB_OUTPUT - else - echo "status=success" >> $GITHUB_OUTPUT - fi - - name: Fail if needed - if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }} - run: exit 1 + flavor=$(echo "$FLAVOR" | tr "[:upper:]" "[:lower:]") + case "$flavor" in + debug | release) echo "${flavor}_status=failure" >> "$GITHUB_OUTPUT" ;; + *) + echo "::error::No job output for flavor '$FLAVOR'. Add a ${flavor}_status output to this job and to the workflow status output." + exit 1 + ;; + esac diff --git a/.github/workflows/fantom-tests.yml b/.github/workflows/fantom-tests.yml index 82cdd462e6d3..85baed28f907 100644 --- a/.github/workflows/fantom-tests.yml +++ b/.github/workflows/fantom-tests.yml @@ -11,8 +11,8 @@ on: default: false outputs: status: - description: 'The result of the Fantom tests (success or failure)' - value: ${{ jobs.report.outputs.status }} + description: "'failure' if the Fantom tests failed, empty otherwise" + value: ${{ jobs.test.outputs.status }} jobs: test: @@ -28,27 +28,16 @@ jobs: uses: actions/checkout@v6 - name: Run Fantom Tests id: run-tests + if: ${{ !inputs.fail-on-error }} continue-on-error: true uses: ./.github/actions/run-fantom-tests + - name: Run Fantom Tests + # Same step without continue-on-error, so the final attempt fails + # here. A step's continue-on-error expression doesn't see inputs. + id: run-tests-final + if: ${{ inputs.fail-on-error }} + uses: ./.github/actions/run-fantom-tests - name: Report status id: report-status - if: ${{ always() && steps.run-tests.outcome == 'failure' }} + if: ${{ always() && (steps.run-tests.outcome == 'failure' || steps.run-tests-final.outcome == 'failure') }} run: echo "status=failure" >> $GITHUB_OUTPUT - - report: - needs: test - if: always() - runs-on: ubuntu-latest - outputs: - status: ${{ steps.check.outputs.status }} - steps: - - id: check - run: | - if [[ "${{ needs.test.outputs.status }}" == "failure" ]]; then - echo "status=failure" >> $GITHUB_OUTPUT - else - echo "status=success" >> $GITHUB_OUTPUT - fi - - name: Fail if needed - if: ${{ inputs.fail-on-error && steps.check.outputs.status == 'failure' }} - run: exit 1 diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index f724039a046f..ac75b5b8dfe0 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -197,13 +197,13 @@ jobs: test_e2e_ios_rntester_retry_1: needs: test_e2e_ios_rntester - if: ${{ always() && needs.test_e2e_ios_rntester.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_ios_rntester.result == 'failure' || needs.test_e2e_ios_rntester.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-ios-rntester.yml secrets: inherit test_e2e_ios_rntester_retry_2: needs: test_e2e_ios_rntester_retry_1 - if: ${{ always() && needs.test_e2e_ios_rntester_retry_1.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_ios_rntester_retry_1.result == 'failure' || needs.test_e2e_ios_rntester_retry_1.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-ios-rntester.yml with: fail-on-error: true @@ -224,13 +224,13 @@ jobs: test_e2e_ios_templateapp_retry_1: needs: test_e2e_ios_templateapp - if: ${{ always() && needs.test_e2e_ios_templateapp.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_ios_templateapp.result == 'failure' || needs.test_e2e_ios_templateapp.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-ios-templateapp.yml secrets: inherit test_e2e_ios_templateapp_retry_2: needs: test_e2e_ios_templateapp_retry_1 - if: ${{ always() && needs.test_e2e_ios_templateapp_retry_1.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_ios_templateapp_retry_1.result == 'failure' || needs.test_e2e_ios_templateapp_retry_1.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-ios-templateapp.yml with: fail-on-error: true @@ -284,7 +284,7 @@ jobs: test_e2e_android_templateapp_retry_1: needs: test_e2e_android_templateapp - if: ${{ always() && needs.test_e2e_android_templateapp.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_android_templateapp.result == 'failure' || needs.test_e2e_android_templateapp.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-android-templateapp.yml with: retry-attempt: 1 @@ -292,7 +292,7 @@ jobs: test_e2e_android_templateapp_retry_2: needs: test_e2e_android_templateapp_retry_1 - if: ${{ always() && needs.test_e2e_android_templateapp_retry_1.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_android_templateapp_retry_1.result == 'failure' || needs.test_e2e_android_templateapp_retry_1.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-android-templateapp.yml with: fail-on-error: true @@ -330,13 +330,13 @@ jobs: run_fantom_tests_retry_1: needs: run_fantom_tests - if: ${{ always() && needs.run_fantom_tests.outputs.status == 'failure' }} + if: ${{ always() && (needs.run_fantom_tests.result == 'failure' || needs.run_fantom_tests.outputs.status == 'failure') }} uses: ./.github/workflows/fantom-tests.yml secrets: inherit run_fantom_tests_retry_2: needs: run_fantom_tests_retry_1 - if: ${{ always() && needs.run_fantom_tests_retry_1.outputs.status == 'failure' }} + if: ${{ always() && (needs.run_fantom_tests_retry_1.result == 'failure' || needs.run_fantom_tests_retry_1.outputs.status == 'failure') }} uses: ./.github/workflows/fantom-tests.yml with: fail-on-error: true @@ -398,7 +398,7 @@ jobs: test_e2e_android_rntester_retry_1: needs: test_e2e_android_rntester - if: ${{ always() && needs.test_e2e_android_rntester.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_android_rntester.result == 'failure' || needs.test_e2e_android_rntester.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-android-rntester.yml with: retry-attempt: 1 @@ -406,7 +406,7 @@ jobs: test_e2e_android_rntester_retry_2: needs: test_e2e_android_rntester_retry_1 - if: ${{ always() && needs.test_e2e_android_rntester_retry_1.outputs.status == 'failure' }} + if: ${{ always() && (needs.test_e2e_android_rntester_retry_1.result == 'failure' || needs.test_e2e_android_rntester_retry_1.outputs.status == 'failure') }} uses: ./.github/workflows/e2e-android-rntester.yml with: fail-on-error: true