From de3f067372e747dd2ee6c2c205ccb8e62a5728a1 Mon Sep 17 00:00:00 2001 From: Jeppe B <2jepp9350@gmail.com> Date: Tue, 21 Jul 2026 16:03:57 +0200 Subject: [PATCH] Run frontend tests concurrently on GitHub-hosted runners (#209) Parallelize quality and Playwright jobs while preserving the existing test suite and required CI contracts. --- .github/workflows/tests.yml | 179 ++++++++++++++++++++------ scripts/run-playwright-full-slice.mjs | 140 +++++++++++++++++++- 2 files changed, 273 insertions(+), 46 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 4ac00f36..6e6053f0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -48,15 +48,17 @@ permissions: contents: read concurrency: - group: frontend-tests-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} + group: frontend-tests-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.event_name == 'push' && github.ref || github.run_id }} + cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }} # Repository variables used as CI runner and credit controls: # - FRONTEND_CI_STANDARD_RUNNER: JSON runs-on value for format/build/unit jobs. # - FRONTEND_CI_E2E_RUNNER: JSON runs-on value for Playwright jobs. # - FRONTEND_CI_PR_E2E_MAX_PARALLEL: numeric Playwright PR job parallelism. # - FRONTEND_CI_FULL_E2E_MAX_PARALLEL: numeric full-suite job parallelism. -# GitHub-hosted example: ["ubuntu-22.04"], with PR parallelism 2 and full parallelism 1. +# - FRONTEND_CI_FULL_E2E_CONCURRENT_MAX_PARALLEL: full-suite parallelism while PR E2E runs beside it. +# GitHub-hosted target: ["ubuntu-24.04"], with PR parallelism 10, concurrent full parallelism 26, +# and standalone scheduled full parallelism 36. This keeps the workflow peak at 36 hosted jobs. jobs: format-tests: runs-on: ${{ fromJSON(vars.FRONTEND_CI_STANDARD_RUNNER || '["self-hosted","Linux","X64","pleno","frontend"]') }} @@ -94,10 +96,15 @@ jobs: - name: Check frontend test formatting run: npm run format:tests:check - build-and-unit: - needs: format-tests + quality-checks: + name: Quality-${{ matrix.check }} runs-on: ${{ fromJSON(vars.FRONTEND_CI_STANDARD_RUNNER || '["self-hosted","Linux","X64","pleno","frontend"]') }} timeout-minutes: 30 + strategy: + fail-fast: false + max-parallel: 5 + matrix: + check: [lint, i18n, build, unit-fast, unit-serial] steps: - name: Repair self-hosted workspace permissions if: ${{ contains(vars.FRONTEND_CI_STANDARD_RUNNER || 'self-hosted', 'self-hosted') }} @@ -125,19 +132,58 @@ jobs: - name: Install dependencies run: npm ci --legacy-peer-deps - - name: Lint - run: npm run lint - - - name: Check i18n source consistency - run: npm run i18n:v2:check - - - name: Build sanity check - run: npm run build - - - name: Unit tests - run: npm run test:unit + - name: Run quality check + shell: bash env: - VITEST_BATCH_SIZE: 5 + MATRIX_CHECK: ${{ matrix.check }} + run: | + set -euo pipefail + case "$MATRIX_CHECK" in + lint) + npm run lint + ;; + i18n) + npm run i18n:v2:check + ;; + build) + npm run build + ;; + unit-fast) + npm run text:check-encoding + npm run test:unit:fast + ;; + unit-serial) + VITEST_BATCH_SIZE=5 npm run test:unit:serial + ;; + *) + echo "Unsupported quality check: $MATRIX_CHECK" >&2 + exit 1 + ;; + esac + + build-and-unit: + if: ${{ always() }} + name: Build and unit summary + needs: [format-tests, quality-checks] + runs-on: ${{ fromJSON(vars.FRONTEND_CI_STANDARD_RUNNER || '["self-hosted","Linux","X64","pleno","frontend"]') }} + timeout-minutes: 5 + steps: + - name: Verify quality jobs succeeded + shell: bash + env: + FORMAT_TESTS_RESULT: ${{ needs.format-tests.result }} + QUALITY_CHECKS_RESULT: ${{ needs.quality-checks.result }} + run: | + set -euo pipefail + failed=0 + for required_job in FORMAT_TESTS_RESULT QUALITY_CHECKS_RESULT; do + result="${!required_job:-missing}" + if [[ "$result" != "success" ]]; then + echo "${required_job}=${result}" >&2 + failed=1 + fi + done + exit "$failed" e2e-targeted: if: > @@ -329,7 +375,7 @@ jobs: fail-fast: false max-parallel: ${{ fromJSON(vars.FRONTEND_CI_PR_E2E_MAX_PARALLEL || '2') }} matrix: - suite: [core, changed] + suite: [changed-1-of-2, changed-2-of-2, smoke, pr, ct] project: [chromium-desktop, chromium-mobile] env: PLAYWRIGHT_ARTIFACT_NAMESPACE: e2e-pr-${{ matrix.suite }}-${{ matrix.project }} @@ -405,8 +451,11 @@ jobs: run: | set -euo pipefail case "$MATRIX_SUITE" in - core) suite_offset=0 ;; - changed) suite_offset=10 ;; + changed-1-of-2) suite_offset=0 ;; + changed-2-of-2) suite_offset=10 ;; + smoke) suite_offset=20 ;; + pr) suite_offset=30 ;; + ct) suite_offset=40 ;; *) echo "Unsupported Playwright PR suite: $MATRIX_SUITE" >&2; exit 1 ;; esac case "$MATRIX_PROJECT" in @@ -414,7 +463,7 @@ jobs: chromium-mobile) project_offset=2 ;; *) echo "Unsupported Playwright PR project: $MATRIX_PROJECT" >&2; exit 1 ;; esac - port_seed=$((20000 + (RUN_ID % 20000) + suite_offset + project_offset)) + port_seed=$((21000 + (RUN_ID % 20000) + suite_offset + project_offset)) lock_root="${PLAYWRIGHT_PORT_LOCK_ROOT:-/tmp/pleno-playwright-port-locks}" mkdir -p "$lock_root" chmod 1777 "$lock_root" 2>/dev/null || true @@ -485,13 +534,27 @@ jobs: } install_dependencies ulimit -n 16384 || true - if [[ "$MATRIX_SUITE" == "core" ]]; then - PLAYWRIGHT_ARTIFACT_NAMESPACE="${PLAYWRIGHT_ARTIFACT_NAMESPACE}-ct" npm run test:ct -- --project="$MATRIX_PROJECT" - npx playwright test --grep @smoke --project="$MATRIX_PROJECT" - npm run test:e2e:pr -- --core-only --project="$MATRIX_PROJECT" - else - npm run test:e2e:pr -- --changed-only --project="$MATRIX_PROJECT" --base="$DIFF_BASE_REF" --head="$DIFF_HEAD_REF" - fi + case "$MATRIX_SUITE" in + ct) + npm run test:ct -- --project="$MATRIX_PROJECT" + ;; + smoke) + npx playwright test --grep @smoke --project="$MATRIX_PROJECT" + ;; + pr) + npm run test:e2e:pr -- --core-only --project="$MATRIX_PROJECT" + ;; + changed-1-of-2) + npm run test:e2e:pr -- --changed-only --project="$MATRIX_PROJECT" --base="$DIFF_BASE_REF" --head="$DIFF_HEAD_REF" -- --shard=1/2 --pass-with-no-tests + ;; + changed-2-of-2) + npm run test:e2e:pr -- --changed-only --project="$MATRIX_PROJECT" --base="$DIFF_BASE_REF" --head="$DIFF_HEAD_REF" -- --shard=2/2 --pass-with-no-tests + ;; + *) + echo "Unsupported Playwright PR suite: $MATRIX_SUITE" >&2 + exit 1 + ;; + esac ' - name: Runner diagnostics after Playwright failure @@ -542,23 +605,28 @@ jobs: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.ref_name == github.event.repository.default_branch) && !(github.event_name == 'workflow_dispatch' && inputs.mode == 'targeted') && needs.build-and-unit.result == 'success' && - (github.event_name == 'schedule' || needs.e2e-pr.result == 'success') && ( github.event_name != 'workflow_dispatch' || inputs.mode == 'full' || needs.e2e-targeted.result == 'success' ) - needs: [build-and-unit, e2e-pr, e2e-targeted] - name: E2E-full-${{ matrix.browser_label }}-${{ matrix.device }}-${{ matrix.role }} + needs: [build-and-unit, e2e-targeted] + name: E2E-full-${{ matrix.browser_label }}-${{ matrix.device }}-${{ matrix.role }}-shard-${{ matrix.shard_index }}-of-${{ (matrix.role == 'superuser' || matrix.role == 'admin') && 2 || 1 }} runs-on: ${{ fromJSON(vars.FRONTEND_CI_E2E_RUNNER || '["self-hosted","Linux","X64","pleno","frontend","docker"]') }} timeout-minutes: 60 strategy: fail-fast: false - max-parallel: ${{ fromJSON(vars.FRONTEND_CI_FULL_E2E_MAX_PARALLEL || '1') }} + max-parallel: ${{ fromJSON(vars.FRONTEND_CI_FULL_E2E_MAX_PARALLEL || '36') > 0 && github.event_name == 'schedule' && fromJSON(vars.FRONTEND_CI_FULL_E2E_MAX_PARALLEL || '36') || fromJSON(vars.FRONTEND_CI_FULL_E2E_CONCURRENT_MAX_PARALLEL || '26') }} matrix: - browser: [chromium, webkit, firefox] device: [mobile, desktop, tablet] role: [superuser, admin, customer, subuser] + shard_index: [1, 2] + browser: [chromium, webkit, firefox] + exclude: + - role: customer + shard_index: 2 + - role: subuser + shard_index: 2 include: - browser: chromium browser_label: Chromium @@ -570,7 +638,7 @@ jobs: browser_label: Firefox browser_install: firefox env: - PLAYWRIGHT_ARTIFACT_NAMESPACE: e2e-full-${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }} + PLAYWRIGHT_ARTIFACT_NAMESPACE: e2e-full-${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}-shard-${{ matrix.shard_index }}-of-${{ (matrix.role == 'superuser' || matrix.role == 'admin') && 2 || 1 }} PLAYWRIGHT_REPORTER_MODE: line-html PLAYWRIGHT_WORKERS: 1 PLAYWRIGHT_VIDEO_MODE: off @@ -604,6 +672,8 @@ jobs: MATRIX_ROLE: ${{ matrix.role }} MATRIX_BROWSER: ${{ matrix.browser }} MATRIX_DEVICE: ${{ matrix.device }} + MATRIX_SHARD_INDEX: ${{ matrix.shard_index }} + MATRIX_SHARD_TOTAL: ${{ (matrix.role == 'superuser' || matrix.role == 'admin') && 2 || 1 }} RUN_ID: ${{ github.run_id }} run: | set -euo pipefail @@ -626,7 +696,12 @@ jobs: tablet) device_offset=3 ;; *) echo "Unsupported Playwright device: $MATRIX_DEVICE" >&2; exit 1 ;; esac - port_seed=$((20000 + (RUN_ID % 20000) + role_offset + browser_offset + device_offset)) + case "$MATRIX_SHARD_INDEX" in + 1) shard_offset=0 ;; + 2) shard_offset=400 ;; + *) echo "Unsupported Playwright shard index: $MATRIX_SHARD_INDEX" >&2; exit 1 ;; + esac + port_seed=$((22000 + (RUN_ID % 20000) + role_offset + browser_offset + device_offset + shard_offset)) lock_root="${PLAYWRIGHT_PORT_LOCK_ROOT:-/tmp/pleno-playwright-port-locks}" mkdir -p "$lock_root" chmod 1777 "$lock_root" 2>/dev/null || true @@ -660,8 +735,8 @@ jobs: exit 1 fi mkdir -p output/playwright - scripts/ci/runner-diagnostics.sh "before Playwright full ${MATRIX_BROWSER}/${MATRIX_DEVICE}/${MATRIX_ROLE}" -- "${docker_cmd[@]}" - SYSTEMD_INHIBIT_REASON="Frontend Playwright full ${MATRIX_BROWSER}/${MATRIX_DEVICE}/${MATRIX_ROLE}" \ + scripts/ci/runner-diagnostics.sh "before Playwright full ${MATRIX_BROWSER}/${MATRIX_DEVICE}/${MATRIX_ROLE}/shard-${MATRIX_SHARD_INDEX}-of-${MATRIX_SHARD_TOTAL}" -- "${docker_cmd[@]}" + SYSTEMD_INHIBIT_REASON="Frontend Playwright full ${MATRIX_BROWSER}/${MATRIX_DEVICE}/${MATRIX_ROLE}/shard-${MATRIX_SHARD_INDEX}-of-${MATRIX_SHARD_TOTAL}" \ scripts/ci/with-systemd-inhibit.sh "${docker_cmd[@]}" run --rm --ipc=host --network host \ --volume "$PWD:/source:ro" \ --volume "$PWD/output/playwright:/work/output/playwright" \ @@ -676,6 +751,8 @@ jobs: --env MATRIX_ROLE="$MATRIX_ROLE" \ --env MATRIX_BROWSER="$MATRIX_BROWSER" \ --env MATRIX_DEVICE="$MATRIX_DEVICE" \ + --env MATRIX_SHARD_INDEX="$MATRIX_SHARD_INDEX" \ + --env MATRIX_SHARD_TOTAL="$MATRIX_SHARD_TOTAL" \ mcr.microsoft.com/playwright:v1.58.2-noble \ bash -lc ' set -euo pipefail @@ -696,23 +773,45 @@ jobs: } install_dependencies ulimit -n 16384 || true - npm run test:e2e:full:slice -- --role="$MATRIX_ROLE" --project="$MATRIX_BROWSER-$MATRIX_DEVICE" + npm run test:e2e:full:slice -- --role="$MATRIX_ROLE" --project="$MATRIX_BROWSER-$MATRIX_DEVICE" --shard="$MATRIX_SHARD_INDEX/$MATRIX_SHARD_TOTAL" ' - name: Runner diagnostics after Playwright failure if: failure() || cancelled() continue-on-error: true - run: scripts/ci/runner-diagnostics.sh "after Playwright full ${{ matrix.browser }}/${{ matrix.device }}/${{ matrix.role }}" + run: scripts/ci/runner-diagnostics.sh "after Playwright full ${{ matrix.browser }}/${{ matrix.device }}/${{ matrix.role }}/shard-${{ matrix.shard_index }}-of-${{ (matrix.role == 'superuser' || matrix.role == 'admin') && 2 || 1 }}" - name: Upload Playwright report if: failure() || cancelled() continue-on-error: true uses: actions/upload-artifact@v4 with: - name: playwright-report-full-${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }} + name: playwright-report-full-${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}-shard-${{ matrix.shard_index }}-of-${{ (matrix.role == 'superuser' || matrix.role == 'admin') && 2 || 1 }} path: | output/playwright/${{ env.PLAYWRIGHT_ARTIFACT_NAMESPACE }}/report output/playwright/${{ env.PLAYWRIGHT_ARTIFACT_NAMESPACE }}/test-results output/playwright/test-lists/${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}.txt + output/playwright/test-lists/${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}-shard-${{ matrix.shard_index }}-of-${{ (matrix.role == 'superuser' || matrix.role == 'admin') && 2 || 1 }}.txt if-no-files-found: ignore retention-days: 1 + + full-e2e-summary: + if: > + always() && + (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.ref_name == github.event.repository.default_branch) && + !(github.event_name == 'workflow_dispatch' && inputs.mode == 'targeted') + name: Full E2E summary + needs: [e2e-full] + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - name: Verify full E2E succeeded + shell: bash + env: + FULL_E2E_RESULT: ${{ needs.e2e-full.result }} + run: | + set -euo pipefail + if [[ "$FULL_E2E_RESULT" != "success" ]]; then + echo "E2E_FULL_RESULT=${FULL_E2E_RESULT:-missing}" >&2 + exit 1 + fi diff --git a/scripts/run-playwright-full-slice.mjs b/scripts/run-playwright-full-slice.mjs index 38d271aa..71f52603 100644 --- a/scripts/run-playwright-full-slice.mjs +++ b/scripts/run-playwright-full-slice.mjs @@ -182,6 +182,11 @@ function parseCliArgs(argv) { role: "", project: "", listOnly: false, + shard: { + current: 1, + total: 1, + explicit: false, + }, }; for (let index = 0; index < optionArgs.length; index += 1) { @@ -214,12 +219,38 @@ function parseCliArgs(argv) { continue; } + if (arg === "--shard") { + options.shard = parseShard(optionArgs[index + 1] || ""); + index += 1; + continue; + } + + if (arg.startsWith("--shard=")) { + options.shard = parseShard(arg.slice("--shard=".length)); + continue; + } + throw new Error(`Unknown argument: ${arg}`); } return { options, forwardedArgs }; } +function parseShard(value) { + const match = value.match(/^(\d+)\/(\d+)$/u); + if (!match) { + throw new Error("--shard must use the form /, for example 1/2."); + } + + const current = Number(match[1]); + const total = Number(match[2]); + if (!Number.isSafeInteger(current) || !Number.isSafeInteger(total) || total < 1 || current < 1 || current > total) { + throw new Error("--shard requires positive integers with current less than or equal to total."); + } + + return { current, total, explicit: true }; +} + function validateOptions(options, forwardedArgs) { if (!roles.includes(options.role)) { throw new Error(`--role must be one of: ${roles.join(", ")}`); @@ -234,11 +265,16 @@ function validateOptions(options, forwardedArgs) { } for (const arg of forwardedArgs) { - if (arg === "--list" || arg === "--test-list" || arg === "--project") { + if (arg === "--list" || arg === "--test-list" || arg === "--project" || arg === "--shard") { throw new Error(`Forwarded Playwright argument is not supported here: ${arg}`); } - if (arg.startsWith("--list=") || arg.startsWith("--test-list=") || arg.startsWith("--project=")) { + if ( + arg.startsWith("--list=") || + arg.startsWith("--test-list=") || + arg.startsWith("--project=") || + arg.startsWith("--shard=") + ) { throw new Error(`Forwarded Playwright argument is not supported here: ${arg}`); } } @@ -317,6 +353,32 @@ async function listProjectTests(project, forwardedArgs) { return stdout; } +async function listProjectShardTests(project, testListPath, shard, forwardedArgs) { + const { stdout, stderr } = await execFileAsync( + process.execPath, + [ + playwrightCliPath, + "test", + "--list", + "--reporter=list", + `--project=${project}`, + `--test-list=${testListPath}`, + `--shard=${shard.current}/${shard.total}`, + ...forwardedArgs, + ], + { + cwd: workingDirectory, + maxBuffer: 64 * 1024 * 1024, + } + ); + + if (stderr.trim()) { + process.stderr.write(stderr); + } + + return stdout; +} + const getTestListDirectory = () => process.env.PLAYWRIGHT_TEST_LIST_DIR || path.join(workingDirectory, "output", "playwright", "test-lists"); @@ -328,6 +390,10 @@ export function getLegacyTestListPath(role, project) { return path.join(getTestListDirectory(), `${role}-${project}.txt`); } +export function getShardTestListPath(project, role, shard) { + return path.join(getTestListDirectory(), `${project}-${role}-shard-${shard.current}-of-${shard.total}.txt`); +} + export async function writeTestList(role, project, matchingTests) { const outputDirectory = getTestListDirectory(); await fs.mkdir(outputDirectory, { recursive: true }); @@ -340,6 +406,40 @@ export async function writeTestList(role, project, matchingTests) { return testListPath; } +export async function writeShardTestList(role, project, shard, matchingTests) { + const outputDirectory = getTestListDirectory(); + await fs.mkdir(outputDirectory, { recursive: true }); + + const contents = `${matchingTests.map((testEntry) => testEntry.listLine).join("\n")}\n`; + const testListPath = getShardTestListPath(project, role, shard); + await fs.writeFile(testListPath, contents, "utf8"); + return testListPath; +} + +function getTestIdentity(testEntry) { + return `${testEntry.relativeFile}:${testEntry.lineNumber}:${testEntry.columnNumber} › ${testEntry.title}`; +} + +function validateShardSelection(role, matchingTests, shardTests) { + if (shardTests.length === 0) { + throw new Error(`Shard contains no ${role} tests.`); + } + + const matchingIdentities = new Set(matchingTests.map(getTestIdentity)); + const shardIdentities = new Set(); + + for (const testEntry of shardTests) { + const identity = getTestIdentity(testEntry); + if (!matchingIdentities.has(identity)) { + throw new Error(`Shard selected a test outside the ${role} role: ${identity}`); + } + if (shardIdentities.has(identity)) { + throw new Error(`Shard selected a duplicate test: ${identity}`); + } + shardIdentities.add(identity); + } +} + async function runPlaywright(project, testListPath, forwardedArgs) { const args = ["test", `--project=${project}`, `--test-list=${testListPath}`, ...forwardedArgs]; @@ -392,16 +492,44 @@ export async function main(argv = process.argv.slice(2)) { return; } - if (options.listOnly) { + if (options.listOnly && !options.shard.explicit) { for (const testEntry of matchingTests) { console.log(testEntry.listLine); } return; } - const testListPath = await writeTestList(options.role, options.project, matchingTests); - console.log(`Using generated test list: ${path.relative(workingDirectory, testListPath)}`); - await runPlaywright(options.project, testListPath, forwardedArgs); + const roleTestListPath = await writeTestList(options.role, options.project, matchingTests); + let executionTests = matchingTests; + let executionTestListPath = roleTestListPath; + + if (options.shard.explicit) { + if (options.shard.total > 1) { + const shardListOutput = await listProjectShardTests( + options.project, + roleTestListPath, + options.shard, + forwardedArgs + ); + executionTests = parseListedTests(shardListOutput); + validateShardSelection(options.role, matchingTests, executionTests); + } + + executionTestListPath = await writeShardTestList(options.role, options.project, options.shard, executionTests); + console.log( + `Resolved ${executionTests.length} test(s) for ${options.role} shard ${options.shard.current}/${options.shard.total} on ${options.project}.` + ); + } + + if (options.listOnly) { + for (const testEntry of executionTests) { + console.log(testEntry.listLine); + } + return; + } + + console.log(`Using generated test list: ${path.relative(workingDirectory, executionTestListPath)}`); + await runPlaywright(options.project, executionTestListPath, forwardedArgs); } async function isDirectRun() {