copenhagentruckwash/pleno-vue
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run frontend tests concurrently on GitHub-hosted runners (#209)

Browse Source 
Parallelize quality and Playwright jobs while preserving the existing test suite and required CI contracts.
This commit is contained in:
Jeppe B
2026-07-21 16:03:57 +02:00
committed by GitHub
parent a0c11e4bb7
commit de3f067372
2 changed files with 273 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/tests.yml
+139 -40
View File
@@ -48,15 +48,17 @@ permissions:
contents: read
concurrency:
group: frontend-tests-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
group: frontend-tests-${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.event.pull_request.number || github.event_name == 'push' && github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' || github.event_name == 'push' }}
# Repository variables used as CI runner and credit controls:
# - FRONTEND_CI_STANDARD_RUNNER: JSON runs-on value for format/build/unit jobs.
# - FRONTEND_CI_E2E_RUNNER: JSON runs-on value for Playwright jobs.
# - FRONTEND_CI_PR_E2E_MAX_PARALLEL: numeric Playwright PR job parallelism.
# - FRONTEND_CI_FULL_E2E_MAX_PARALLEL: numeric full-suite job parallelism.
# GitHub-hosted example: ["ubuntu-22.04"], with PR parallelism 2 and full parallelism 1.
# - FRONTEND_CI_FULL_E2E_CONCURRENT_MAX_PARALLEL: full-suite parallelism while PR E2E runs beside it.
# GitHub-hosted target: ["ubuntu-24.04"], with PR parallelism 10, concurrent full parallelism 26,
# and standalone scheduled full parallelism 36. This keeps the workflow peak at 36 hosted jobs.
jobs:
format-tests:
runs-on: ${{ fromJSON(vars.FRONTEND_CI_STANDARD_RUNNER || '["self-hosted","Linux","X64","pleno","frontend"]') }}
@@ -94,10 +96,15 @@ jobs:
- name: Check frontend test formatting
run: npm run format:tests:check
build-and-unit:
needs: format-tests
quality-checks:
name: Quality-${{ matrix.check }}
runs-on: ${{ fromJSON(vars.FRONTEND_CI_STANDARD_RUNNER || '["self-hosted","Linux","X64","pleno","frontend"]') }}
timeout-minutes: 30
strategy:
fail-fast: false
max-parallel: 5
matrix:
check: [lint, i18n, build, unit-fast, unit-serial]
steps:
- name: Repair self-hosted workspace permissions
if: ${{ contains(vars.FRONTEND_CI_STANDARD_RUNNER || 'self-hosted', 'self-hosted') }}
@@ -125,19 +132,58 @@ jobs:
- name: Install dependencies
run: npm ci --legacy-peer-deps
- name: Lint
run: npm run lint
- name: Check i18n source consistency
run: npm run i18n:v2:check
- name: Build sanity check
run: npm run build
- name: Unit tests
run: npm run test:unit
- name: Run quality check
shell: bash
env:
VITEST_BATCH_SIZE: 5
MATRIX_CHECK: ${{ matrix.check }}
run: |
set -euo pipefail
case "$MATRIX_CHECK" in
lint)
npm run lint
;;
i18n)
npm run i18n:v2:check
;;
build)
npm run build
;;
unit-fast)
npm run text:check-encoding
npm run test:unit:fast
;;
unit-serial)
VITEST_BATCH_SIZE=5 npm run test:unit:serial
;;
*)
echo "Unsupported quality check: $MATRIX_CHECK" >&2
exit 1
;;
esac
build-and-unit:
if: ${{ always() }}
name: Build and unit summary
needs: [format-tests, quality-checks]
runs-on: ${{ fromJSON(vars.FRONTEND_CI_STANDARD_RUNNER || '["self-hosted","Linux","X64","pleno","frontend"]') }}
timeout-minutes: 5
steps:
- name: Verify quality jobs succeeded
shell: bash
env:
FORMAT_TESTS_RESULT: ${{ needs.format-tests.result }}
QUALITY_CHECKS_RESULT: ${{ needs.quality-checks.result }}
run: |
set -euo pipefail
failed=0
for required_job in FORMAT_TESTS_RESULT QUALITY_CHECKS_RESULT; do
result="${!required_job:-missing}"
if [[ "$result" != "success" ]]; then
echo "${required_job}=${result}" >&2
failed=1
fi
done
exit "$failed"
e2e-targeted:
if: >
@@ -329,7 +375,7 @@ jobs:
fail-fast: false
max-parallel: ${{ fromJSON(vars.FRONTEND_CI_PR_E2E_MAX_PARALLEL || '2') }}
matrix:
suite: [core, changed]
suite: [changed-1-of-2, changed-2-of-2, smoke, pr, ct]
project: [chromium-desktop, chromium-mobile]
env:
PLAYWRIGHT_ARTIFACT_NAMESPACE: e2e-pr-${{ matrix.suite }}-${{ matrix.project }}
@@ -405,8 +451,11 @@ jobs:
run: |
set -euo pipefail
case "$MATRIX_SUITE" in
core) suite_offset=0 ;;
changed) suite_offset=10 ;;
changed-1-of-2) suite_offset=0 ;;
changed-2-of-2) suite_offset=10 ;;
smoke) suite_offset=20 ;;
pr) suite_offset=30 ;;
ct) suite_offset=40 ;;
*) echo "Unsupported Playwright PR suite: $MATRIX_SUITE" >&2; exit 1 ;;
esac
case "$MATRIX_PROJECT" in
@@ -414,7 +463,7 @@ jobs:
chromium-mobile) project_offset=2 ;;
*) echo "Unsupported Playwright PR project: $MATRIX_PROJECT" >&2; exit 1 ;;
esac
port_seed=$((20000 + (RUN_ID % 20000) + suite_offset + project_offset))
port_seed=$((21000 + (RUN_ID % 20000) + suite_offset + project_offset))
lock_root="${PLAYWRIGHT_PORT_LOCK_ROOT:-/tmp/pleno-playwright-port-locks}"
mkdir -p "$lock_root"
chmod 1777 "$lock_root" 2>/dev/null || true
@@ -485,13 +534,27 @@ jobs:
}
install_dependencies
ulimit -n 16384 || true
if [[ "$MATRIX_SUITE" == "core" ]]; then
PLAYWRIGHT_ARTIFACT_NAMESPACE="${PLAYWRIGHT_ARTIFACT_NAMESPACE}-ct" npm run test:ct -- --project="$MATRIX_PROJECT"
npx playwright test --grep @smoke --project="$MATRIX_PROJECT"
npm run test:e2e:pr -- --core-only --project="$MATRIX_PROJECT"
else
npm run test:e2e:pr -- --changed-only --project="$MATRIX_PROJECT" --base="$DIFF_BASE_REF" --head="$DIFF_HEAD_REF"
fi
case "$MATRIX_SUITE" in
ct)
npm run test:ct -- --project="$MATRIX_PROJECT"
;;
smoke)
npx playwright test --grep @smoke --project="$MATRIX_PROJECT"
;;
pr)
npm run test:e2e:pr -- --core-only --project="$MATRIX_PROJECT"
;;
changed-1-of-2)
npm run test:e2e:pr -- --changed-only --project="$MATRIX_PROJECT" --base="$DIFF_BASE_REF" --head="$DIFF_HEAD_REF" -- --shard=1/2 --pass-with-no-tests
;;
changed-2-of-2)
npm run test:e2e:pr -- --changed-only --project="$MATRIX_PROJECT" --base="$DIFF_BASE_REF" --head="$DIFF_HEAD_REF" -- --shard=2/2 --pass-with-no-tests
;;
*)
echo "Unsupported Playwright PR suite: $MATRIX_SUITE" >&2
exit 1
;;
esac
'
- name: Runner diagnostics after Playwright failure
@@ -542,23 +605,28 @@ jobs:
(github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.ref_name == github.event.repository.default_branch) &&
!(github.event_name == 'workflow_dispatch' && inputs.mode == 'targeted') &&
needs.build-and-unit.result == 'success' &&
(github.event_name == 'schedule' || needs.e2e-pr.result == 'success') &&
(
github.event_name != 'workflow_dispatch' ||
inputs.mode == 'full' ||
needs.e2e-targeted.result == 'success'
)
needs: [build-and-unit, e2e-pr, e2e-targeted]
name: E2E-full-${{ matrix.browser_label }}-${{ matrix.device }}-${{ matrix.role }}
needs: [build-and-unit, e2e-targeted]
name: E2E-full-${{ matrix.browser_label }}-${{ matrix.device }}-${{ matrix.role }}-shard-${{ matrix.shard_index }}-of-${{ (matrix.role == 'superuser' || matrix.role == 'admin') && 2 || 1 }}
runs-on: ${{ fromJSON(vars.FRONTEND_CI_E2E_RUNNER || '["self-hosted","Linux","X64","pleno","frontend","docker"]') }}
timeout-minutes: 60
strategy:
fail-fast: false
max-parallel: ${{ fromJSON(vars.FRONTEND_CI_FULL_E2E_MAX_PARALLEL || '1') }}
max-parallel: ${{ fromJSON(vars.FRONTEND_CI_FULL_E2E_MAX_PARALLEL || '36') > 0 && github.event_name == 'schedule' && fromJSON(vars.FRONTEND_CI_FULL_E2E_MAX_PARALLEL || '36') || fromJSON(vars.FRONTEND_CI_FULL_E2E_CONCURRENT_MAX_PARALLEL || '26') }}
matrix:
browser: [chromium, webkit, firefox]
device: [mobile, desktop, tablet]
role: [superuser, admin, customer, subuser]
shard_index: [1, 2]
browser: [chromium, webkit, firefox]
exclude:
- role: customer
shard_index: 2
- role: subuser
shard_index: 2
include:
- browser: chromium
browser_label: Chromium
@@ -570,7 +638,7 @@ jobs:
browser_label: Firefox
browser_install: firefox
env:
PLAYWRIGHT_ARTIFACT_NAMESPACE: e2e-full-${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}
PLAYWRIGHT_ARTIFACT_NAMESPACE: e2e-full-${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}-shard-${{ matrix.shard_index }}-of-${{ (matrix.role == 'superuser' || matrix.role == 'admin') && 2 || 1 }}
PLAYWRIGHT_REPORTER_MODE: line-html
PLAYWRIGHT_WORKERS: 1
PLAYWRIGHT_VIDEO_MODE: off
@@ -604,6 +672,8 @@ jobs:
MATRIX_ROLE: ${{ matrix.role }}
MATRIX_BROWSER: ${{ matrix.browser }}
MATRIX_DEVICE: ${{ matrix.device }}
MATRIX_SHARD_INDEX: ${{ matrix.shard_index }}
MATRIX_SHARD_TOTAL: ${{ (matrix.role == 'superuser' || matrix.role == 'admin') && 2 || 1 }}
RUN_ID: ${{ github.run_id }}
run: |
set -euo pipefail
@@ -626,7 +696,12 @@ jobs:
tablet) device_offset=3 ;;
*) echo "Unsupported Playwright device: $MATRIX_DEVICE" >&2; exit 1 ;;
esac
port_seed=$((20000 + (RUN_ID % 20000) + role_offset + browser_offset + device_offset))
case "$MATRIX_SHARD_INDEX" in
1) shard_offset=0 ;;
2) shard_offset=400 ;;
*) echo "Unsupported Playwright shard index: $MATRIX_SHARD_INDEX" >&2; exit 1 ;;
esac
port_seed=$((22000 + (RUN_ID % 20000) + role_offset + browser_offset + device_offset + shard_offset))
lock_root="${PLAYWRIGHT_PORT_LOCK_ROOT:-/tmp/pleno-playwright-port-locks}"
mkdir -p "$lock_root"
chmod 1777 "$lock_root" 2>/dev/null || true
@@ -660,8 +735,8 @@ jobs:
exit 1
fi
mkdir -p output/playwright
scripts/ci/runner-diagnostics.sh "before Playwright full ${MATRIX_BROWSER}/${MATRIX_DEVICE}/${MATRIX_ROLE}" -- "${docker_cmd[@]}"
SYSTEMD_INHIBIT_REASON="Frontend Playwright full ${MATRIX_BROWSER}/${MATRIX_DEVICE}/${MATRIX_ROLE}" \
scripts/ci/runner-diagnostics.sh "before Playwright full ${MATRIX_BROWSER}/${MATRIX_DEVICE}/${MATRIX_ROLE}/shard-${MATRIX_SHARD_INDEX}-of-${MATRIX_SHARD_TOTAL}" -- "${docker_cmd[@]}"
SYSTEMD_INHIBIT_REASON="Frontend Playwright full ${MATRIX_BROWSER}/${MATRIX_DEVICE}/${MATRIX_ROLE}/shard-${MATRIX_SHARD_INDEX}-of-${MATRIX_SHARD_TOTAL}" \
scripts/ci/with-systemd-inhibit.sh "${docker_cmd[@]}" run --rm --ipc=host --network host \
--volume "$PWD:/source:ro" \
--volume "$PWD/output/playwright:/work/output/playwright" \
@@ -676,6 +751,8 @@ jobs:
--env MATRIX_ROLE="$MATRIX_ROLE" \
--env MATRIX_BROWSER="$MATRIX_BROWSER" \
--env MATRIX_DEVICE="$MATRIX_DEVICE" \
--env MATRIX_SHARD_INDEX="$MATRIX_SHARD_INDEX" \
--env MATRIX_SHARD_TOTAL="$MATRIX_SHARD_TOTAL" \
mcr.microsoft.com/playwright:v1.58.2-noble \
bash -lc '
set -euo pipefail
@@ -696,23 +773,45 @@ jobs:
}
install_dependencies
ulimit -n 16384 || true
npm run test:e2e:full:slice -- --role="$MATRIX_ROLE" --project="$MATRIX_BROWSER-$MATRIX_DEVICE"
npm run test:e2e:full:slice -- --role="$MATRIX_ROLE" --project="$MATRIX_BROWSER-$MATRIX_DEVICE" --shard="$MATRIX_SHARD_INDEX/$MATRIX_SHARD_TOTAL"
'
- name: Runner diagnostics after Playwright failure
if: failure() || cancelled()
continue-on-error: true
run: scripts/ci/runner-diagnostics.sh "after Playwright full ${{ matrix.browser }}/${{ matrix.device }}/${{ matrix.role }}"
run: scripts/ci/runner-diagnostics.sh "after Playwright full ${{ matrix.browser }}/${{ matrix.device }}/${{ matrix.role }}/shard-${{ matrix.shard_index }}-of-${{ (matrix.role == 'superuser' || matrix.role == 'admin') && 2 || 1 }}"
- name: Upload Playwright report
if: failure() || cancelled()
continue-on-error: true
uses: actions/upload-artifact@v4
with:
name: playwright-report-full-${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}
name: playwright-report-full-${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}-shard-${{ matrix.shard_index }}-of-${{ (matrix.role == 'superuser' || matrix.role == 'admin') && 2 || 1 }}
path: |
output/playwright/${{ env.PLAYWRIGHT_ARTIFACT_NAMESPACE }}/report
output/playwright/${{ env.PLAYWRIGHT_ARTIFACT_NAMESPACE }}/test-results
output/playwright/test-lists/${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}.txt
output/playwright/test-lists/${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}-shard-${{ matrix.shard_index }}-of-${{ (matrix.role == 'superuser' || matrix.role == 'admin') && 2 || 1 }}.txt
if-no-files-found: ignore
retention-days: 1
full-e2e-summary:
if: >
always() &&
(github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.ref_name == github.event.repository.default_branch) &&
!(github.event_name == 'workflow_dispatch' && inputs.mode == 'targeted')
name: Full E2E summary
needs: [e2e-full]
runs-on: ubuntu-24.04
timeout-minutes: 5
steps:
- name: Verify full E2E succeeded
shell: bash
env:
FULL_E2E_RESULT: ${{ needs.e2e-full.result }}
run: |
set -euo pipefail
if [[ "$FULL_E2E_RESULT" != "success" ]]; then
echo "E2E_FULL_RESULT=${FULL_E2E_RESULT:-missing}" >&2
exit 1
fi
scripts/run-playwright-full-slice.mjs
+134 -6
View File
@@ -182,6 +182,11 @@ function parseCliArgs(argv) {
role: "",
project: "",
listOnly: false,
shard: {
current: 1,
total: 1,
explicit: false,
},
};
for (let index = 0; index < optionArgs.length; index += 1) {
@@ -214,12 +219,38 @@ function parseCliArgs(argv) {
continue;
}
if (arg === "--shard") {
options.shard = parseShard(optionArgs[index + 1] || "");
index += 1;
continue;
}
if (arg.startsWith("--shard=")) {
options.shard = parseShard(arg.slice("--shard=".length));
continue;
}
throw new Error(`Unknown argument: ${arg}`);
}
return { options, forwardedArgs };
}
function parseShard(value) {
const match = value.match(/^(\d+)\/(\d+)$/u);
if (!match) {
throw new Error("--shard must use the form <current>/<total>, for example 1/2.");
}
const current = Number(match[1]);
const total = Number(match[2]);
if (!Number.isSafeInteger(current) || !Number.isSafeInteger(total) || total < 1 || current < 1 || current > total) {
throw new Error("--shard requires positive integers with current less than or equal to total.");
}
return { current, total, explicit: true };
}
function validateOptions(options, forwardedArgs) {
if (!roles.includes(options.role)) {
throw new Error(`--role must be one of: ${roles.join(", ")}`);
@@ -234,11 +265,16 @@ function validateOptions(options, forwardedArgs) {
}
for (const arg of forwardedArgs) {
if (arg === "--list" || arg === "--test-list" || arg === "--project") {
if (arg === "--list" || arg === "--test-list" || arg === "--project" || arg === "--shard") {
throw new Error(`Forwarded Playwright argument is not supported here: ${arg}`);
}
if (arg.startsWith("--list=") || arg.startsWith("--test-list=") || arg.startsWith("--project=")) {
if (
arg.startsWith("--list=") ||
arg.startsWith("--test-list=") ||
arg.startsWith("--project=") ||
arg.startsWith("--shard=")
) {
throw new Error(`Forwarded Playwright argument is not supported here: ${arg}`);
}
}
@@ -317,6 +353,32 @@ async function listProjectTests(project, forwardedArgs) {
return stdout;
}
async function listProjectShardTests(project, testListPath, shard, forwardedArgs) {
const { stdout, stderr } = await execFileAsync(
process.execPath,
[
playwrightCliPath,
"test",
"--list",
"--reporter=list",
`--project=${project}`,
`--test-list=${testListPath}`,
`--shard=${shard.current}/${shard.total}`,
...forwardedArgs,
],
{
cwd: workingDirectory,
maxBuffer: 64 * 1024 * 1024,
}
);
if (stderr.trim()) {
process.stderr.write(stderr);
}
return stdout;
}
const getTestListDirectory = () =>
process.env.PLAYWRIGHT_TEST_LIST_DIR || path.join(workingDirectory, "output", "playwright", "test-lists");
@@ -328,6 +390,10 @@ export function getLegacyTestListPath(role, project) {
return path.join(getTestListDirectory(), `${role}-${project}.txt`);
}
export function getShardTestListPath(project, role, shard) {
return path.join(getTestListDirectory(), `${project}-${role}-shard-${shard.current}-of-${shard.total}.txt`);
}
export async function writeTestList(role, project, matchingTests) {
const outputDirectory = getTestListDirectory();
await fs.mkdir(outputDirectory, { recursive: true });
@@ -340,6 +406,40 @@ export async function writeTestList(role, project, matchingTests) {
return testListPath;
}
export async function writeShardTestList(role, project, shard, matchingTests) {
const outputDirectory = getTestListDirectory();
await fs.mkdir(outputDirectory, { recursive: true });
const contents = `${matchingTests.map((testEntry) => testEntry.listLine).join("\n")}\n`;
const testListPath = getShardTestListPath(project, role, shard);
await fs.writeFile(testListPath, contents, "utf8");
return testListPath;
}
function getTestIdentity(testEntry) {
return `${testEntry.relativeFile}:${testEntry.lineNumber}:${testEntry.columnNumber} › ${testEntry.title}`;
}
function validateShardSelection(role, matchingTests, shardTests) {
if (shardTests.length === 0) {
throw new Error(`Shard contains no ${role} tests.`);
}
const matchingIdentities = new Set(matchingTests.map(getTestIdentity));
const shardIdentities = new Set();
for (const testEntry of shardTests) {
const identity = getTestIdentity(testEntry);
if (!matchingIdentities.has(identity)) {
throw new Error(`Shard selected a test outside the ${role} role: ${identity}`);
}
if (shardIdentities.has(identity)) {
throw new Error(`Shard selected a duplicate test: ${identity}`);
}
shardIdentities.add(identity);
}
}
async function runPlaywright(project, testListPath, forwardedArgs) {
const args = ["test", `--project=${project}`, `--test-list=${testListPath}`, ...forwardedArgs];
@@ -392,16 +492,44 @@ export async function main(argv = process.argv.slice(2)) {
return;
}
if (options.listOnly) {
if (options.listOnly && !options.shard.explicit) {
for (const testEntry of matchingTests) {
console.log(testEntry.listLine);
}
return;
}
const testListPath = await writeTestList(options.role, options.project, matchingTests);
console.log(`Using generated test list: ${path.relative(workingDirectory, testListPath)}`);
await runPlaywright(options.project, testListPath, forwardedArgs);
const roleTestListPath = await writeTestList(options.role, options.project, matchingTests);
let executionTests = matchingTests;
let executionTestListPath = roleTestListPath;
if (options.shard.explicit) {
if (options.shard.total > 1) {
const shardListOutput = await listProjectShardTests(
options.project,
roleTestListPath,
options.shard,
forwardedArgs
);
executionTests = parseListedTests(shardListOutput);
validateShardSelection(options.role, matchingTests, executionTests);
}
executionTestListPath = await writeShardTestList(options.role, options.project, options.shard, executionTests);
console.log(
`Resolved ${executionTests.length} test(s) for ${options.role} shard ${options.shard.current}/${options.shard.total} on ${options.project}.`
);
}
if (options.listOnly) {
for (const testEntry of executionTests) {
console.log(testEntry.listLine);
}
return;
}
console.log(`Using generated test list: ${path.relative(workingDirectory, executionTestListPath)}`);
await runPlaywright(options.project, executionTestListPath, forwardedArgs);
}
async function isDirectRun() {