Group full E2E slices by browser device and role

.github/workflows/tests.yml

@@ -259,28 +259,28 @@ jobs:
       needs.build-and-unit.result == 'success' &&
       (github.event_name == 'schedule' || needs.e2e-pr.result == 'success')
     needs: [build-and-unit, e2e-pr]
-    name: E2E-full-${{ matrix.role }}-${{ matrix.browser_label }}-${{ matrix.device }}
+    name: E2E-full-${{ matrix.browser_label }}-${{ matrix.device }}-${{ matrix.role }}
     runs-on: [self-hosted, Linux, X64, pleno, frontend, docker]
     timeout-minutes: 60
     strategy:
       fail-fast: false
       max-parallel: 4
       matrix:
-        role: [customer, subuser, admin, superuser]
-        browser: [chromium, firefox, webkit]
-        device: [mobile, tablet, desktop]
+        browser: [chromium, webkit, firefox]
+        device: [mobile, desktop, tablet]
+        role: [superuser, admin, customer, subuser]
         include:
           - browser: chromium
             browser_label: Chromium
             browser_install: chromium
-          - browser: firefox
-            browser_label: Firefox
-            browser_install: firefox
           - browser: webkit
             browser_label: WebKit
             browser_install: webkit
+          - browser: firefox
+            browser_label: Firefox
+            browser_install: firefox
     env:
-      PLAYWRIGHT_ARTIFACT_NAMESPACE: e2e-full-${{ matrix.role }}-${{ matrix.browser }}-${{ matrix.device }}
+      PLAYWRIGHT_ARTIFACT_NAMESPACE: e2e-full-${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}
       PLAYWRIGHT_REPORTER_MODE: line-html
     steps:
       - name: Repair self-hosted workspace permissions
@@ -315,22 +315,22 @@ jobs:
         run: |
           set -euo pipefail
           case "$MATRIX_ROLE" in
-            customer) role_offset=0 ;;
-            subuser) role_offset=100 ;;
-            admin) role_offset=200 ;;
-            superuser) role_offset=300 ;;
+            superuser) role_offset=0 ;;
+            admin) role_offset=100 ;;
+            customer) role_offset=200 ;;
+            subuser) role_offset=300 ;;
             *) echo "Unsupported Playwright role: $MATRIX_ROLE" >&2; exit 1 ;;
           esac
           case "$MATRIX_BROWSER" in
             chromium) browser_offset=0 ;;
-            firefox) browser_offset=30 ;;
-            webkit) browser_offset=60 ;;
+            webkit) browser_offset=30 ;;
+            firefox) browser_offset=60 ;;
             *) echo "Unsupported Playwright browser: $MATRIX_BROWSER" >&2; exit 1 ;;
           esac
           case "$MATRIX_DEVICE" in
             mobile) device_offset=1 ;;
-            tablet) device_offset=2 ;;
-            desktop) device_offset=3 ;;
+            desktop) device_offset=2 ;;
+            tablet) device_offset=3 ;;
             *) echo "Unsupported Playwright device: $MATRIX_DEVICE" >&2; exit 1 ;;
           esac
           port_seed=$((20000 + (RUN_ID % 20000) + role_offset + browser_offset + device_offset))
@@ -392,10 +392,10 @@ jobs:
         continue-on-error: true
         uses: actions/upload-artifact@v4
         with:
-          name: playwright-report-full-${{ matrix.role }}-${{ matrix.browser }}-${{ matrix.device }}
+          name: playwright-report-full-${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}
           path: |
             output/playwright/${{ env.PLAYWRIGHT_ARTIFACT_NAMESPACE }}/report
             output/playwright/${{ env.PLAYWRIGHT_ARTIFACT_NAMESPACE }}/test-results
-            output/playwright/test-lists/${{ matrix.role }}-${{ matrix.browser }}-${{ matrix.device }}.txt
+            output/playwright/test-lists/${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}.txt
           if-no-files-found: ignore
           retention-days: 1

README.md

@@ -96,16 +96,22 @@ Artifacts and summaries:
 
 ## Playwright Full E2E
 
-Run the permanent grouped full-suite entrypoint with a hard max of 5 total workers across:
+Run the permanent grouped full-suite entrypoint with a hard max of 5 total workers across the browser-engine groups:
 
-- `chromium-(desktop|tablet|mobile)`
-- `firefox-(desktop|tablet|mobile)`
-- `webkit-(desktop|tablet|mobile)`
+- Chromium
+- WebKit
+- Firefox
 
 ```sh
 npm run test:e2e:ci
 ```
 
+The full CI matrix is ordered by browser engine, then device class, then user role:
+
+- browsers: `chromium`, `webkit`, `firefox`
+- devices: `mobile`, `desktop`, `tablet`
+- roles: `superuser`, `admin`, `customer`, `subuser`
+
 Run a single full-suite slice for one role and one Playwright project:
 
 ```sh
@@ -137,7 +143,8 @@ Artifacts and summaries:
 - `output/playwright/ci-parallel-chromium/report/index.html`
 - `output/playwright/ci-parallel-firefox/report/index.html`
 - `output/playwright/ci-parallel-webkit/report/index.html`
-- `output/playwright/test-lists/<role>-<project>.txt`
+- `output/playwright/test-lists/<project>-<role>.txt`
+- `output/playwright/test-lists/<role>-<project>.txt` (legacy compatibility copy)
 
 ## Bubblewrap (TWA) Build and Install
 

playwright.config.ts

@@ -19,7 +19,7 @@ const reporter =
     : [["list"], ["html", { open: "never", outputFolder: htmlReportOutputFolder }]];
 
 function buildProject(name: string, browserName: "chromium" | "firefox" | "webkit", deviceName: keyof typeof devices) {
-  const { defaultBrowserType, ...device } = devices[deviceName];
+  const { defaultBrowserType: _defaultBrowserType, ...device } = devices[deviceName];
   const use = {
     browserName,
     ...device,
@@ -62,14 +62,14 @@ export default defineConfig({
     video: "retain-on-failure",
   },
   projects: [
+    buildProject("chromium-mobile", "chromium", "Pixel 5"),
     buildProject("chromium-desktop", "chromium", "Desktop Chrome"),
     buildProject("chromium-tablet", "chromium", "iPad Mini"),
-    buildProject("chromium-mobile", "chromium", "Pixel 5"),
-    buildProject("firefox-desktop", "firefox", "Desktop Firefox"),
-    buildProject("firefox-tablet", "firefox", "iPad Mini"),
-    buildProject("firefox-mobile", "firefox", "Pixel 5"),
+    buildProject("webkit-mobile", "webkit", "iPhone 12"),
     buildProject("webkit-desktop", "webkit", "Desktop Safari"),
     buildProject("webkit-tablet", "webkit", "iPad Mini"),
-    buildProject("webkit-mobile", "webkit", "iPhone 12"),
+    buildProject("firefox-mobile", "firefox", "Pixel 5"),
+    buildProject("firefox-desktop", "firefox", "Desktop Firefox"),
+    buildProject("firefox-tablet", "firefox", "iPad Mini"),
   ],
 });

scripts/run-playwright-full-slice.mjs

@@ -7,7 +7,12 @@ import { promisify } from "node:util";
 const workingDirectory = process.cwd();
 const playwrightCliPath = path.join(workingDirectory, "node_modules", "@playwright", "test", "cli.js");
 const execFileAsync = promisify(execFile);
-export const roles = ["customer", "subuser", "admin", "superuser"];
+export const browserEngines = ["chromium", "webkit", "firefox"];
+export const deviceClasses = ["mobile", "desktop", "tablet"];
+export const roles = ["superuser", "admin", "customer", "subuser"];
+export const fullSuiteProjects = browserEngines.flatMap((browser) =>
+  deviceClasses.map((device) => `${browser}-${device}`)
+);
 const listEntryPattern = /^\s+\[[^\]]+\]\s+›\s+(.+?):(\d+):(\d+)\s+›\s+(.+)\s*$/u;
 
 export const ownedFilesByRole = {
@@ -208,6 +213,10 @@ function validateOptions(options, forwardedArgs) {
     throw new Error("--project is required.");
   }
 
+  if (!fullSuiteProjects.includes(options.project)) {
+    throw new Error(`--project must be one of: ${fullSuiteProjects.join(", ")}`);
+  }
+
   for (const arg of forwardedArgs) {
     if (arg === "--list" || arg === "--test-list" || arg === "--project") {
       throw new Error(`Forwarded Playwright argument is not supported here: ${arg}`);
@@ -278,7 +287,7 @@ export function classifyTest(testEntry) {
 async function listProjectTests(project, forwardedArgs) {
   const { stdout, stderr } = await execFileAsync(
     process.execPath,
-    [playwrightCliPath, "test", "--list", `--project=${project}`, ...forwardedArgs],
+    [playwrightCliPath, "test", "--list", "--reporter=list", `--project=${project}`, ...forwardedArgs],
     {
       cwd: workingDirectory,
       maxBuffer: 64 * 1024 * 1024,
@@ -292,12 +301,26 @@ async function listProjectTests(project, forwardedArgs) {
   return stdout;
 }
 
-async function writeTestList(role, project, matchingTests) {
-  const outputDirectory = path.join(workingDirectory, "output", "playwright", "test-lists");
+const getTestListDirectory = () =>
+  process.env.PLAYWRIGHT_TEST_LIST_DIR || path.join(workingDirectory, "output", "playwright", "test-lists");
+
+export function getPrimaryTestListPath(project, role) {
+  return path.join(getTestListDirectory(), `${project}-${role}.txt`);
+}
+
+export function getLegacyTestListPath(role, project) {
+  return path.join(getTestListDirectory(), `${role}-${project}.txt`);
+}
+
+export async function writeTestList(role, project, matchingTests) {
+  const outputDirectory = getTestListDirectory();
   await fs.mkdir(outputDirectory, { recursive: true });
 
-  const testListPath = path.join(outputDirectory, `${role}-${project}.txt`);
-  await fs.writeFile(testListPath, `${matchingTests.map((testEntry) => testEntry.listLine).join("\n")}\n`, "utf8");
+  const contents = `${matchingTests.map((testEntry) => testEntry.listLine).join("\n")}\n`;
+  const testListPath = getPrimaryTestListPath(project, role);
+  const legacyTestListPath = getLegacyTestListPath(role, project);
+  await fs.writeFile(testListPath, contents, "utf8");
+  await fs.writeFile(legacyTestListPath, contents, "utf8");
   return testListPath;
 }
 

tests/unit/playwright-full-slice-ownership.spec.js

@@ -1,12 +1,22 @@
 import { execFileSync } from "node:child_process";
+import { readFileSync } from "node:fs";
+import fs from "node:fs/promises";
 import { readdirSync } from "node:fs";
+import { tmpdir } from "node:os";
 import { join } from "node:path";
-import { describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it } from "vitest";
 import {
+  browserEngines,
   classifyTest,
+  deviceClasses,
+  fullSuiteProjects,
+  getLegacyTestListPath,
+  getPrimaryTestListPath,
   ownedFilesByRole,
   parseListedTests,
+  roles,
   titleRules,
+  writeTestList,
 } from "../../scripts/run-playwright-full-slice.mjs";
 
 const root = process.cwd();
@@ -15,8 +25,35 @@ const titleRuleFiles = new Set(titleRules.map((rule) => rule.file));
 const e2eSpecFiles = readdirSync(join(root, "tests/e2e"))
   .filter((file) => /\.spec\.(?:js|ts)$/u.test(file))
   .sort();
+const generatedTestListPaths = [];
+const generatedTestListDirectories = [];
 
 describe("Playwright full-slice ownership", () => {
+  afterEach(async () => {
+    await Promise.all(generatedTestListPaths.splice(0).map((filePath) => fs.rm(filePath, { force: true })));
+    await Promise.all(
+      generatedTestListDirectories.splice(0).map((directory) => fs.rm(directory, { recursive: true, force: true }))
+    );
+    delete process.env.PLAYWRIGHT_TEST_LIST_DIR;
+  });
+
+  it("keeps full-suite slices ordered by browser, device, then role", () => {
+    expect(browserEngines).toEqual(["chromium", "webkit", "firefox"]);
+    expect(deviceClasses).toEqual(["mobile", "desktop", "tablet"]);
+    expect(roles).toEqual(["superuser", "admin", "customer", "subuser"]);
+    expect(fullSuiteProjects).toEqual([
+      "chromium-mobile",
+      "chromium-desktop",
+      "chromium-tablet",
+      "webkit-mobile",
+      "webkit-desktop",
+      "webkit-tablet",
+      "firefox-mobile",
+      "firefox-desktop",
+      "firefox-tablet",
+    ]);
+  });
+
   it("assigns every top-level e2e spec to one role or a title rule", () => {
     const directOwners = new Map();
 
@@ -65,7 +102,7 @@ describe("Playwright full-slice ownership", () => {
   it("classifies every listed test before role filtering", () => {
     const listOutput = execFileSync(
       process.execPath,
-      [playwrightCliPath, "test", "--list", "--project=chromium-desktop"],
+      [playwrightCliPath, "test", "--list", "--reporter=list", "--project=chromium-desktop"],
       {
         cwd: root,
         encoding: "utf8",
@@ -84,4 +121,33 @@ describe("Playwright full-slice ownership", () => {
 
     expect(classificationErrors).toEqual([]);
   });
+
+  it("writes project-role test lists and a legacy compatibility copy", async () => {
+    const testListDirectory = await fs.mkdtemp(join(tmpdir(), "playwright-test-lists-"));
+    process.env.PLAYWRIGHT_TEST_LIST_DIR = testListDirectory;
+    generatedTestListDirectories.push(testListDirectory);
+
+    const matchingTests = [
+      {
+        listLine: "[webkit-tablet] › tests/e2e/admin-pos-orders.spec.ts:10:1 › admin order list",
+      },
+    ];
+    const primaryPath = getPrimaryTestListPath("webkit-tablet", "admin");
+    const legacyPath = getLegacyTestListPath("admin", "webkit-tablet");
+    generatedTestListPaths.push(primaryPath, legacyPath);
+
+    await expect(writeTestList("admin", "webkit-tablet", matchingTests)).resolves.toBe(primaryPath);
+
+    expect(readFileSync(primaryPath, "utf8")).toBe(`${matchingTests[0].listLine}\n`);
+    expect(readFileSync(legacyPath, "utf8")).toBe(`${matchingTests[0].listLine}\n`);
+  });
+});
+
+describe("Playwright full-suite project order", () => {
+  it("keeps playwright.config projects ordered by browser engine and device class", () => {
+    const source = readFileSync(join(root, "playwright.config.ts"), "utf8");
+    const projectNames = [...source.matchAll(/buildProject\("([^"]+)"/gu)].map((match) => match[1]);
+
+    expect(projectNames).toEqual(fullSuiteProjects);
+  });
 });

tests/unit/playwright-full-workflow.spec.js

@@ -0,0 +1,31 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, it } from "vitest";
+
+const root = process.cwd();
+const workflowSource = () => readFileSync(join(root, ".github/workflows/tests.yml"), "utf8");
+
+describe("Playwright full E2E workflow grouping", () => {
+  it("orders full-suite matrix dimensions by browser, device, then role", () => {
+    const source = workflowSource();
+
+    expect(source).toContain("name: E2E-full-${{ matrix.browser_label }}-${{ matrix.device }}-${{ matrix.role }}");
+    expect(source).toContain("browser: [chromium, webkit, firefox]");
+    expect(source).toContain("device: [mobile, desktop, tablet]");
+    expect(source).toContain("role: [superuser, admin, customer, subuser]");
+  });
+
+  it("uses browser-device-role artifact namespaces and generated test lists", () => {
+    const source = workflowSource();
+
+    expect(source).toContain(
+      "PLAYWRIGHT_ARTIFACT_NAMESPACE: e2e-full-${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}"
+    );
+    expect(source).toContain(
+      "name: playwright-report-full-${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}"
+    );
+    expect(source).toContain(
+      "output/playwright/test-lists/${{ matrix.browser }}-${{ matrix.device }}-${{ matrix.role }}.txt"
+    );
+  });
+});