Skip to content

Commit 61a5286

Browse files
RomneyDaRomneyDa
committed
test(qa): run vitest and playwright scenarios from qa suite
1 parent 7c08804 commit 61a5286

18 files changed

Lines changed: 1115 additions & 22 deletions

‎docs/concepts/qa-e2e-automation.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -915,6 +915,7 @@ The report should answer:
915915
For the inventory of available scenarios - useful when sizing follow-up work or wiring a new transport - run `pnpm openclaw qa coverage` (add `--json` for machine-readable output).
916916
When choosing focused proof for a touched behavior or file path, run `pnpm openclaw qa coverage --match <query>`.
917917
The match report searches scenario metadata, docs refs, code refs, coverage IDs, plugins, and provider requirements, then prints matching `qa suite --scenario ...` targets.
918+
When a scenario declares `execution.kind: vitest` or `execution.kind: playwright`, `qa suite --scenario <scenario-id>` runs the matching test path and writes `qa-vitest-report.md` or `qa-playwright-report.md`, per-scenario logs, and `qa-evidence.json`.
918919
Treat it as a discovery aid, not a gate replacement; the selected scenario still needs the right provider mode, live transport, Multipass, Testbox, or release lane for the behavior under test.
919920

920921
For character and style checks, run the same scenario across multiple live model

‎extensions/qa-lab/src/cli.runtime.test.ts‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
66

77
const {
88
runQaManualLane,
9+
runQaPlaywrightScenarios,
10+
runQaVitestScenarios,
911
runQaSuiteFromRuntime,
1012
runQaCharacterEval,
1113
runQaMultipass,
@@ -18,6 +20,8 @@ const {
1820
defaultQaRuntimeModelForMode,
1921
} = vi.hoisted(() => ({
2022
runQaManualLane: vi.fn(),
23+
runQaPlaywrightScenarios: vi.fn(),
24+
runQaVitestScenarios: vi.fn(),
2125
runQaSuiteFromRuntime: vi.fn(),
2226
runQaCharacterEval: vi.fn(),
2327
runQaMultipass: vi.fn(),
@@ -47,6 +51,16 @@ vi.mock("./multipass.runtime.js", () => ({
4751
runQaMultipass,
4852
}));
4953

54+
vi.mock("./playwright-scenario-runner.js", async (importOriginal) => ({
55+
...(await importOriginal<typeof import("./playwright-scenario-runner.js")>()),
56+
runQaPlaywrightScenarios,
57+
}));
58+
59+
vi.mock("./vitest-scenario-runner.js", async (importOriginal) => ({
60+
...(await importOriginal<typeof import("./vitest-scenario-runner.js")>()),
61+
runQaVitestScenarios,
62+
}));
63+
5064
vi.mock("./live-transports/telegram/telegram-live.runtime.js", () => ({
5165
listTelegramQaScenarioCatalog,
5266
runTelegramQaLive,
@@ -160,6 +174,8 @@ describe("qa cli runtime", () => {
160174
runQaSuiteFromRuntime.mockReset();
161175
runQaCharacterEval.mockReset();
162176
runQaManualLane.mockReset();
177+
runQaPlaywrightScenarios.mockReset();
178+
runQaVitestScenarios.mockReset();
163179
runQaMultipass.mockReset();
164180
listTelegramQaScenarioCatalog.mockReset();
165181
runTelegramQaLive.mockReset();
@@ -187,6 +203,18 @@ describe("qa cli runtime", () => {
187203
reply: "done",
188204
watchUrl: "http://127.0.0.1:43124",
189205
});
206+
runQaPlaywrightScenarios.mockResolvedValue({
207+
outputDir: "/tmp/scenario-playwright",
208+
reportPath: "/tmp/scenario-playwright/qa-playwright-report.md",
209+
evidencePath: "/tmp/scenario-playwright/qa-evidence.json",
210+
results: [{ status: "pass" }],
211+
});
212+
runQaVitestScenarios.mockResolvedValue({
213+
outputDir: "/tmp/scenario-vitest",
214+
reportPath: "/tmp/scenario-vitest/qa-vitest-report.md",
215+
evidencePath: "/tmp/scenario-vitest/qa-evidence.json",
216+
results: [{ status: "pass" }],
217+
});
190218
runQaMultipass.mockResolvedValue({
191219
outputDir: "/tmp/multipass",
192220
reportPath: "/tmp/multipass/qa-suite-report.md",
@@ -242,6 +270,48 @@ describe("qa cli runtime", () => {
242270
await fs.rm(telegramArtifactsDir, { recursive: true, force: true });
243271
});
244272

273+
it("runs selected Playwright scenarios through the suite command", async () => {
274+
await runQaSuiteCommand({
275+
repoRoot: process.cwd(),
276+
outputDir: ".artifacts/qa-e2e/scenario-test",
277+
providerMode: "mock-openai",
278+
primaryModel: "mock-openai/gpt-5.5",
279+
scenarioIds: ["control-ui-chat-flow-playwright"],
280+
});
281+
282+
expect(runQaPlaywrightScenarios).toHaveBeenCalledTimes(1);
283+
expect(runQaVitestScenarios).not.toHaveBeenCalled();
284+
const call = mockFirstObjectArg(runQaPlaywrightScenarios);
285+
expectFields(call, {
286+
repoRoot: process.cwd(),
287+
outputDir: path.join(process.cwd(), ".artifacts", "qa-e2e", "scenario-test"),
288+
providerMode: "mock-openai",
289+
primaryModel: "mock-openai/gpt-5.5",
290+
});
291+
expect(
292+
(call.scenarios as Array<{ id: string; execution: { kind: string } }>).map((scenario) => ({
293+
id: scenario.id,
294+
kind: scenario.execution.kind,
295+
})),
296+
).toEqual([{ id: "control-ui-chat-flow-playwright", kind: "playwright" }]);
297+
expectWriteContains(
298+
stdoutWrite,
299+
"QA suite evidence: /tmp/scenario-playwright/qa-evidence.json",
300+
);
301+
});
302+
303+
it("rejects mixed flow and Vitest or Playwright scenarios in one suite command", async () => {
304+
await expect(
305+
runQaSuiteCommand({
306+
repoRoot: process.cwd(),
307+
scenarioIds: ["channel-chat-baseline", "control-ui-chat-flow-playwright"],
308+
}),
309+
).rejects.toThrow("qa suite cannot mix qa-flow and Vitest/Playwright scenarios");
310+
311+
expect(runQaPlaywrightScenarios).not.toHaveBeenCalled();
312+
expect(runQaVitestScenarios).not.toHaveBeenCalled();
313+
});
314+
245315
it("resolves suite repo-root-relative paths before dispatching", async () => {
246316
await runQaSuiteCommand({
247317
repoRoot: "/tmp/openclaw-repo",

‎extensions/qa-lab/src/cli.runtime.ts‎

Lines changed: 68 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@ import {
3939
import { startQaLabServer } from "./lab-server.js";
4040
import { runQaManualLane } from "./manual-lane.runtime.js";
4141
import { runQaMultipass } from "./multipass.runtime.js";
42+
import { isQaPlaywrightScenario, runQaPlaywrightScenarios } from "./playwright-scenario-runner.js";
4243
import { DEFAULT_QA_LIVE_PROVIDER_MODE, getQaProvider } from "./providers/index.js";
4344
import {
4445
QA_FRONTIER_PARITY_BASELINE_LABEL,
@@ -80,6 +81,7 @@ import {
8081
renderQaToolCoverageMarkdownReport,
8182
type QaToolCoverageSuiteSummary,
8283
} from "./tool-coverage-report.js";
84+
import { isQaVitestScenario, runQaVitestScenarios } from "./vitest-scenario-runner.js";
8385

8486
const QA_SUITE_INFRA_RETRY_LIMIT = 1;
8587
const QA_SUITE_INFRA_RETRY_NETWORK_ERROR_CODES = new Set([
@@ -230,6 +232,20 @@ function parseQaRuntimeParityTierFilters(input: string[] | undefined): QaRuntime
230232
return rawValues as QaRuntimeParityTier[];
231233
}
232234

235+
function selectExplicitQaSuiteScenarios(params: {
236+
scenarioIds: readonly string[];
237+
scenarios: ReturnType<typeof readQaScenarioPack>["scenarios"];
238+
}) {
239+
const scenarioById = new Map(params.scenarios.map((scenario) => [scenario.id, scenario]));
240+
return params.scenarioIds.map((scenarioId) => {
241+
const scenario = scenarioById.get(scenarioId);
242+
if (!scenario) {
243+
throw new Error(`unknown QA scenario id(s): ${scenarioId}`);
244+
}
245+
return scenario;
246+
});
247+
}
248+
233249
function resolveQaRuntimeParityTierScenarioIds(params: {
234250
scenarioIds: string[];
235251
runtimeParityTiers: readonly QaRuntimeParityTier[];
@@ -605,14 +621,63 @@ export async function runQaSuiteCommand(opts: {
605621
runtimeParityTiers,
606622
});
607623
const allowFailures = opts.allowFailures === true;
608-
if (runner !== "host" && runner !== "multipass") {
609-
throw new Error(`--runner must be one of host or multipass, got "${opts.runner}".`);
610-
}
611624
const providerMode = normalizeQaProviderMode(opts.providerMode);
612625
const runtimePair = parseQaRuntimePair(opts.runtimePair);
613626
const claudeCliAuthMode = parseQaCliBackendAuthMode(opts.cliAuthMode);
614627
const primaryModel = normalizeQaOptionalModelRef(opts.primaryModel);
615628
const alternateModel = normalizeQaOptionalModelRef(opts.alternateModel);
629+
const scenarioCatalog = readQaScenarioPack().scenarios;
630+
const explicitlySelectedScenarios =
631+
scenarioIds.length > 0
632+
? selectExplicitQaSuiteScenarios({ scenarioIds, scenarios: scenarioCatalog })
633+
: [];
634+
const vitestScenarios = explicitlySelectedScenarios.filter(isQaVitestScenario);
635+
const playwrightScenarios = explicitlySelectedScenarios.filter(isQaPlaywrightScenario);
636+
const nonFlowScenarioCount = vitestScenarios.length + playwrightScenarios.length;
637+
if (nonFlowScenarioCount > 0) {
638+
if (nonFlowScenarioCount !== explicitlySelectedScenarios.length) {
639+
throw new Error(
640+
"qa suite cannot mix qa-flow and Vitest/Playwright scenarios in one invocation.",
641+
);
642+
}
643+
if (vitestScenarios.length > 0 && playwrightScenarios.length > 0) {
644+
throw new Error("qa suite cannot mix Vitest and Playwright scenarios in one invocation.");
645+
}
646+
if (runner !== "host") {
647+
throw new Error("Vitest/Playwright scenarios require --runner host.");
648+
}
649+
if (opts.preflight === true) {
650+
throw new Error("--preflight requires qa-flow scenarios.");
651+
}
652+
const outputDir =
653+
resolveRepoRelativeOutputDir(repoRoot, opts.outputDir) ??
654+
path.join(repoRoot, ".artifacts", "qa-e2e", `qa-suite-test-${Date.now().toString(36)}`);
655+
const scenarioRunnerParams = {
656+
repoRoot,
657+
outputDir,
658+
providerMode,
659+
primaryModel: primaryModel ?? defaultQaModelForMode(providerMode),
660+
};
661+
const result =
662+
vitestScenarios.length > 0
663+
? await runQaVitestScenarios({
664+
...scenarioRunnerParams,
665+
scenarios: vitestScenarios,
666+
})
667+
: await runQaPlaywrightScenarios({
668+
...scenarioRunnerParams,
669+
scenarios: playwrightScenarios,
670+
});
671+
process.stdout.write(`QA suite report: ${result.reportPath}\n`);
672+
process.stdout.write(`QA suite evidence: ${result.evidencePath}\n`);
673+
if (!allowFailures && result.results.some((scenario) => scenario.status !== "pass")) {
674+
process.exitCode = 1;
675+
}
676+
return;
677+
}
678+
if (runner !== "host" && runner !== "multipass") {
679+
throw new Error(`--runner must be one of host or multipass, got "${opts.runner}".`);
680+
}
616681
if (opts.preflight === true && runner !== "host") {
617682
throw new Error("--preflight requires --runner host.");
618683
}

‎extensions/qa-lab/src/cli.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -444,7 +444,7 @@ export function registerQaLabCli(program: Command) {
444444
.option("--summary <path>", "Runtime qa-suite-summary.json to overlay on --tools coverage")
445445
.option(
446446
"--match <query>",
447-
"Search scenario metadata and print matching qa suite targets (repeatable)",
447+
"Search scenario metadata and print matching scenario refs (repeatable)",
448448
collectString,
449449
[],
450450
)

‎extensions/qa-lab/src/coverage-report.test.ts‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
// Qa Lab tests cover coverage report plugin behavior.
22
import { describe, expect, it } from "vitest";
3-
import { buildQaCoverageInventory, renderQaCoverageMarkdownReport } from "./coverage-report.js";
3+
import {
4+
buildQaCoverageInventory,
5+
findQaScenarioMatches,
6+
renderQaCoverageMarkdownReport,
7+
renderQaScenarioMatchesMarkdownReport,
8+
} from "./coverage-report.js";
49
import { readQaScenarioPack } from "./scenario-catalog.js";
510
import { buildQaScorecardTaxonomyReport, parseQaScorecardTaxonomy } from "./scorecard-taxonomy.js";
611

@@ -138,6 +143,19 @@ describe("qa coverage report", () => {
138143
expect(report).toContain("agents.subagents");
139144
});
140145

146+
it("renders Playwright matches as qa suite targets", () => {
147+
const matches = findQaScenarioMatches(readQaScenarioPack().scenarios, "chat-flow.e2e");
148+
const report = renderQaScenarioMatchesMarkdownReport({
149+
query: "chat-flow.e2e",
150+
matches,
151+
});
152+
153+
expect(report).toContain(
154+
"- Suite command: `pnpm openclaw qa suite --scenario control-ui-chat-flow-playwright`",
155+
);
156+
expect(report).toContain(" - execution: playwright ui/src/ui/e2e/chat-flow.e2e.test.ts");
157+
});
158+
141159
it("reports taxonomy mapping gaps as scorecard signals", () => {
142160
const taxonomy = parseQaScorecardTaxonomy({
143161
version: 1,

‎extensions/qa-lab/src/coverage-report.ts‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ type QaScenarioSearchMatch = QaCoverageScenarioSummary & {
2323
coverageIds: string[];
2424
docsRefs: string[];
2525
codeRefs: string[];
26+
executionKind: QaSeedScenarioWithSource["execution"]["kind"];
27+
executionPath?: string;
2628
runtimeParityTier?: string;
2729
requiredProviderMode?: string;
2830
requiredProvider?: string;
@@ -138,6 +140,8 @@ function summarizeScenarioSearchMatch(scenario: QaSeedScenarioWithSource): QaSce
138140
].toSorted((left, right) => left.localeCompare(right)),
139141
docsRefs: [...(scenario.docsRefs ?? [])],
140142
codeRefs: [...(scenario.codeRefs ?? [])],
143+
executionKind: scenario.execution.kind,
144+
...(scenario.execution.kind !== "flow" ? { executionPath: scenario.execution.path } : {}),
141145
runtimeParityTier: scenario.runtimeParityTier,
142146
requiredProviderMode: stringifyConfigValue(config.requiredProviderMode),
143147
requiredProvider: stringifyConfigValue(config.requiredProvider),
@@ -470,6 +474,11 @@ export function renderQaScenarioMatchesMarkdownReport(params: {
470474
lines.push(`- ${match.id}: ${match.title}`);
471475
lines.push(` - source: ${match.sourcePath}`);
472476
lines.push(` - surface: ${match.surfaces.join(", ")}`);
477+
lines.push(
478+
match.executionKind === "flow"
479+
? " - execution: qa-flow"
480+
: ` - execution: ${match.executionKind} ${match.executionPath ?? "missing"}`,
481+
);
473482
lines.push(` - coverage: ${match.coverageIds.join(", ") || "none"}`);
474483
lines.push(` - live requirements: ${formatOptionalScenarioMetadata(match)}`);
475484
if (match.codeRefs.length > 0) {

‎extensions/qa-lab/src/evidence-summary.ts‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,9 @@ type QaEvidenceTestTargetInput = {
210210
id: string;
211211
title: string;
212212
sourcePath: string;
213-
coverageIds: readonly string[];
213+
coverageIds?: readonly string[];
214+
primaryCoverageIds?: readonly string[];
215+
secondaryCoverageIds?: readonly string[];
214216
surfaceIds: readonly string[];
215217
categoryIds: readonly string[];
216218
docsRefs?: readonly string[];
@@ -578,7 +580,8 @@ function buildTestRunnerEvidenceSummary(
578580
mapping: {
579581
profile,
580582
coverage: buildQaEvidenceCoverage({
581-
primaryIds: target?.coverageIds ?? [],
583+
primaryIds: target?.primaryCoverageIds ?? target?.coverageIds ?? [],
584+
secondaryIds: target?.secondaryCoverageIds ?? [],
582585
surfaceIds: target?.surfaceIds ?? [],
583586
categoryIds: target?.categoryIds ?? [],
584587
}),

0 commit comments

Comments
 (0)