Compare commits

...

3 Commits

Author SHA1 Message Date
Joe Li
33db206fd0 fix(testing): keep promoted retry results marked as interrupted in summary
Track hasPromotedResults on FileResult so printFinalSummary correctly
identifies specs with promoted pending retries as interrupted (! marker)
rather than definitively failed (✗ marker). Transient first-attempt
failures from interrupted runs no longer inflate the failed spec count.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-07 11:14:27 -07:00
Joe Li
013933dff9 fix(testing): fix interrupted flush, pending retries, and spec-level footer
- Set started=true in onStdOut/onStdErr so files with buffered test
  output are flushed on interrupted runs even without an onTestEnd call
- Store non-terminal retry results in pendingResults map and promote
  them in onEnd, preserving the last failure message when a run is
  interrupted before retries complete
- Change footer to count failed/total spec files instead of individual
  tests, matching the per-file summary table semantics

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-06 18:19:53 -07:00
Joe Li
fd53284ac7 feat(testing): add Cypress-style Playwright reporter with grouped output
Replace the flat `list` reporter with a custom reporter that groups test
results by spec file — matching the familiar Cypress output format with
per-file headers, nested describe blocks, summary boxes, and a final
run-finished table.

Key behaviors: atomic per-file buffering for parallel workers, retry/flaky/
skipped handling, multi-project awareness, ANSI color management with
NO_COLOR support, and graceful interrupted-run reporting.

Includes fixes for: never-started files not emitting misleading blocks
(maxFailures/SIGINT), test.fail() expected failures treated as terminal
with retries enabled, and ANSI escape stripping from Playwright error
messages in NO_COLOR mode.

24 Jest unit tests covering all reporter behaviors.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-04-06 18:00:27 -07:00
3 changed files with 1452 additions and 2 deletions

View File

@@ -51,12 +51,12 @@ export default defineConfig({
reporter: process.env.CI
? [
['github'], // GitHub Actions annotations
['list'], // Detailed output with summary table
['./playwright/reporters/cypress-style-reporter.ts'], // Cypress-style grouped output
['html', { outputFolder: 'playwright-report', open: 'never' }], // Interactive report
['json', { outputFile: 'test-results/results.json' }], // Machine-readable
]
: [
['list'], // Shows summary table locally
['./playwright/reporters/cypress-style-reporter.ts'], // Cypress-style grouped output
['html', { outputFolder: 'playwright-report', open: 'on-failure' }], // Auto-open on failure
],

View File

@@ -0,0 +1,529 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import type {
Reporter,
FullConfig,
Suite,
TestCase,
TestResult,
FullResult,
TestError,
} from '@playwright/test/reporter';
interface CompletedTest {
title: string;
titlePath: string[];
duration: number;
outcome: 'expected' | 'unexpected' | 'flaky' | 'skipped';
errors: TestError[];
}
interface FileResult {
relativePath: string;
projectName: string;
fileIndex: number;
totalExpected: number;
results: Map<string, CompletedTest>;
pendingResults: Map<string, CompletedTest>;
retryDurations: Map<string, number>;
bufferedOutput: string[];
duration: number;
started: boolean;
hasPromotedResults: boolean;
}
interface Colors {
green: string;
red: string;
yellow: string;
cyan: string;
dim: string;
reset: string;
}
function formatDuration(ms: number): string {
if (ms < 1000) return `${ms}ms`;
return `${(ms / 1000).toFixed(1)}s`;
}
export default class CypressStyleReporter implements Reporter {
private fileResults = new Map<string, FileResult>();
private fileOrder: string[] = [];
private totalFiles = 0;
private multiProject = false;
private flushedFiles = new Set<string>();
private colors: Colors;
private useColor: boolean;
private boxWidth: number;
constructor() {
const useColor = !process.env.NO_COLOR && !!process.stdout.isTTY;
this.useColor = useColor;
this.colors = {
green: useColor ? '\x1B[32m' : '',
red: useColor ? '\x1B[31m' : '',
yellow: useColor ? '\x1B[33m' : '',
cyan: useColor ? '\x1B[36m' : '',
dim: useColor ? '\x1B[2m' : '',
reset: useColor ? '\x1B[0m' : '',
};
this.boxWidth = Math.min(process.stdout.columns || 90, 90);
}
onBegin(_config: FullConfig, suite: Suite): void {
const allTests = suite.allTests();
const filesByPath = new Map<string, Set<string>>();
for (const test of allTests) {
const tp = test.titlePath();
const projectName = tp[1] || '';
const relativePath = tp[2] || '';
const fileKey = `${projectName}::${relativePath}`;
// Track which projects each file path appears in
if (!filesByPath.has(relativePath)) {
filesByPath.set(relativePath, new Set());
}
filesByPath.get(relativePath)!.add(projectName);
if (!this.fileResults.has(fileKey)) {
this.fileResults.set(fileKey, {
relativePath,
projectName,
fileIndex: this.fileOrder.length,
totalExpected: 0,
results: new Map(),
pendingResults: new Map(),
retryDurations: new Map(),
bufferedOutput: [],
duration: 0,
started: false,
hasPromotedResults: false,
});
this.fileOrder.push(fileKey);
}
this.fileResults.get(fileKey)!.totalExpected += 1;
}
this.totalFiles = this.fileOrder.length;
// Only show project labels when the same file runs in multiple projects
this.multiProject = [...filesByPath.values()].some(
projects => projects.size > 1,
);
}
onTestEnd(test: TestCase, result: TestResult): void {
const tp = test.titlePath();
const fileKey = `${tp[1] || ''}::${tp[2] || ''}`;
const fileResult = this.fileResults.get(fileKey);
if (!fileResult) return;
fileResult.started = true;
// Accumulate duration across all attempts for accurate per-spec timing
const prevDuration = fileResult.retryDurations.get(test.id) || 0;
fileResult.retryDurations.set(test.id, prevDuration + result.duration);
const isTerminal =
result.status === 'passed' ||
result.status === 'skipped' ||
result.status === 'interrupted' ||
result.status === 'timedOut' ||
result.retry === test.retries ||
// Expected failure (test.fail()) — Playwright won't retry
(test.expectedStatus === 'failed' && result.status === 'failed');
const completedTest: CompletedTest = {
title: test.title,
titlePath: tp,
duration: fileResult.retryDurations.get(test.id)!,
outcome: test.outcome(),
errors: result.errors,
};
if (!isTerminal) {
fileResult.pendingResults.set(test.id, completedTest);
return;
}
fileResult.results.set(test.id, completedTest);
fileResult.pendingResults.delete(test.id);
if (fileResult.results.size === fileResult.totalExpected) {
this.flushFileBlock(fileKey, false);
}
}
onStdOut(
chunk: string | Buffer,
test?: TestCase,
_result?: TestResult,
): void {
if (test) {
const tp = test.titlePath();
const fileKey = `${tp[1] || ''}::${tp[2] || ''}`;
const fileResult = this.fileResults.get(fileKey);
if (fileResult && !this.flushedFiles.has(fileKey)) {
fileResult.bufferedOutput.push(chunk.toString());
fileResult.started = true;
return;
}
}
process.stdout.write(chunk);
}
onStdErr(
chunk: string | Buffer,
test?: TestCase,
_result?: TestResult,
): void {
if (test) {
const tp = test.titlePath();
const fileKey = `${tp[1] || ''}::${tp[2] || ''}`;
const fileResult = this.fileResults.get(fileKey);
if (fileResult && !this.flushedFiles.has(fileKey)) {
fileResult.bufferedOutput.push(chunk.toString());
fileResult.started = true;
return;
}
}
process.stderr.write(chunk);
}
onError(error: TestError): void {
const c = this.colors;
const rawMsg = error.message
? error.message.split('\n')[0]
: 'Unknown error';
const msg = this.useColor ? rawMsg : this.stripAnsi(rawMsg);
process.stderr.write(`\n${c.red} Global error: ${msg}${c.reset}\n`);
if (error.stack) {
const stack = this.useColor ? error.stack : this.stripAnsi(error.stack);
const trimmed = stack.split('\n').slice(0, 4).join('\n');
process.stderr.write(`${c.dim}${trimmed}${c.reset}\n`);
}
}
onEnd(result: FullResult): void {
// Promote pending (non-terminal) retry results so interrupted runs
// preserve the last observed failure instead of losing it entirely
for (const fileKey of this.fileOrder) {
const file = this.fileResults.get(fileKey);
if (!file) continue;
for (const [testId, pending] of file.pendingResults) {
if (!file.results.has(testId)) {
file.results.set(testId, pending);
file.hasPromotedResults = true;
}
}
}
const runFailed = result.status !== 'passed';
for (const fileKey of this.fileOrder) {
if (!this.flushedFiles.has(fileKey)) {
const file = this.fileResults.get(fileKey);
if (!file) continue;
// Show files with partial results, or 0-result files when the run
// was interrupted/failed (worker crash, SIGINT, timeout).
// Skip 0-result files on successful runs (e.g., --list mode).
if (file.results.size > 0 || (runFailed && file.started)) {
this.flushFileBlock(fileKey, true);
}
}
}
this.printFinalSummary(result.status);
}
printsToStdio(): boolean {
return true;
}
// --- Private helpers ---
private flushFileBlock(fileKey: string, interrupted: boolean): void {
this.flushedFiles.add(fileKey);
const file = this.fileResults.get(fileKey);
if (!file) return;
const c = this.colors;
const lines: string[] = [];
// Header
const displayPath = this.multiProject
? `[${file.projectName}] ${file.relativePath}`
: file.relativePath;
const counter = `(${file.fileIndex + 1} of ${this.totalFiles})`;
const gap = Math.max(
2,
this.boxWidth - 12 - displayPath.length - counter.length,
);
lines.push('');
lines.push(
`${c.cyan} Running: ${displayPath}${' '.repeat(gap)}${counter}${c.reset}`,
);
lines.push('');
// Sort results by titlePath for deterministic ordering
const sorted = [...file.results.values()].sort((a, b) =>
a.titlePath.join('\0').localeCompare(b.titlePath.join('\0')),
);
// Render test results with describe nesting
let currentDescribes: string[] = [];
for (const t of sorted) {
const describes = t.titlePath.slice(3, -1);
// Print new describe headers when context changes
for (let i = 0; i < describes.length; i += 1) {
if (currentDescribes[i] !== describes[i]) {
const indent = ` ${' '.repeat(i)}`;
lines.push(`${indent}${describes[i]}`);
currentDescribes = describes.slice(0, i + 1);
}
}
if (describes.length < currentDescribes.length) {
currentDescribes = describes.slice();
}
const indent = ` ${' '.repeat(describes.length)}`;
const dur = formatDuration(t.duration);
if (t.outcome === 'expected') {
lines.push(
`${indent}${c.green}${c.reset} ${t.title} ${c.dim}(${dur})${c.reset}`,
);
} else if (t.outcome === 'unexpected') {
lines.push(
`${indent}${c.red}${c.reset} ${t.title} ${c.dim}(${dur})${c.reset}`,
);
for (const err of t.errors) {
lines.push('');
if (err.message) {
const msg = this.useColor
? err.message
: this.stripAnsi(err.message);
for (const msgLine of msg.split('\n')) {
lines.push(`${indent} ${c.red}${msgLine}${c.reset}`);
}
} else {
lines.push(`${indent} ${c.red}Unknown error${c.reset}`);
}
if (err.location) {
lines.push(
`${indent} ${c.dim}at ${err.location.file}:${err.location.line}${c.reset}`,
);
}
if (err.stack) {
const rawStack = this.useColor
? err.stack
: this.stripAnsi(err.stack);
const stackLines = rawStack
.split('\n')
.filter(l => l.trimStart().startsWith('at '))
.slice(0, 5);
for (const sl of stackLines) {
lines.push(`${indent} ${c.dim}${sl.trim()}${c.reset}`);
}
}
}
} else if (t.outcome === 'flaky') {
lines.push(
`${indent}${c.yellow}${c.reset} ${t.title} ${c.yellow}(flaky)${c.reset} ${c.dim}(${dur})${c.reset}`,
);
} else if (t.outcome === 'skipped') {
lines.push(
`${indent}${c.cyan}-${c.reset} ${t.title} ${c.dim}(skipped)${c.reset}`,
);
}
}
// Buffered stdout/stderr from tests
if (file.bufferedOutput.length > 0) {
lines.push('');
for (const chunk of file.bufferedOutput) {
lines.push(` ${chunk.trimEnd()}`);
}
}
lines.push('');
// Compute counts
let passing = 0;
let failing = 0;
let skipped = 0;
let totalDuration = 0;
for (const r of file.results.values()) {
totalDuration += r.duration;
if (r.outcome === 'expected' || r.outcome === 'flaky') passing += 1;
else if (r.outcome === 'unexpected') failing += 1;
else if (r.outcome === 'skipped') skipped += 1;
}
file.duration = totalDuration;
// Summary box
const testsLabel = interrupted
? `${file.results.size} of ${file.totalExpected} (interrupted)`
: `${file.results.size}`;
const boxRows: string[] = [
`Tests: ${testsLabel}`,
`Passing: ${passing}`,
`Failing: ${failing}`,
];
if (skipped > 0) {
boxRows.push(`Skipped: ${skipped}`);
}
boxRows.push(`Duration: ${formatDuration(totalDuration)}`);
boxRows.push(`Spec Ran: ${file.relativePath}`);
lines.push(this.boxTop());
for (const row of boxRows) {
lines.push(this.boxLine(row));
}
lines.push(this.boxBottom());
process.stdout.write(`${lines.join('\n')}\n`);
}
private printFinalSummary(runStatus: string): void {
// No tests ran and run succeeded (e.g., --list mode) — nothing to summarize
if (this.flushedFiles.size === 0 && runStatus === 'passed') return;
const c = this.colors;
const lines: string[] = [];
lines.push('');
lines.push('='.repeat(this.boxWidth));
lines.push('');
lines.push(' (Run Finished)');
lines.push('');
// Table header
const specColWidth = Math.max(this.boxWidth - 40, 10);
if (this.multiProject) {
lines.push(
` ${'Spec'.padEnd(specColWidth - 14)}${'Project'.padEnd(14)}Tests Passing Failing Duration`,
);
} else {
lines.push(
` ${'Spec'.padEnd(specColWidth)}Tests Passing Failing Duration`,
);
}
lines.push(this.boxTop());
let totalSpecs = 0;
let failedSpecs = 0;
let totalSkipped = 0;
for (const fileKey of this.fileOrder) {
const file = this.fileResults.get(fileKey)!;
// Skip files that were never flushed (e.g., --list mode)
if (!this.flushedFiles.has(fileKey)) continue;
let passing = 0;
let failing = 0;
let skippedCount = 0;
const wasInterrupted =
file.hasPromotedResults || file.results.size < file.totalExpected;
for (const r of file.results.values()) {
if (r.outcome === 'expected' || r.outcome === 'flaky') passing += 1;
else if (r.outcome === 'unexpected') failing += 1;
else if (r.outcome === 'skipped') skippedCount += 1;
}
const tests = file.results.size;
totalSpecs += 1;
if (failing > 0 && !wasInterrupted) failedSpecs += 1;
totalSkipped += skippedCount;
const marker = wasInterrupted
? `${c.yellow}!${c.reset}`
: failing > 0
? `${c.red}${c.reset}`
: `${c.green}${c.reset}`;
const dur = formatDuration(file.duration);
let row: string;
if (this.multiProject) {
row =
` ${marker} ${file.relativePath.padEnd(specColWidth - 14)}` +
`${file.projectName.padEnd(14)}` +
`${String(tests).padStart(5)} ` +
`${String(passing).padStart(7)} ` +
`${String(failing).padStart(7)} ` +
`${dur.padStart(8)}`;
} else {
row =
` ${marker} ${file.relativePath.padEnd(specColWidth)}` +
`${String(tests).padStart(5)} ` +
`${String(passing).padStart(7)} ` +
`${String(failing).padStart(7)} ` +
`${dur.padStart(8)}`;
}
lines.push(this.boxLine(row));
}
lines.push(this.boxBottom());
lines.push('');
const skippedSuffix = totalSkipped > 0 ? ` (${totalSkipped} skipped)` : '';
if (failedSpecs > 0) {
lines.push(
` ${c.red}${failedSpecs} of ${totalSpecs} failed${c.reset}${skippedSuffix}`,
);
} else if (runStatus === 'interrupted') {
lines.push(
` ${c.yellow}! Run was interrupted${c.reset}${skippedSuffix}`,
);
} else if (runStatus === 'timedout') {
lines.push(` ${c.red}✗ Run timed out${c.reset}${skippedSuffix}`);
} else if (runStatus !== 'passed') {
lines.push(` ${c.red}✗ Run failed${c.reset}${skippedSuffix}`);
} else {
lines.push(
` ${c.green}✓ All ${totalSpecs} passed${c.reset}${skippedSuffix}`,
);
}
lines.push('');
process.stdout.write(`${lines.join('\n')}\n`);
}
private boxTop(): string {
return `${'─'.repeat(this.boxWidth - 4)}`;
}
private boxBottom(): string {
return `${'─'.repeat(this.boxWidth - 4)}`;
}
private stripAnsi(str: string): string {
return str.replace(/\x1B\[[0-9;]*m/g, '');
}
private boxLine(content: string): string {
const inner = this.boxWidth - 6;
const visibleLen = this.stripAnsi(content).length;
if (visibleLen >= inner) {
return `${content}`;
}
return `${content}${' '.repeat(inner - visibleLen)}`;
}
}

View File

@@ -0,0 +1,921 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import CypressStyleReporter from '../../playwright/reporters/cypress-style-reporter';
/* eslint-disable @typescript-eslint/no-explicit-any */
function mockTest(opts: {
id: string;
title: string;
project?: string;
file?: string;
describes?: string[];
retries?: number;
outcome?: string;
expectedStatus?: string;
}): any {
const project = opts.project ?? 'chromium';
const file = opts.file ?? 'tests/example.spec.ts';
const describes = opts.describes ?? [];
const titlePath = ['', project, file, ...describes, opts.title];
return {
id: opts.id,
title: opts.title,
titlePath: () => titlePath,
retries: opts.retries ?? 0,
outcome: () => opts.outcome ?? 'expected',
expectedStatus: opts.expectedStatus ?? 'passed',
};
}
function mockResult(
opts: {
status?: string;
duration?: number;
retry?: number;
errors?: any[];
} = {},
): any {
return {
status: opts.status ?? 'passed',
duration: opts.duration ?? 1000,
retry: opts.retry ?? 0,
errors: opts.errors ?? [],
};
}
function mockSuite(tests: any[]): any {
return { allTests: () => tests };
}
const mockConfig: any = {};
const mockFullResult: any = { status: 'passed' };
let stdoutChunks: string[];
let stderrChunks: string[];
function getStdout(): string {
return stdoutChunks.join('');
}
function getStderr(): string {
return stderrChunks.join('');
}
beforeEach(() => {
stdoutChunks = [];
stderrChunks = [];
jest.spyOn(process.stdout, 'write').mockImplementation((chunk: any) => {
stdoutChunks.push(chunk.toString());
return true;
});
jest.spyOn(process.stderr, 'write').mockImplementation((chunk: any) => {
stderrChunks.push(chunk.toString());
return true;
});
});
afterEach(() => {
jest.restoreAllMocks();
delete process.env.NO_COLOR;
});
test('renders a single file block with header, results, and summary box', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'test one',
file: 'auth/login.spec.ts',
});
const t2 = mockTest({
id: '2',
title: 'test two',
file: 'auth/login.spec.ts',
});
reporter.onBegin(mockConfig, mockSuite([t1, t2]));
// File should not flush after first test (1 of 2)
reporter.onTestEnd(t1, mockResult({ duration: 2000 }));
expect(getStdout()).toBe('');
// File should flush after second test (2 of 2)
reporter.onTestEnd(t2, mockResult({ duration: 3000 }));
const output = getStdout();
expect(output).toContain('Running: auth/login.spec.ts');
expect(output).toContain('(1 of 1)');
expect(output).toContain('✓ test one');
expect(output).toContain('✓ test two');
expect(output).toContain('Tests: 2');
expect(output).toContain('Passing: 2');
expect(output).toContain('Failing: 0');
expect(output).toContain('┌');
expect(output).toContain('└');
});
test('non-terminal retry results are not stored; terminal result flushes correctly', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'retry test',
file: 'tests/retry.spec.ts',
retries: 2,
outcome: 'unexpected',
});
reporter.onBegin(mockConfig, mockSuite([t1]));
// Attempt 0: fails, retry=0 !== retries=2 → not terminal
reporter.onTestEnd(
t1,
mockResult({ status: 'failed', retry: 0, errors: [{ message: 'fail' }] }),
);
expect(getStdout()).toBe('');
// Attempt 1: fails, retry=1 !== retries=2 → not terminal
reporter.onTestEnd(
t1,
mockResult({ status: 'failed', retry: 1, errors: [{ message: 'fail' }] }),
);
expect(getStdout()).toBe('');
// Attempt 2: fails, retry=2 === retries=2 → terminal
reporter.onTestEnd(
t1,
mockResult({
status: 'failed',
retry: 2,
duration: 500,
errors: [{ message: 'final fail' }],
}),
);
const output = getStdout();
expect(output).toContain('✗ retry test');
expect(output).toContain('Tests: 1');
expect(output).toContain('Failing: 1');
});
test('flaky tests show (flaky) annotation when test fails then passes on retry', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'sometimes fails',
file: 'tests/flaky.spec.ts',
retries: 1,
outcome: 'flaky',
});
reporter.onBegin(mockConfig, mockSuite([t1]));
// First attempt fails (retry=0 !== retries=1 → not terminal)
reporter.onTestEnd(
t1,
mockResult({ status: 'failed', retry: 0, errors: [{ message: 'oops' }] }),
);
expect(getStdout()).toBe('');
// Retry passes (terminal: status === 'passed')
reporter.onTestEnd(
t1,
mockResult({ status: 'passed', retry: 1, duration: 800 }),
);
const output = getStdout();
expect(output).toContain('✓ sometimes fails');
expect(output).toContain('(flaky)');
expect(output).toContain('Passing: 1');
});
test('skipped tests render with dash marker and summary count', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'passing test',
file: 'tests/skip.spec.ts',
});
const t2 = mockTest({
id: '2',
title: 'skipped test',
file: 'tests/skip.spec.ts',
outcome: 'skipped',
});
reporter.onBegin(mockConfig, mockSuite([t1, t2]));
reporter.onTestEnd(t1, mockResult({ duration: 500 }));
reporter.onTestEnd(t2, mockResult({ status: 'skipped', duration: 0 }));
const output = getStdout();
expect(output).toContain('✓ passing test');
expect(output).toContain('- skipped test');
expect(output).toContain('(skipped)');
expect(output).toContain('Skipped: 1');
expect(output).toContain('Passing: 1');
});
test('NO_COLOR mode produces no ANSI escape sequences', () => {
const origIsTTY = process.stdout.isTTY;
try {
// With TTY and no NO_COLOR → colors present
Object.defineProperty(process.stdout, 'isTTY', {
value: true,
writable: true,
configurable: true,
});
delete process.env.NO_COLOR;
const colorReporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'color test',
file: 'tests/c.spec.ts',
});
colorReporter.onBegin(mockConfig, mockSuite([t1]));
colorReporter.onTestEnd(t1, mockResult());
expect(getStdout()).toContain('\x1B[');
// Reset capture
stdoutChunks = [];
// With NO_COLOR=1 → no ANSI codes
process.env.NO_COLOR = '1';
const noColorReporter = new CypressStyleReporter();
const t2 = mockTest({
id: '1',
title: 'no color test',
file: 'tests/nc.spec.ts',
});
noColorReporter.onBegin(mockConfig, mockSuite([t2]));
noColorReporter.onTestEnd(t2, mockResult());
expect(getStdout()).not.toContain('\x1B[');
} finally {
Object.defineProperty(process.stdout, 'isTTY', {
value: origIsTTY,
writable: true,
configurable: true,
});
}
});
test('onError prints to stderr', () => {
const reporter = new CypressStyleReporter();
reporter.onError({
message: 'Worker crashed',
stack: 'Error: Worker crashed\n at test.ts:10',
} as any);
const output = getStderr();
expect(output).toContain('Global error: Worker crashed');
expect(output).toContain('Worker crashed');
});
test('incomplete file buckets flush as interrupted in onEnd', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'completed test',
file: 'tests/interrupted.spec.ts',
});
const t2 = mockTest({
id: '2',
title: 'never finished',
file: 'tests/interrupted.spec.ts',
});
reporter.onBegin(mockConfig, mockSuite([t1, t2]));
// Only complete one of two tests
reporter.onTestEnd(t1, mockResult({ duration: 1000 }));
expect(getStdout()).toBe('');
// Simulate run end (e.g., SIGINT)
reporter.onEnd(mockFullResult);
const output = getStdout();
expect(output).toContain('Running: tests/interrupted.spec.ts');
expect(output).toContain('1 of 2 (interrupted)');
expect(output).toContain('✓ completed test');
});
test('test-associated stdout is buffered; global stdout passes through immediately', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'logging test',
file: 'tests/stdout.spec.ts',
});
reporter.onBegin(mockConfig, mockSuite([t1]));
// Global stdout (no test) passes through immediately
reporter.onStdOut('global setup message\n');
expect(getStdout()).toBe('global setup message\n');
// Test-associated stdout is buffered
stdoutChunks = [];
reporter.onStdOut('test log line\n', t1);
expect(getStdout()).toBe('');
// Complete the test → flush includes buffered output
reporter.onTestEnd(t1, mockResult());
const output = getStdout();
expect(output).toContain('✓ logging test');
expect(output).toContain('test log line');
});
test('final summary table shows per-file rows and totals', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'passes',
file: 'auth/login.spec.ts',
});
const t2 = mockTest({
id: '2',
title: 'fails',
file: 'dashboard/list.spec.ts',
outcome: 'unexpected',
});
reporter.onBegin(mockConfig, mockSuite([t1, t2]));
reporter.onTestEnd(t1, mockResult());
reporter.onTestEnd(
t2,
mockResult({ status: 'failed', errors: [{ message: 'boom' }] }),
);
// Trigger final summary
reporter.onEnd(mockFullResult);
const output = getStdout();
expect(output).toContain('(Run Finished)');
expect(output).toContain('auth/login.spec.ts');
expect(output).toContain('dashboard/list.spec.ts');
expect(output).toContain('✓');
expect(output).toContain('✗');
// Total line: 1 failure out of 2
expect(output).toContain('1 of 2 failed');
});
test('disjoint projects do not show project labels', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'auth test',
project: 'chromium-unauth',
file: 'auth/login.spec.ts',
});
const t2 = mockTest({
id: '2',
title: 'dashboard test',
project: 'chromium',
file: 'dashboard/list.spec.ts',
});
reporter.onBegin(mockConfig, mockSuite([t1, t2]));
reporter.onTestEnd(t1, mockResult());
reporter.onTestEnd(t2, mockResult());
reporter.onEnd(mockFullResult);
const output = getStdout();
// Disjoint files across projects — no brackets, no Project column
expect(output).not.toContain('[chromium');
expect(output).not.toContain('Project');
expect(output).toContain('auth/login.spec.ts');
expect(output).toContain('dashboard/list.spec.ts');
});
test('overlapping projects show project labels in headers and summary', () => {
const reporter = new CypressStyleReporter();
// Same file in two projects (overlapping)
const t1 = mockTest({
id: '1',
title: 'test in project A',
project: 'chromium',
file: 'shared/test.spec.ts',
});
const t2 = mockTest({
id: '2',
title: 'test in project B',
project: 'firefox',
file: 'shared/test.spec.ts',
});
reporter.onBegin(mockConfig, mockSuite([t1, t2]));
reporter.onTestEnd(t1, mockResult());
reporter.onTestEnd(t2, mockResult());
reporter.onEnd(mockFullResult);
const output = getStdout();
expect(output).toContain('[chromium] shared/test.spec.ts');
expect(output).toContain('[firefox] shared/test.spec.ts');
expect(output).toContain('Project');
});
test('failed tests render error message and location', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'fails with location',
file: 'tests/error.spec.ts',
outcome: 'unexpected',
});
const t2 = mockTest({
id: '2',
title: 'fails with stack only',
file: 'tests/error.spec.ts',
outcome: 'unexpected',
});
reporter.onBegin(mockConfig, mockSuite([t1, t2]));
reporter.onTestEnd(
t1,
mockResult({
status: 'failed',
errors: [
{
message:
'Expected true to be false\n\nExpected: false\nReceived: true',
location: { file: 'error.spec.ts', line: 42, column: 5 },
},
],
}),
);
reporter.onTestEnd(
t2,
mockResult({
status: 'failed',
errors: [
{
message: 'Timeout exceeded',
stack:
'Error: Timeout exceeded\n at Object.<anonymous> (error.spec.ts:55:3)',
},
],
}),
);
const output = getStdout();
// Error with location — multiline message preserved
expect(output).toContain('Expected true to be false');
expect(output).toContain('Expected: false');
expect(output).toContain('Received: true');
expect(output).toContain('at error.spec.ts:42');
// Error with stack fallback — full message and stack lines preserved
expect(output).toContain('Timeout exceeded');
expect(output).toContain('at Object.<anonymous>');
});
test('onStdErr for a test is buffered; global stderr passes through', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'stderr test',
file: 'tests/stderr.spec.ts',
});
reporter.onBegin(mockConfig, mockSuite([t1]));
// Global stderr (no test) passes through immediately
reporter.onStdErr('global warning\n');
expect(getStderr()).toBe('global warning\n');
// Test-associated stderr is buffered
stderrChunks = [];
reporter.onStdErr('test warning\n', t1);
expect(getStderr()).toBe('');
// Complete the test → flush includes buffered stderr in stdout output
reporter.onTestEnd(t1, mockResult());
const output = getStdout();
expect(output).toContain('test warning');
});
test('describe nesting renders group headers with indentation', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'inner test',
file: 'tests/nested.spec.ts',
describes: ['Auth', 'Login Form'],
});
const t2 = mockTest({
id: '2',
title: 'top level test',
file: 'tests/nested.spec.ts',
});
reporter.onBegin(mockConfig, mockSuite([t1, t2]));
reporter.onTestEnd(t1, mockResult());
reporter.onTestEnd(t2, mockResult());
const output = getStdout();
expect(output).toContain('Auth');
expect(output).toContain('Login Form');
expect(output).toContain('✓ inner test');
expect(output).toContain('✓ top level test');
});
test('skipped suffix appears in final summary when tests are skipped', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'runs',
file: 'tests/skip-summary.spec.ts',
});
const t2 = mockTest({
id: '2',
title: 'is skipped',
file: 'tests/skip-summary.spec.ts',
outcome: 'skipped',
});
reporter.onBegin(mockConfig, mockSuite([t1, t2]));
reporter.onTestEnd(t1, mockResult());
reporter.onTestEnd(t2, mockResult({ status: 'skipped', duration: 0 }));
reporter.onEnd(mockFullResult);
const output = getStdout();
// Final summary accounts for skipped
expect(output).toContain('All 1 passed');
expect(output).toContain('(1 skipped)');
});
test('file that never started is not shown when run is interrupted', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'never started',
file: 'tests/crashed.spec.ts',
});
reporter.onBegin(mockConfig, mockSuite([t1]));
// No onTestEnd calls — file never started (e.g., maxFailures, SIGINT before this file)
reporter.onEnd({ status: 'interrupted' } as any);
const output = getStdout();
// File never received any test results — do not emit a misleading block
expect(output).not.toContain('Running: tests/crashed.spec.ts');
expect(output).not.toContain('0 of 1');
// Final summary still shows the run was interrupted
expect(output).toContain('interrupted');
});
test('--list mode: no file blocks or summary printed when no tests ran', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'listed test',
file: 'auth/login.spec.ts',
});
reporter.onBegin(mockConfig, mockSuite([t1]));
// No onTestEnd calls — --list mode only collects, never runs
reporter.onEnd({ status: 'passed' } as any);
const output = getStdout();
expect(output).not.toContain('interrupted');
expect(output).not.toContain('Running:');
expect(output).not.toContain('(Run Finished)');
});
test('interrupted run footer shows interruption, not "All 0 passed"', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({ id: '1', title: 'completed', file: 'tests/a.spec.ts' });
const t2 = mockTest({ id: '2', title: 'never ran', file: 'tests/a.spec.ts' });
reporter.onBegin(mockConfig, mockSuite([t1, t2]));
reporter.onTestEnd(t1, mockResult());
reporter.onEnd({ status: 'interrupted' } as any);
const output = getStdout();
expect(output).not.toContain('All');
expect(output).toContain('interrupted');
});
test('timed-out run footer shows timeout, not success', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({ id: '1', title: 'ran ok', file: 'tests/b.spec.ts' });
reporter.onBegin(mockConfig, mockSuite([t1]));
reporter.onTestEnd(t1, mockResult());
reporter.onEnd({ status: 'timedout' } as any);
const output = getStdout();
expect(output).not.toContain('All');
expect(output).toContain('timed out');
});
test('retry durations are accumulated across all attempts', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'flaky test',
file: 'tests/duration.spec.ts',
retries: 1,
outcome: 'flaky',
});
reporter.onBegin(mockConfig, mockSuite([t1]));
// First attempt: fails after 10s (not terminal: retry 0 !== retries 1)
reporter.onTestEnd(
t1,
mockResult({
status: 'failed',
retry: 0,
duration: 10000,
errors: [{ message: 'fail' }],
}),
);
// Retry: passes after 1s (terminal: status === 'passed')
reporter.onTestEnd(
t1,
mockResult({ status: 'passed', retry: 1, duration: 1000 }),
);
const output = getStdout();
// Duration should reflect both attempts: 11.0s, not just the final 1.0s
expect(output).toContain('Duration: 11.0s');
});
test('maxFailures: file that never started is not shown', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'test a',
file: 'tests/a.spec.ts',
outcome: 'unexpected',
});
const t2 = mockTest({
id: '2',
title: 'test b',
file: 'tests/b.spec.ts',
});
reporter.onBegin(mockConfig, mockSuite([t1, t2]));
// File A runs and fails — maxFailures stops the run before file B starts
reporter.onTestEnd(
t1,
mockResult({ status: 'failed', errors: [{ message: 'fail' }] }),
);
reporter.onEnd({ status: 'interrupted' } as any);
const output = getStdout();
// File A: should appear (it started and has results)
expect(output).toContain('Running: tests/a.spec.ts');
// File B: should NOT appear (it never started)
expect(output).not.toContain('Running: tests/b.spec.ts');
});
test('test.fail() expected failure is treated as terminal even with retries', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'expected to fail',
file: 'tests/expected-fail.spec.ts',
retries: 2,
expectedStatus: 'failed',
outcome: 'expected',
});
reporter.onBegin(mockConfig, mockSuite([t1]));
// Single onTestEnd: status 'failed', retry 0 — but expectedStatus is 'failed'
// so Playwright won't retry. Must be treated as terminal.
reporter.onTestEnd(
t1,
mockResult({ status: 'failed', retry: 0, duration: 1000 }),
);
const output = getStdout();
// outcome is 'expected' → green check
expect(output).toContain('✓ expected to fail');
expect(output).toContain('Tests: 1');
});
test('NO_COLOR strips ANSI escapes from error messages and stack traces', () => {
process.env.NO_COLOR = '1';
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'fails with ansi',
file: 'tests/ansi-err.spec.ts',
outcome: 'unexpected',
});
reporter.onBegin(mockConfig, mockSuite([t1]));
reporter.onTestEnd(
t1,
mockResult({
status: 'failed',
errors: [
{
message: '\x1B[31mExpected\x1B[0m value to be true',
stack: 'Error: Expected value\n at \x1B[2mtest.ts:10\x1B[0m',
},
],
}),
);
const output = getStdout();
expect(output).not.toContain('\x1B[31m');
expect(output).not.toContain('\x1B[2m');
expect(output).not.toContain('\x1B[0m');
expect(output).toContain('Expected');
expect(output).toContain('value to be true');
expect(output).toContain('at test.ts:10');
});
test('buffered test output marks file as started for interrupted flush', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'crashes after output',
file: 'tests/output-crash.spec.ts',
});
reporter.onBegin(mockConfig, mockSuite([t1]));
// Test writes to stdout but worker crashes before onTestEnd fires
reporter.onStdOut('debug: starting test\n', t1);
reporter.onEnd({ status: 'interrupted' } as any);
const output = getStdout();
// File should be shown because it started (received test output)
expect(output).toContain('Running: tests/output-crash.spec.ts');
expect(output).toContain('0 of 1 (interrupted)');
expect(output).toContain('debug: starting test');
});
test('interrupted retry preserves last failed attempt with error details', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'passes',
file: 'tests/retry-interrupt.spec.ts',
retries: 1,
});
const t2 = mockTest({
id: '2',
title: 'fails then interrupted',
file: 'tests/retry-interrupt.spec.ts',
retries: 1,
outcome: 'unexpected',
});
reporter.onBegin(mockConfig, mockSuite([t1, t2]));
// Test 1 passes (terminal)
reporter.onTestEnd(t1, mockResult({ duration: 500 }));
// Test 2 fails on retry 0 (non-terminal: 0 !== 1), stored as pending
reporter.onTestEnd(
t2,
mockResult({
status: 'failed',
retry: 0,
duration: 2000,
errors: [{ message: 'assertion failed' }],
}),
);
// Run interrupted before retry 1 — onEnd promotes pending results
reporter.onEnd({ status: 'interrupted' } as any);
const output = getStdout();
expect(output).toContain('Running: tests/retry-interrupt.spec.ts');
expect(output).toContain('✓ passes');
expect(output).toContain('✗ fails then interrupted');
expect(output).toContain('assertion failed');
// Both tests reported (pending promoted to results)
expect(output).toContain('2 of 2 (interrupted)');
});
test('footer counts failed specs, not individual failed tests', () => {
const reporter = new CypressStyleReporter();
// One spec with 3 tests: 1 passes, 2 fail
const t1 = mockTest({
id: '1',
title: 'passes',
file: 'tests/multi.spec.ts',
});
const t2 = mockTest({
id: '2',
title: 'fails one',
file: 'tests/multi.spec.ts',
outcome: 'unexpected',
});
const t3 = mockTest({
id: '3',
title: 'fails two',
file: 'tests/multi.spec.ts',
outcome: 'unexpected',
});
// Another spec that passes
const t4 = mockTest({
id: '4',
title: 'passes too',
file: 'tests/other.spec.ts',
});
reporter.onBegin(mockConfig, mockSuite([t1, t2, t3, t4]));
reporter.onTestEnd(t1, mockResult());
reporter.onTestEnd(
t2,
mockResult({ status: 'failed', errors: [{ message: 'a' }] }),
);
reporter.onTestEnd(
t3,
mockResult({ status: 'failed', errors: [{ message: 'b' }] }),
);
reporter.onTestEnd(t4, mockResult());
reporter.onEnd(mockFullResult);
const output = getStdout();
// Footer: 1 spec failed (not 2 tests), out of 2 total specs
expect(output).toContain('1 of 2 failed');
expect(output).not.toContain('2 of 4 failed');
});
test('promoted retry results keep spec marked as interrupted in summary', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'fails once',
file: 'tests/promoted.spec.ts',
retries: 2,
outcome: 'unexpected',
});
reporter.onBegin(mockConfig, mockSuite([t1]));
// First attempt fails (non-terminal: retry 0 !== retries 2)
reporter.onTestEnd(
t1,
mockResult({
status: 'failed',
retry: 0,
duration: 1000,
errors: [{ message: 'first fail' }],
}),
);
// Run interrupted before retry 1 — onEnd promotes pending result
reporter.onEnd({ status: 'interrupted' } as any);
const output = getStdout();
// Per-file box: promoted result shown with interrupted label
expect(output).toContain('✗ fails once');
expect(output).toContain('1 of 1 (interrupted)');
// Footer: "Run was interrupted", NOT "1 of 1 failed"
// (the failure is transient — test never exhausted its retries)
expect(output).toContain('interrupted');
expect(output).not.toContain('1 of 1 failed');
});
test('interrupted test with retries is treated as terminal, not dropped', () => {
const reporter = new CypressStyleReporter();
const t1 = mockTest({
id: '1',
title: 'was running when interrupted',
file: 'tests/interrupt-retry.spec.ts',
retries: 2,
outcome: 'unexpected',
});
reporter.onBegin(mockConfig, mockSuite([t1]));
// Test is on retry 0 when SIGINT fires — status: 'interrupted', retry: 0
// Without fix: not terminal (interrupted !== passed/skipped, 0 !== 2)
// With fix: terminal (interrupted is always terminal)
reporter.onTestEnd(
t1,
mockResult({ status: 'interrupted', retry: 0, duration: 5000 }),
);
const output = getStdout();
expect(output).toContain('✗ was running when interrupted');
expect(output).toContain('Tests: 1');
});