feat: add location information for annotations (#36542)

Author: aslushnikov

docs/src/test-api/class-testinfo.md

@@ -19,6 +19,7 @@ test('basic test', async ({ page }, testInfo) => {
 - type: <[Array]<[Object]>>
   - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`.
   - `description` ?<[string]> Optional description.
+  - `location` ?<[Location]> Optional location in the source where the annotation is added.
 
 The list of annotations applicable to the current test. Includes annotations from the test, annotations from all [`method: Test.describe`] groups the test belongs to and file-level annotations for the test file.
 

docs/src/test-reporter-api/class-testcase.md

@@ -9,6 +9,7 @@
 - type: <[Array]<[Object]>>
   - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`.
   - `description` ?<[string]> Optional description.
+  - `location` ?<[Location]> Optional location in the source where the annotation is added.
 
 [`property: TestResult.annotations`] of the last test run.
 

docs/src/test-reporter-api/class-testresult.md

@@ -19,6 +19,7 @@ The list of files or buffers attached during the test execution through [`proper
 - type: <[Array]<[Object]>>
   - `type` <[string]> Annotation type, for example `'skip'` or `'fail'`.
   - `description` ?<[string]> Optional description.
+  - `location` ?<[Location]> Optional location in the source where the annotation is added.
 
 The list of annotations applicable to the current test. Includes:
 * annotations defined on the test or suite via [`method: Test.(call)`] and [`method: Test.describe`];

docs/src/test-reporter-api/class-teststep.md

@@ -58,6 +58,7 @@ List of steps inside this step.
 - type: <[Array]<[Object]>>
   - `type` <[string]> Annotation type, for example `'skip'`.
   - `description` ?<[string]> Optional description.
+  - `location` ?<[Location]> Optional location in the source where the annotation is added.
 
 The list of annotations applicable to the current test step.
 

packages/playwright/src/common/testType.ts

@@ -102,7 +102,7 @@ export class TestTypeImpl {
       details = fnOrDetails;
     }
 
-    const validatedDetails = validateTestDetails(details);
+    const validatedDetails = validateTestDetails(details, location);
     const test = new TestCase(title, body, this, location);
     test._requireFile = suite._requireFile;
     test.annotations.push(...validatedDetails.annotations);
@@ -112,9 +112,9 @@ export class TestTypeImpl {
     if (type === 'only' || type === 'fail.only')
       test._only = true;
     if (type === 'skip' || type === 'fixme' || type === 'fail')
-      test.annotations.push({ type });
+      test.annotations.push({ type, location });
     else if (type === 'fail.only')
-      test.annotations.push({ type: 'fail' });
+      test.annotations.push({ type: 'fail', location });
   }
 
   private _describe(type: 'default' | 'only' | 'serial' | 'serial.only' | 'parallel' | 'parallel.only' | 'skip' | 'fixme', location: Location, titleOrFn: string | Function, fnOrDetails?: TestDetails | Function, fn?: Function) {
@@ -141,7 +141,7 @@ export class TestTypeImpl {
       body = fn!;
     }
 
-    const validatedDetails = validateTestDetails(details);
+    const validatedDetails = validateTestDetails(details, location);
     const child = new Suite(title, 'describe');
     child._requireFile = suite._requireFile;
     child.location = location;
@@ -156,7 +156,7 @@ export class TestTypeImpl {
     if (type === 'parallel' || type === 'parallel.only')
       child._parallelMode = 'parallel';
     if (type === 'skip' || type === 'fixme')
-      child._staticAnnotations.push({ type });
+      child._staticAnnotations.push({ type, location });
 
     for (let parent: Suite | undefined = suite; parent; parent = parent.parent) {
       if (parent._parallelMode === 'serial' && child._parallelMode === 'parallel')
@@ -227,7 +227,7 @@ export class TestTypeImpl {
         if (modifierArgs.length >= 1 && !modifierArgs[0])
           return;
         const description = modifierArgs[1];
-        suite._staticAnnotations.push({ type, description });
+        suite._staticAnnotations.push({ type, description, location });
       }
       return;
     }
@@ -237,7 +237,7 @@ export class TestTypeImpl {
       throw new Error(`test.${type}() can only be called inside test, describe block or fixture`);
     if (typeof modifierArgs[0] === 'function')
       throw new Error(`test.${type}() with a function can only be called inside describe block`);
-    testInfo[type](...modifierArgs as [any, any]);
+    testInfo._modifier(type, location, modifierArgs as [any, any]);
   }
 
   private _setTimeout(location: Location, timeout: number) {
@@ -270,7 +270,7 @@ export class TestTypeImpl {
         let result: Awaited<ReturnType<typeof raceAgainstDeadline<T>>> | undefined = undefined;
         result = await raceAgainstDeadline(async () => {
           try {
-            return await step.info._runStepBody(expectation === 'skip', body);
+            return await step.info._runStepBody(expectation === 'skip', body, step.location);
           } catch (e) {
             // If the step timed out, the test fixtures will tear down, which in turn
             // will abort unfinished actions in the step body. Record such errors here.
@@ -309,8 +309,9 @@ function throwIfRunningInsideJest() {
   }
 }
 
-function validateTestDetails(details: TestDetails) {
-  const annotations = Array.isArray(details.annotation) ? details.annotation : (details.annotation ? [details.annotation] : []);
+function validateTestDetails(details: TestDetails, location: Location) {
+  const originalAnnotations = Array.isArray(details.annotation) ? details.annotation : (details.annotation ? [details.annotation] : []);
+  const annotations = originalAnnotations.map(annotation => ({ ...annotation, location }));
   const tags = Array.isArray(details.tag) ? details.tag : (details.tag ? [details.tag] : []);
   for (const tag of tags) {
     if (tag[0] !== '@')

packages/playwright/src/isomorphic/teleReceiver.ts

@@ -355,8 +355,9 @@ export class TeleReporterReceiver {
     if (!!payload.attachments)
       result.attachments = this._parseAttachments(payload.attachments);
     if (payload.annotations) {
+      this._absoluteAnnotationLocationsInplace(payload.annotations);
       result.annotations = payload.annotations;
-      test.annotations = result.annotations;
+      test.annotations = payload.annotations;
     }
     this._reporter.onTestEnd?.(test, result);
     // Free up the memory as won't see these step ids.
@@ -499,9 +500,17 @@ export class TeleReporterReceiver {
     test.retries = payload.retries;
     test.tags = payload.tags ?? [];
     test.annotations = payload.annotations ?? [];
+    this._absoluteAnnotationLocationsInplace(test.annotations);
     return test;
   }
 
+  private _absoluteAnnotationLocationsInplace(annotations: TestAnnotation[]) {
+    for (const annotation of annotations) {
+      if (annotation.location)
+        annotation.location = this._absoluteLocation(annotation.location);
+    }
+  }
+
   private _absoluteLocation(location: reporterTypes.Location): reporterTypes.Location;
   private _absoluteLocation(location?: reporterTypes.Location): reporterTypes.Location | undefined;
   private _absoluteLocation(location: reporterTypes.Location | undefined): reporterTypes.Location | undefined {

packages/playwright/src/reporters/html.ts

@@ -502,7 +502,15 @@ class HtmlBuilder {
 
   private _serializeAnnotations(annotations: api.TestCase['annotations']): TestAnnotation[] {
     // Annotations can be pushed directly, with a wrong type.
-    return annotations.map(a => ({ type: a.type, description: a.description === undefined ? undefined : String(a.description) }));
+    return annotations.map(a => ({
+      type: a.type,
+      description: a.description === undefined ? undefined : String(a.description),
+      location: a.location ? {
+        file: a.location.file,
+        line: a.location.line,
+        column: a.location.column,
+      } : undefined,
+    }));
   }
 
   private _createTestResult(test: api.TestCase, result: api.TestResult): TestResult {

packages/playwright/src/reporters/merge.ts

@@ -26,7 +26,7 @@ import { TeleReporterReceiver } from '../isomorphic/teleReceiver';
 import { createReporters } from '../runner/reporters';
 import { relativeFilePath } from '../util';
 
-import type { ReporterDescription } from '../../types/test';
+import type { ReporterDescription, TestAnnotation } from '../../types/test';
 import type { TestError } from '../../types/testReporter';
 import type { FullConfigInternal } from '../common/config';
 import type { BlobReportMetadata, JsonAttachment, JsonConfig, JsonEvent, JsonFullResult, JsonLocation, JsonOnConfigureEvent, JsonOnEndEvent, JsonOnProjectEvent, JsonProject, JsonSuite, JsonTestCase } from '../isomorphic/teleReceiver';
@@ -484,7 +484,10 @@ class PathSeparatorPatcher {
       return;
     }
     if (jsonEvent.method === 'onTestEnd') {
+      const test = jsonEvent.params.test;
+      test.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
       const testResult = jsonEvent.params.result;
+      testResult.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
       testResult.errors.forEach(error => this._updateErrorLocations(error));
       (testResult.attachments ?? []).forEach(attachment => {
         if (attachment.path)
@@ -500,6 +503,7 @@ class PathSeparatorPatcher {
     if (jsonEvent.method === 'onStepEnd') {
       const step = jsonEvent.params.step;
       this._updateErrorLocations(step.error);
+      step.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
       return;
     }
     if (jsonEvent.method === 'onAttach') {
@@ -524,10 +528,12 @@ class PathSeparatorPatcher {
     if (isFileSuite)
       suite.title = this._updatePath(suite.title);
     for (const entry of suite.entries) {
-      if ('testId' in entry)
+      if ('testId' in entry) {
         this._updateLocation(entry.location);
-      else
+        entry.annotations?.forEach(annotation => this._updateAnnotationLocation(annotation));
+      } else {
         this._updateSuite(entry);
+      }
     }
   }
 
@@ -538,6 +544,10 @@ class PathSeparatorPatcher {
     }
   }
 
+  private _updateAnnotationLocation(annotation: TestAnnotation) {
+    this._updateLocation(annotation.location);
+  }
+
   private _updateLocation(location?: JsonLocation) {
     if (location)
       location.file = this._updatePath(location.file);

packages/playwright/src/reporters/teleEmitter.ts

@@ -22,6 +22,7 @@ import { serializeRegexPatterns } from '../isomorphic/teleReceiver';
 
 import type { ReporterV2 } from './reporterV2';
 import type * as reporterTypes from '../../types/testReporter';
+import type { TestAnnotation } from '../../types/test';
 import type * as teleReceiver from '../isomorphic/teleReceiver';
 
 export type TeleReporterEmitterOptions = {
@@ -224,7 +225,7 @@ export class TeleReporterEmitter implements ReporterV2 {
       retries: test.retries,
       tags: test.tags,
       repeatEachIndex: test.repeatEachIndex,
-      annotations: test.annotations,
+      annotations: this._relativeAnnotationLocations(test.annotations),
     };
   }
 
@@ -244,7 +245,7 @@ export class TeleReporterEmitter implements ReporterV2 {
       duration: result.duration,
       status: result.status,
       errors: result.errors,
-      annotations: result.annotations?.length ? result.annotations : undefined,
+      annotations: result.annotations?.length ? this._relativeAnnotationLocations(result.annotations) : undefined,
     };
   }
 
@@ -294,10 +295,17 @@ export class TeleReporterEmitter implements ReporterV2 {
       duration: step.duration,
       error: step.error,
       attachments: step.attachments.length ? step.attachments.map(a => result.attachments.indexOf(a)) : undefined,
-      annotations: step.annotations.length ? step.annotations : undefined,
+      annotations: step.annotations.length ? this._relativeAnnotationLocations(step.annotations) : undefined,
     };
   }
 
+  private _relativeAnnotationLocations(annotations: TestAnnotation[]): TestAnnotation[] {
+    return annotations.map(annotation => ({
+      ...annotation,
+      location: annotation.location ? this._relativeLocation(annotation.location) : undefined,
+    }));
+  }
+
   private _relativeLocation(location: reporterTypes.Location): reporterTypes.Location;
   private _relativeLocation(location?: reporterTypes.Location): reporterTypes.Location | undefined;
   private _relativeLocation(location: reporterTypes.Location | undefined): reporterTypes.Location | undefined {

packages/playwright/src/worker/testInfo.ts

@@ -23,6 +23,7 @@ import { TimeoutManager, TimeoutManagerError, kMaxDeadline } from './timeoutMana
 import { addSuffixToFilePath, filteredStackTrace, getContainedPath, normalizeAndSaveAttachment, sanitizeFilePathBeforeExtension, trimLongString, windowsFilesystemFriendlyLength } from '../util';
 import { TestTracing } from './testTracing';
 import { testInfoError } from './util';
+import { wrapFunctionWithLocation } from '../transform/transform';
 
 import type { RunnableDescription } from './timeoutManager';
 import type { FullProject, TestInfo, TestStatus, TestStepInfo, TestAnnotation } from '../../types/test';
@@ -80,6 +81,12 @@ export class TestInfoImpl implements TestInfo {
   _hasUnhandledError = false;
   _allowSkips = false;
 
+  // ------------ Main methods ------------
+  skip: (arg?: any, description?: string) => void;
+  fixme: (arg?: any, description?: string) => void;
+  fail: (arg?: any, description?: string) => void;
+  slow: (arg?: any, description?: string) => void;
+
   // ------------ TestInfo fields ------------
   readonly testId: string;
   readonly repeatEachIndex: number;
@@ -205,9 +212,14 @@ export class TestInfoImpl implements TestInfo {
     };
 
     this._tracing = new TestTracing(this, workerParams.artifactsDir);
+
+    this.skip = wrapFunctionWithLocation((location, ...args) => this._modifier('skip', location, args));
+    this.fixme = wrapFunctionWithLocation((location, ...args) => this._modifier('fixme', location, args));
+    this.fail = wrapFunctionWithLocation((location, ...args) => this._modifier('fail', location, args));
+    this.slow = wrapFunctionWithLocation((location, ...args) => this._modifier('slow', location, args));
   }
 
-  private _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', modifierArgs: [arg?: any, description?: string]) {
+  _modifier(type: 'skip' | 'fail' | 'fixme' | 'slow', location: Location, modifierArgs: [arg?: any, description?: string]) {
     if (typeof modifierArgs[1] === 'function') {
       throw new Error([
         'It looks like you are calling test.skip() inside the test and pass a callback.',
@@ -222,7 +234,7 @@ export class TestInfoImpl implements TestInfo {
       return;
 
     const description = modifierArgs[1];
-    this.annotations.push({ type, description });
+    this.annotations.push({ type, description, location });
     if (type === 'slow') {
       this._timeoutManager.slow();
     } else if (type === 'skip' || type === 'fixme') {
@@ -567,22 +579,6 @@ export class TestInfoImpl implements TestInfo {
     return this._resolveSnapshotPaths(kind, name.length <= 1 ? name[0] : name, 'dontUpdateSnapshotIndex').absoluteSnapshotPath;
   }
 
-  skip(...args: [arg?: any, description?: string]) {
-    this._modifier('skip', args);
-  }
-
-  fixme(...args: [arg?: any, description?: string]) {
-    this._modifier('fixme', args);
-  }
-
-  fail(...args: [arg?: any, description?: string]) {
-    this._modifier('fail', args);
-  }
-
-  slow(...args: [arg?: any, description?: string]) {
-    this._modifier('slow', args);
-  }
-
   setTimeout(timeout: number) {
     this._timeoutManager.setTimeout(timeout);
   }
@@ -594,14 +590,25 @@ export class TestStepInfoImpl implements TestStepInfo {
   private _testInfo: TestInfoImpl;
   private _stepId: string;
 
+  skip: (arg?: any, description?: string) => void;
+
   constructor(testInfo: TestInfoImpl, stepId: string) {
     this._testInfo = testInfo;
     this._stepId = stepId;
+    this.skip = wrapFunctionWithLocation((location: Location, ...args: unknown[]) => {
+      // skip();
+      // skip(condition: boolean, description: string);
+      if (args.length > 0 && !args[0])
+        return;
+      const description = args[1] as (string|undefined);
+      this.annotations.push({ type: 'skip', description, location });
+      throw new StepSkipError(description);
+    });
   }
 
-  async _runStepBody<T>(skip: boolean, body: (step: TestStepInfo) => T | Promise<T>) {
+  async _runStepBody<T>(skip: boolean, body: (step: TestStepInfo) => T | Promise<T>, location?: Location) {
     if (skip) {
-      this.annotations.push({ type: 'skip' });
+      this.annotations.push({ type: 'skip', location });
       return undefined as T;
     }
     try {
@@ -620,16 +627,6 @@ export class TestStepInfoImpl implements TestStepInfo {
   async attach(name: string, options?: { body?: string | Buffer; contentType?: string; path?: string; }): Promise<void> {
     this._attachToStep(await normalizeAndSaveAttachment(this._testInfo.outputPath(), name, options));
   }
-
-  skip(...args: unknown[]) {
-    // skip();
-    // skip(condition: boolean, description: string);
-    if (args.length > 0 && !args[0])
-      return;
-    const description = args[1] as (string|undefined);
-    this.annotations.push({ type: 'skip', description });
-    throw new StepSkipError(description);
-  }
 }
 
 export class TestSkipError extends Error {