diff --git a/README.md b/README.md index 31e5bc8..f41f96f 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,7 @@ The `junit-upload` and `playwright-json-upload` commands upload test results fro - `--run-name` - Optional name template for creating new test run when run url is not specified (supports `{env:VAR}`, `{YYYY}`, `{YY}`, `{MM}`, `{MMM}`, `{DD}`, `{HH}`, `{hh}`, `{mm}`, `{ss}`, `{AMPM}` placeholders). If not specified, `Automated test run - {MMM} {DD}, {YYYY}, {hh}:{mm}:{ss} {AMPM}` is used as default - `--attachments` - Try to detect and upload any attachments with the test result - `--force` - Ignore API request errors, invalid test cases, or attachments +- `--ignore-unmatched` - Suppress individual unmatched test messages, show summary only - `-h, --help` - Show command help ### Run Name Template Placeholders @@ -130,6 +131,12 @@ Ensure the required environment variables are defined before running these comma qasphere junit-upload --force ./test-results.xml ``` +8. Suppress unmatched test messages (useful during gradual test case linking): + ```bash + qasphere junit-upload --ignore-unmatched ./test-results.xml + ``` + This will show only a summary like "Skipped 5 unmatched tests" instead of individual error messages for each unmatched test. + ## Test Report Requirements The QAS CLI requires test cases in your reports (JUnit XML or Playwright JSON) to reference corresponding test cases in QA Sphere. These references are used to map test results from your automation to the appropriate test cases in QA Sphere. If a report lacks these references or the referenced test case doesn't exist in QA Sphere, the tool will display an error message. diff --git a/package.json b/package.json index 296da98..fe8eed8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "qas-cli", - "version": "0.4.0", + "version": "0.4.1", "description": "QAS CLI is a command line tool for submitting your automation test results to QA Sphere at https://qasphere.com/", "type": "module", "main": "./build/bin/qasphere.js", diff --git a/src/commands/resultUpload.ts b/src/commands/resultUpload.ts index 2d171b0..b24acc1 100644 --- a/src/commands/resultUpload.ts +++ b/src/commands/resultUpload.ts @@ -49,6 +49,10 @@ export class ResultUploadCommandModule implements CommandModule { expect(tcaseUploadCount()).toBe(4) }) + test('Test cases on reports with missing test cases should be successful with --ignore-unmatched', async () => { + const fileUploadCount = countFileUploadApiCalls() + const tcaseUploadCount = countResultUploadApiCalls() + await run( + `${fileType.command} -r ${runURL} --ignore-unmatched ${fileType.dataBasePath}/missing-tcases.${fileType.fileExtension}` + ) + expect(fileUploadCount()).toBe(0) + expect(tcaseUploadCount()).toBe(4) + }) + test('Test cases from multiple reports should be processed successfully', async () => { const fileUploadCount = countFileUploadApiCalls() const tcaseUploadCount = countResultUploadApiCalls() diff --git a/src/utils/result-upload/ResultUploadCommandHandler.ts b/src/utils/result-upload/ResultUploadCommandHandler.ts index d80dec9..d05db1d 100644 --- a/src/utils/result-upload/ResultUploadCommandHandler.ts +++ b/src/utils/result-upload/ResultUploadCommandHandler.ts @@ -21,6 +21,7 @@ export interface ResultUploadCommandArgs { files: string[] force: boolean attachments: boolean + ignoreUnmatched: boolean } interface FileResults { @@ -117,11 +118,12 @@ export class ResultUploadCommandHandler { protected extractTestCaseRefs(projectCode: string, fileResults: FileResults[]): Set { const tcaseRefs = new Set() + const shouldFailOnInvalid = !this.args.force && !this.args.ignoreUnmatched for (const { file, results } of fileResults) { for (const result of results) { if (!result.name) { - if (!this.args.force) { + if (shouldFailOnInvalid) { return printErrorThenExit(`Test case in ${file} has no name`) } continue @@ -130,7 +132,10 @@ export class ResultUploadCommandHandler { const match = new RegExp(`${projectCode}-(\\d{3,})`).exec(result.name) if (match) { tcaseRefs.add(`${projectCode}-${match[1]}`) - } else if (!this.args.force) { + continue + } + + if (shouldFailOnInvalid) { return printErrorThenExit( `Test case name "${result.name}" in ${file} does not contain valid sequence number with project code (e.g., ${projectCode}-123)` ) diff --git a/src/utils/result-upload/ResultUploader.ts b/src/utils/result-upload/ResultUploader.ts index fa0abbb..d3c26d5 100644 --- a/src/utils/result-upload/ResultUploader.ts +++ b/src/utils/result-upload/ResultUploader.ts @@ -39,6 +39,24 @@ export class ResultUploader { } private validateAndPrintMissingTestCases(missing: TestCaseResult[]) { + if (!missing.length) { + return + } + + if (this.args.ignoreUnmatched) { + this.printMissingTestCaseSummary(missing.length) + return + } + + this.printMissingTestCaseErrors(missing) + this.printMissingTestCaseGuidance(missing) + + if (!this.args.force) { + process.exit(1) + } + } + + private printMissingTestCaseErrors(missing: TestCaseResult[]) { missing.forEach((item) => { const folderMessage = item.folder ? ` "${item.folder}" ->` : '' const header = this.args.force ? chalk.yellow('Warning:') : chalk.red('Error:') @@ -46,10 +64,23 @@ export class ResultUploader { `${header}${chalk.blue(`${folderMessage} "${item.name}"`)} does not match any test cases` ) }) + } + + private printMissingTestCaseSummary(count: number) { + console.log(chalk.dim(`\nSkipped ${count} unmatched test${count === 1 ? '' : 's'}`)) + } - if (missing.length) { - if (this.type === 'junit-upload') { - console.error(` + private printMissingTestCaseGuidance(missing: TestCaseResult[]) { + if (this.type === 'junit-upload') { + this.printJUnitGuidance() + } else if (this.type === 'playwright-json-upload') { + this.printPlaywrightGuidance(missing[0]?.name || 'your test name') + } + console.error(chalk.yellow('Also ensure that the test cases exist in the QA Sphere project and the test run (if run URL is provided).')) + } + + private printJUnitGuidance() { + console.error(` ${chalk.yellow('To fix this issue, include the test case marker in your test names:')} Format: ${chalk.green(`${this.project}-: Your test name`)} @@ -58,14 +89,16 @@ ${chalk.yellow('To fix this issue, include the test case marker in your test nam ${chalk.dim('Where is the test case number (minimum 3 digits, zero-padded if needed)')} `) - } else { - console.error(` + } + + private printPlaywrightGuidance(exampleTestName: string) { + console.error(` ${chalk.yellow('To fix this issue, choose one of the following options:')} ${chalk.bold('Option 1: Use Test Annotations (Recommended)')} Add a test annotation to your Playwright test: - ${chalk.green(`test('${missing[0]?.name || 'your test name'}', { + ${chalk.green(`test('${exampleTestName}', { annotation: { type: 'test case', description: 'https://your-qas-instance.com/project/${this.project}/tcase/123' @@ -83,14 +116,6 @@ ${chalk.yellow('To fix this issue, choose one of the following options:')} Example: ${chalk.green(`${this.project}-1024: Login with valid credentials`)} ${chalk.dim('Where is the test case number (minimum 3 digits, zero-padded if needed)')} `) - } - - console.error(chalk.yellow('Also ensure that the test cases exist in the QA Sphere project and the test run (if run URL is provided).')) - } - - if (missing.length && !this.args.force) { - process.exit(1) - } } private validateAndPrintMissingAttachments = (results: TCaseWithResult[]) => {