Merge pull request #1 from bencsr/publish_transient_retry_failure

report transient retry failures

Author: ryanwilsonperkin

README.md

@@ -74,6 +74,7 @@ Reporter options should also be strings exception for suiteNameTemplate, classNa
 | `JEST_JUNIT_INCLUDE_SHORT_CONSOLE_OUTPUT` | `includeShortConsoleOutput` | Adds short console output (only message value) to any testSuite that generates stdout during a test run. | `false` | N/A
 | `JEST_JUNIT_REPORT_TEST_SUITE_ERRORS` | `reportTestSuiteErrors` | Reports test suites that failed to execute altogether as `error`. _Note:_ since the suite name cannot be determined from files that fail to load, it will default to file path.| `false` | N/A
 | `JEST_JUNIT_NO_STACK_TRACE` | `noStackTrace` | Omit stack traces from test failure reports, similar to `jest --noStackTrace` | `false` | N/A
+| `JEST_JUNIT_PUBLISH_TRANSIENT_RETRY_FAILURES` | `publishTransientRetryFailures` | Create separate `testcase` entries with `failure` for transient errors when using `jest.retryTimes`. | `false` | N/A
 | `JEST_USE_PATH_FOR_SUITE_NAME` | `usePathForSuiteName` | **DEPRECATED. Use `suiteNameTemplate` instead.** Use file path as the `name` attribute of `<testsuite>` | `"false"` | N/A
 | `JEST_JUNIT_TEST_CASE_PROPERTIES_JSON_FILE` | `testCasePropertiesFile` | Name of the custom testcase properties file | `"junitProperties.js"` | N/A
 | `JEST_JUNIT_TEST_CASE_PROPERTIES_DIR` | `testCasePropertiesDirectory` | Location of the custom testcase properties file | `process.cwd()` | N/A

__mocks__/retried-tests.json

@@ -56,7 +56,7 @@
           "status": "passed",
           "title": "should bar",
           "invocations": 2,
-          "retryReasons": ["error"]
+          "retryReasons": ["error on first try"]
         }
       ],
       "skipped": false

__tests__/buildJsonResults.test.js

@@ -519,4 +519,74 @@ describe('buildJsonResults', () => {
       }
       `)
   });
+
+  it("should add transient retry-failures as testcase", () => {
+    const retriedTestsReport = require("../__mocks__/retried-tests.json");
+
+    // Mock Date.now() to return a fixed later value
+    const startDate = new Date(retriedTestsReport.startTime);
+    jest.spyOn(Date, 'now').mockImplementation(() => startDate.getTime() + 1234);
+
+    jsonResults = buildJsonResults(retriedTestsReport, "/", {
+      ...constants.DEFAULT_OPTIONS,
+      testSuitePropertiesFile: "not-existing-file", // Skip the contents of the default junitProperties.js
+      publishTransientRetryFailures: "true",
+    });
+
+    expect(jsonResults).toMatchInlineSnapshot(`
+      Object {
+        "testsuites": Array [
+          Object {
+            "_attr": Object {
+              "errors": 0,
+              "failures": 0,
+              "name": "jest tests",
+              "tests": 1,
+              "time": 1.234,
+            },
+          },
+          Object {
+            "testsuite": Array [
+              Object {
+                "_attr": Object {
+                  "errors": 0,
+                  "failures": 0,
+                  "name": "foo",
+                  "skipped": 0,
+                  "tests": 1,
+                  "time": 0.12,
+                  "timestamp": "2017-03-17T01:05:47",
+                },
+              },
+              Object {
+                "testcase": Array [
+                  Object {
+                    "_attr": Object {
+                      "classname": "foo baz should bar",
+                      "name": "foo baz should bar",
+                      "time": 0.001,
+                    },
+                  },
+                  Object {
+                    "failure": "error on first try",
+                  },
+                ],
+              },
+              Object {
+                "testcase": Array [
+                  Object {
+                    "_attr": Object {
+                      "classname": "foo baz should bar",
+                      "name": "foo baz should bar",
+                      "time": 0.001,
+                    },
+                  },
+                ],
+              },
+            ],
+          },
+        ],
+      }
+      `);
+  });
 });

constants/index.js

@@ -17,6 +17,7 @@ module.exports = {
     JEST_JUNIT_INCLUDE_SHORT_CONSOLE_OUTPUT: 'includeShortConsoleOutput',
     JEST_JUNIT_REPORT_TEST_SUITE_ERRORS: 'reportTestSuiteErrors',
     JEST_JUNIT_NO_STACK_TRACE: "noStackTrace",
+    JEST_JUNIT_PUBLISH_TRANSIENT_RETRY_FAILURES: "publishTransientRetryFailures",
     JEST_USE_PATH_FOR_SUITE_NAME: 'usePathForSuiteName',
     JEST_JUNIT_TEST_CASE_PROPERTIES_JSON_FILE: 'testCasePropertiesFile',
     JEST_JUNIT_TEST_CASE_PROPERTIES_DIR: 'testCasePropertiesDirectory',
@@ -39,6 +40,7 @@ module.exports = {
     includeShortConsoleOutput: 'false',
     reportTestSuiteErrors: 'false',
     noStackTrace: 'false',
+    publishTransientRetryFailures: 'false',
     testCasePropertiesFile: 'junitTestCaseProperties.js',
     testCasePropertiesDirectory: process.cwd(),
     testSuitePropertiesFile: 'junitProperties.js',

utils/buildJsonResults.js

@@ -265,6 +265,24 @@ module.exports = function (report, appDirectory, options, rootDir = null) {
 
     // Iterate through test cases
     suite.testResults.forEach((tc) => {
+      if (options.publishTransientRetryFailures === 'true' && Array.isArray(tc.retryReasons)) {
+        const retriedFailureCases = tc.retryReasons.map(rr => {
+              const tCase = generateTestCase(
+                  options,
+                  suiteOptions,
+                  tc,
+                  filepath,
+                  filename,
+                  suiteTitle,
+                  displayName,
+                  getTestCaseProperties
+              );
+              tCase.testcase.push({"failure": strip(rr)});
+              return tCase;
+            });
+        testSuite.testsuite.push(...retriedFailureCases);
+      }
+
       const testCase = generateTestCase(
         options,
         suiteOptions,