|
| 1 | +/* |
| 2 | + * Wrap the "composer run wp-tests-phpunit" command to process tests |
| 3 | + * that are expected to error and fail at the moment. |
| 4 | + * |
| 5 | + * This makes sure that the CI job passes, while explicitly tracking |
| 6 | + * the issues that need to be addressed. Ideally, over time this script |
| 7 | + * will become obsolete when all errors and failures are resolved. |
| 8 | + */ |
| 9 | +const { execSync } = require( 'child_process' ); |
| 10 | +const fs = require( 'fs' ); |
| 11 | +const path = require( 'path' ); |
| 12 | + |
| 13 | +const expectedErrors = [ |
| 14 | + 'Tests_DB_Charset::test_strip_invalid_text', |
| 15 | + 'Tests_DB::test_db_reconnect', |
| 16 | + 'Tests_DB::test_get_col_info', |
| 17 | + 'Tests_DB::test_process_fields_value_too_long_for_field with data set "invalid chars"', |
| 18 | + 'Tests_DB::test_process_fields_value_too_long_for_field with data set "too long"', |
| 19 | + 'Tests_DB::test_process_fields', |
| 20 | +]; |
| 21 | + |
| 22 | +const expectedFailures = [ |
| 23 | + 'Tests_Admin_wpSiteHealth::test_object_cache_thresholds with data set #2', |
| 24 | + 'Tests_Admin_wpSiteHealth::test_object_cache_thresholds with data set #3', |
| 25 | + 'Tests_Comment::test_wp_new_comment_respects_comment_field_lengths', |
| 26 | + 'Tests_Comment::test_wp_update_comment', |
| 27 | + 'Tests_DB_dbDelta::test_spatial_indices', |
| 28 | + 'Tests_DB::test_charset_switched_to_utf8mb4', |
| 29 | + 'Tests_DB::test_close', |
| 30 | + 'Tests_DB::test_delete_value_too_long_for_field with data set "too long"', |
| 31 | + 'Tests_DB::test_has_cap', |
| 32 | + 'Tests_DB::test_insert_value_too_long_for_field with data set "too long"', |
| 33 | + 'Tests_DB::test_mysqli_flush_sync', |
| 34 | + 'Tests_DB::test_non_unicode_collations', |
| 35 | + 'Tests_DB::test_query_value_contains_invalid_chars', |
| 36 | + 'Tests_DB::test_replace_value_too_long_for_field with data set "too long"', |
| 37 | + 'Tests_DB::test_replace', |
| 38 | + 'Tests_DB::test_supports_collation', |
| 39 | + 'Tests_DB::test_update_value_too_long_for_field with data set "too long"', |
| 40 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #1', |
| 41 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #2', |
| 42 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #3', |
| 43 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #4', |
| 44 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #5', |
| 45 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #6', |
| 46 | + 'Tests_Menu_Walker_Nav_Menu::test_start_el_with_empty_attributes with data set #7', |
| 47 | + 'Tests_Menu_wpNavMenu::test_wp_nav_menu_should_not_have_has_children_class_with_custom_depth', |
| 48 | + 'WP_Test_REST_Posts_Controller::test_get_items_orderby_modified_query', |
| 49 | +]; |
| 50 | + |
| 51 | +console.log( 'Running WordPress PHPUnit tests with expected failures tracking...' ); |
| 52 | +console.log( 'Expected errors:', expectedErrors ); |
| 53 | +console.log( 'Expected failures:', expectedFailures ); |
| 54 | + |
| 55 | +try { |
| 56 | + try { |
| 57 | + execSync( |
| 58 | + `composer run wp-test-phpunit -- --log-junit=phpunit-results.xml --verbose`, |
| 59 | + { stdio: 'inherit' } |
| 60 | + ); |
| 61 | + console.log( '\n⚠️ All tests passed, checking if expected errors/failures occurred...' ); |
| 62 | + } catch ( error ) { |
| 63 | + console.log( '\n⚠️ Some tests errored/failed (expected). Analyzing results...' ); |
| 64 | + } |
| 65 | + |
| 66 | + // Read the JUnit XML test output: |
| 67 | + const junitOutputFile = path.join( __dirname, '..', '..', 'wordpress', 'phpunit-results.xml' ); |
| 68 | + if ( ! fs.existsSync( junitOutputFile ) ) { |
| 69 | + console.error( 'Error: JUnit output file not found!' ); |
| 70 | + process.exit( 1 ); |
| 71 | + } |
| 72 | + const junitXml = fs.readFileSync( junitOutputFile, 'utf8' ); |
| 73 | + |
| 74 | + // Extract test info from the XML: |
| 75 | + const actualErrors = []; |
| 76 | + const actualFailures = []; |
| 77 | + for ( const testcase of junitXml.matchAll( /<testcase([^>]*)\/>|<testcase([^>]*)>([\s\S]*?)<\/testcase>/g ) ) { |
| 78 | + const attributes = {}; |
| 79 | + const attributesString = testcase[2] ?? testcase[1]; |
| 80 | + for ( const attribute of attributesString.matchAll( /(\w+)="([^"]*)"/g ) ) { |
| 81 | + attributes[attribute[1]] = attribute[2]; |
| 82 | + } |
| 83 | + |
| 84 | + const content = testcase[3] ?? ''; |
| 85 | + const fqn = attributes.class ? `${attributes.class}::${attributes.name}` : attributes.name; |
| 86 | + const hasError = content.includes( '<error' ); |
| 87 | + const hasFailure = content.includes( '<failure' ); |
| 88 | + |
| 89 | + if ( hasError ) { |
| 90 | + actualErrors.push( fqn ); |
| 91 | + } |
| 92 | + |
| 93 | + if ( hasFailure ) { |
| 94 | + actualFailures.push( fqn ); |
| 95 | + } |
| 96 | + } |
| 97 | + |
| 98 | + let isSuccess = true; |
| 99 | + |
| 100 | + // Check if all expected errors actually errored |
| 101 | + const unexpectedNonErrors = expectedErrors.filter( test => ! actualErrors.includes( test ) ); |
| 102 | + if ( unexpectedNonErrors.length > 0 ) { |
| 103 | + console.error( '\n❌ The following tests were expected to error but did not:' ); |
| 104 | + unexpectedNonErrors.forEach( test => console.error( ` - ${test}` ) ); |
| 105 | + isSuccess = false; |
| 106 | + } |
| 107 | + |
| 108 | + // Check if all expected failures actually failed |
| 109 | + const unexpectedPasses = expectedFailures.filter( test => ! actualFailures.includes( test ) ); |
| 110 | + if ( unexpectedPasses.length > 0 ) { |
| 111 | + console.error( '\n❌ The following tests were expected to fail but passed:' ); |
| 112 | + unexpectedPasses.forEach( test => console.error( ` - ${test}` ) ); |
| 113 | + isSuccess = false; |
| 114 | + } |
| 115 | + |
| 116 | + // Check for unexpected errors |
| 117 | + const unexpectedErrors = actualErrors.filter( test => ! expectedErrors.includes( test ) ); |
| 118 | + if ( unexpectedErrors.length > 0 ) { |
| 119 | + console.error( '\n❌ The following tests errored unexpectedly:' ); |
| 120 | + unexpectedErrors.forEach( test => console.error( ` - ${test}` ) ); |
| 121 | + isSuccess = false; |
| 122 | + } |
| 123 | + |
| 124 | + // Check for unexpected failures |
| 125 | + const unexpectedFailures = actualFailures.filter( test => ! expectedFailures.includes( test ) ); |
| 126 | + if ( unexpectedFailures.length > 0 ) { |
| 127 | + console.error( '\n❌ The following tests failed unexpectedly:' ); |
| 128 | + unexpectedFailures.forEach( test => console.error( ` - ${test}` ) ); |
| 129 | + isSuccess = false; |
| 130 | + } |
| 131 | + |
| 132 | + if ( isSuccess ) { |
| 133 | + console.log( '\n✅ All tests behaved as expected!' ); |
| 134 | + process.exit( 0 ); |
| 135 | + } else { |
| 136 | + console.log( '\n❌ Some tests did not behave as expected!' ); |
| 137 | + process.exit( 1 ); |
| 138 | + } |
| 139 | +} catch ( error ) { |
| 140 | + console.error( '\n❌ Script execution error:', error.message ); |
| 141 | + process.exit( 1 ); |
| 142 | +} |
0 commit comments