From 05592c7f6c3f2db42bb1eb94d5d7a49d59541318 Mon Sep 17 00:00:00 2001 From: Eyal Roth Date: Sat, 27 Jul 2024 19:29:45 +0300 Subject: [PATCH 1/3] #15215 Reduce memory leak in node env by shredding global properties in teardown --- packages/jest-circus/src/state.ts | 25 +++- packages/jest-environment-node/src/index.ts | 128 +++++++++++++++++++- packages/jest-repl/src/cli/runtime-cli.ts | 6 +- packages/jest-runner/src/runTest.ts | 2 +- packages/jest-util/src/index.ts | 1 + packages/jest-util/src/setGlobal.ts | 10 +- packages/jest-util/src/shredder.ts | 54 +++++++++ 7 files changed, 210 insertions(+), 16 deletions(-) create mode 100644 packages/jest-util/src/shredder.ts diff --git a/packages/jest-circus/src/state.ts b/packages/jest-circus/src/state.ts index deda31560871..053f251fff07 100644 --- a/packages/jest-circus/src/state.ts +++ b/packages/jest-circus/src/state.ts @@ -6,6 +6,7 @@ */ import type {Circus, Global} from '@jest/types'; +import {setGlobal, setNotShreddable} from 'jest-util'; import eventHandler from './eventHandler'; import formatNodeAssertErrors from './formatNodeAssertErrors'; import {STATE_SYM} from './types'; @@ -39,16 +40,28 @@ const createState = (): Circus.State => { }; /* eslint-disable no-restricted-globals */ +export const getState = (): Circus.State => + (global as Global.Global)[STATE_SYM] as Circus.State; +export const setState = (state: Circus.State): Circus.State => { + setGlobal(global, STATE_SYM, state); + setNotShreddable(state, [ + 'hasFocusedTests', + 'hasStarted', + 'includeTestLocationInResult', + 'maxConcurrency', + 'seed', + 'testNamePattern', + 'testTimeout', + 'unhandledErrors', + 'unhandledRejectionErrorByPromise', + ]); + return state; +}; export const resetState = (): void => { - (global as Global.Global)[STATE_SYM] = createState(); + setState(createState()); }; resetState(); - -export const getState = (): Circus.State => - (global as Global.Global)[STATE_SYM] as Circus.State; -export const setState = (state: Circus.State): Circus.State => - ((global as Global.Global)[STATE_SYM] = state); /* eslint-enable */ export const dispatch = async (event: Circus.AsyncEvent): Promise => { diff --git a/packages/jest-environment-node/src/index.ts b/packages/jest-environment-node/src/index.ts index 11a36e97801b..a6a0afa6dccb 100644 --- a/packages/jest-environment-node/src/index.ts +++ b/packages/jest-environment-node/src/index.ts @@ -14,7 +14,12 @@ import type { import {LegacyFakeTimers, ModernFakeTimers} from '@jest/fake-timers'; import type {Global} from '@jest/types'; import {ModuleMocker} from 'jest-mock'; -import {installCommonGlobals} from 'jest-util'; +import { + installCommonGlobals, + isShreddable, + setNotShreddable, + shred, +} from 'jest-util'; type Timer = { id: number; @@ -80,12 +85,13 @@ export default class NodeEnvironment implements JestEnvironment { moduleMocker: ModuleMocker | null; customExportConditions = ['node', 'node-addons']; private readonly _configuredExportConditions?: Array; + private _globalProxy: GlobalProxy; // while `context` is unused, it should always be passed constructor(config: JestEnvironmentConfig, _context: EnvironmentContext) { const {projectConfig} = config; - this.context = createContext(); - + this._globalProxy = new GlobalProxy(); + this.context = createContext(this._globalProxy.proxy()); const global = runInContext( 'this', Object.assign(this.context, projectConfig.testEnvironmentOptions), @@ -194,6 +200,8 @@ export default class NodeEnvironment implements JestEnvironment { config: projectConfig, global, }); + + this._globalProxy.envSetupCompleted(); } // eslint-disable-next-line @typescript-eslint/no-empty-function @@ -209,6 +217,7 @@ export default class NodeEnvironment implements JestEnvironment { this.context = null; this.fakeTimers = null; this.fakeTimersModern = null; + this._globalProxy.clear(); } exportConditions(): Array { @@ -221,3 +230,116 @@ export default class NodeEnvironment implements JestEnvironment { } export const TestEnvironment = NodeEnvironment; + +/** + * Creates a new empty global object and wraps it with a {@link Proxy}. + * + * The purpose is to register any property set on the global object, + * and {@link #shred} them at environment teardown, to clean up memory and + * prevent leaks. + */ +class GlobalProxy implements ProxyHandler { + private global: typeof globalThis = Object.create( + Object.getPrototypeOf(globalThis), + ); + private globalProxy: typeof globalThis = new Proxy(this.global, this); + private isEnvSetup = false; + private propertyToValue = new Map(); + private leftovers: Array<{property: string | symbol; value: unknown}> = []; + + constructor() { + this.register = this.register.bind(this); + } + + proxy(): typeof globalThis { + return this.globalProxy; + } + + /** + * Marks that the environment setup has completed, and properties set on + * the global object from now on should be shredded at teardown. + */ + envSetupCompleted(): void { + this.isEnvSetup = true; + } + + /** + * Shreds any property that was set on the global object, except for: + * 1. Properties that were set before {@link #envSetupCompleted} was invoked. + * 2. Properties protected by {@link #setNotShreddable}. + */ + clear(): void { + for (const {property, value} of [ + ...[...this.propertyToValue.entries()].map(([property, value]) => ({ + property, + value, + })), + ...this.leftovers, + ]) { + /* + * react-native invoke its custom `performance` property after env teardown. + * its setup file should use `setNotShreddable` to prevent this. + */ + if (property !== 'performance') { + shred(value); + } + } + this.propertyToValue.clear(); + this.leftovers = []; + this.global = {} as typeof globalThis; + this.globalProxy = {} as typeof globalThis; + } + + defineProperty( + target: typeof globalThis, + property: string | symbol, + attributes: PropertyDescriptor, + ): boolean { + const newAttributes = {...attributes}; + + if ('set' in newAttributes && newAttributes.set !== undefined) { + const originalSet = newAttributes.set; + const register = this.register; + newAttributes.set = value => { + originalSet(value); + const newValue = Reflect.get(target, property); + register(property, newValue); + }; + } + + const result = Reflect.defineProperty(target, property, newAttributes); + + if ('value' in newAttributes) { + this.register(property, newAttributes.value); + } + + return result; + } + + deleteProperty( + target: typeof globalThis, + property: string | symbol, + ): boolean { + const result = Reflect.deleteProperty(target, property); + const value = this.propertyToValue.get(property); + if (value) { + this.leftovers.push({property, value}); + this.propertyToValue.delete(property); + } + return result; + } + + private register(property: string | symbol, value: unknown) { + const currentValue = this.propertyToValue.get(property); + if (value !== currentValue) { + if (!this.isEnvSetup && isShreddable(value)) { + setNotShreddable(value); + } + if (currentValue) { + this.leftovers.push({property, value: currentValue}); + } + + this.propertyToValue.set(property, value); + } + } +} diff --git a/packages/jest-repl/src/cli/runtime-cli.ts b/packages/jest-repl/src/cli/runtime-cli.ts index 5997b040a769..394528f1963e 100644 --- a/packages/jest-repl/src/cli/runtime-cli.ts +++ b/packages/jest-repl/src/cli/runtime-cli.ts @@ -99,9 +99,9 @@ export async function run( }, {console: customConsole, docblockPragmas: {}, testPath: filePath}, ); - setGlobal(environment.global, 'console', customConsole); - setGlobal(environment.global, 'jestProjectConfig', projectConfig); - setGlobal(environment.global, 'jestGlobalConfig', globalConfig); + setGlobal(environment.global, 'console', customConsole, false); + setGlobal(environment.global, 'jestProjectConfig', projectConfig, false); + setGlobal(environment.global, 'jestGlobalConfig', globalConfig, false); const runtime = new Runtime( projectConfig, diff --git a/packages/jest-runner/src/runTest.ts b/packages/jest-runner/src/runTest.ts index ed9a5ed951bd..3bac869fa096 100644 --- a/packages/jest-runner/src/runTest.ts +++ b/packages/jest-runner/src/runTest.ts @@ -183,7 +183,7 @@ async function runTestInternal( ? new LeakDetector(environment) : null; - setGlobal(environment.global, 'console', testConsole); + setGlobal(environment.global, 'console', testConsole, false); const runtime = new Runtime( projectConfig, diff --git a/packages/jest-util/src/index.ts b/packages/jest-util/src/index.ts index dbfd9025175b..e82663821315 100644 --- a/packages/jest-util/src/index.ts +++ b/packages/jest-util/src/index.ts @@ -29,3 +29,4 @@ export {default as tryRealpath} from './tryRealpath'; export {default as requireOrImportModule} from './requireOrImportModule'; export {default as invariant} from './invariant'; export {default as isNonNullable} from './isNonNullable'; +export {isShreddable, setNotShreddable, shred} from './shredder'; diff --git a/packages/jest-util/src/setGlobal.ts b/packages/jest-util/src/setGlobal.ts index 4daa5132ec49..ece978243302 100644 --- a/packages/jest-util/src/setGlobal.ts +++ b/packages/jest-util/src/setGlobal.ts @@ -6,12 +6,16 @@ */ import type {Global} from '@jest/types'; +import {isShreddable, setNotShreddable} from './shredder'; export default function setGlobal( globalToMutate: typeof globalThis | Global.Global, - key: string, + key: string | symbol, value: unknown, + shredAfterTeardown = true, ): void { - // @ts-expect-error: no index - globalToMutate[key] = value; + Reflect.set(globalToMutate, key, value); + if (!shredAfterTeardown && isShreddable(value)) { + setNotShreddable(value); + } } diff --git a/packages/jest-util/src/shredder.ts b/packages/jest-util/src/shredder.ts new file mode 100644 index 000000000000..209c6c4a71c7 --- /dev/null +++ b/packages/jest-util/src/shredder.ts @@ -0,0 +1,54 @@ +/** + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +const NO_SHRED_AFTER_TEARDOWN = Symbol.for('$$jest-no-shred'); + +/** + * Deletes all the properties from the given value (if it's an object), + * unless the value was protected via {@link #setNotShreddable}. + * + * @param value the given value. + */ +export function shred(value: unknown): void { + if (isShreddable(value)) { + const protectedProperties = Reflect.get(value, NO_SHRED_AFTER_TEARDOWN); + if (!Array.isArray(protectedProperties) || protectedProperties.length > 0) { + for (const key of Reflect.ownKeys(value)) { + if (!protectedProperties?.includes(key)) { + Reflect.deleteProperty(value, key); + } + } + } + } +} + +/** + * Protects the given value from being shredded by {@link #shred}. + * + * @param value The given value. + * @param properties If the array contains any property, + * then only these properties will not be deleted; otherwise if the array is empty, + * all properties will not be deleted. + */ +export function setNotShreddable( + value: T, + properties: Array = [], +): boolean { + if (isShreddable(value)) { + return Reflect.set(value, NO_SHRED_AFTER_TEARDOWN, properties); + } + return false; +} + +/** + * Whether the given value is possible to be shredded. + * + * @param value The given value. + */ +export function isShreddable(value: unknown): value is object { + return value !== null && ['object', 'function'].includes(typeof value); +} From c1b3d6d786db5a0e4fd2b950927b226dd7040e51 Mon Sep 17 00:00:00 2001 From: Eyal Roth Date: Sat, 24 May 2025 17:11:37 +0300 Subject: [PATCH 2/3] #15215 Rename 'shredder' API to something closer to garbage collection --- packages/jest-circus/src/state.ts | 4 ++-- packages/jest-environment-node/src/index.ts | 24 +++++++++---------- packages/jest-repl/src/cli/runtime-cli.ts | 6 ++--- packages/jest-runner/src/runTest.ts | 2 +- ...hredder.ts => garbage-collection-utils.ts} | 22 ++++++++--------- packages/jest-util/src/index.ts | 6 ++++- packages/jest-util/src/setGlobal.ts | 11 +++++---- 7 files changed, 41 insertions(+), 34 deletions(-) rename packages/jest-util/src/{shredder.ts => garbage-collection-utils.ts} (59%) diff --git a/packages/jest-circus/src/state.ts b/packages/jest-circus/src/state.ts index 5e1f8468fe17..45a8192627c9 100644 --- a/packages/jest-circus/src/state.ts +++ b/packages/jest-circus/src/state.ts @@ -6,7 +6,7 @@ */ import type {Circus, Global} from '@jest/types'; -import {setGlobal, setNotShreddable} from 'jest-util'; +import {protectProperties, setGlobal} from 'jest-util'; import eventHandler from './eventHandler'; import formatNodeAssertErrors from './formatNodeAssertErrors'; import {EVENT_HANDLERS, STATE_SYM} from './types'; @@ -44,7 +44,7 @@ export const getState = (): Circus.State => (globalThis as Global.Global)[STATE_SYM] as Circus.State; export const setState = (state: Circus.State): Circus.State => { setGlobal(globalThis, STATE_SYM, state); - setNotShreddable(state, [ + protectProperties(state, [ 'hasFocusedTests', 'hasStarted', 'includeTestLocationInResult', diff --git a/packages/jest-environment-node/src/index.ts b/packages/jest-environment-node/src/index.ts index a6a0afa6dccb..173a3e744d66 100644 --- a/packages/jest-environment-node/src/index.ts +++ b/packages/jest-environment-node/src/index.ts @@ -15,10 +15,10 @@ import {LegacyFakeTimers, ModernFakeTimers} from '@jest/fake-timers'; import type {Global} from '@jest/types'; import {ModuleMocker} from 'jest-mock'; import { + canDeleteProperties, + deleteProperties, installCommonGlobals, - isShreddable, - setNotShreddable, - shred, + protectProperties, } from 'jest-util'; type Timer = { @@ -235,8 +235,8 @@ export const TestEnvironment = NodeEnvironment; * Creates a new empty global object and wraps it with a {@link Proxy}. * * The purpose is to register any property set on the global object, - * and {@link #shred} them at environment teardown, to clean up memory and - * prevent leaks. + * and {@link #deleteProperties} on them at environment teardown, + * to clean up memory and prevent leaks. */ class GlobalProxy implements ProxyHandler { private global: typeof globalThis = Object.create( @@ -257,16 +257,16 @@ class GlobalProxy implements ProxyHandler { /** * Marks that the environment setup has completed, and properties set on - * the global object from now on should be shredded at teardown. + * the global object from now on should be deleted at teardown. */ envSetupCompleted(): void { this.isEnvSetup = true; } /** - * Shreds any property that was set on the global object, except for: + * Deletes any property that was set on the global object, except for: * 1. Properties that were set before {@link #envSetupCompleted} was invoked. - * 2. Properties protected by {@link #setNotShreddable}. + * 2. Properties protected by {@link #protectProperties}. */ clear(): void { for (const {property, value} of [ @@ -278,10 +278,10 @@ class GlobalProxy implements ProxyHandler { ]) { /* * react-native invoke its custom `performance` property after env teardown. - * its setup file should use `setNotShreddable` to prevent this. + * its setup file should use `protectProperties` to prevent this. */ if (property !== 'performance') { - shred(value); + deleteProperties(value); } } this.propertyToValue.clear(); @@ -332,8 +332,8 @@ class GlobalProxy implements ProxyHandler { private register(property: string | symbol, value: unknown) { const currentValue = this.propertyToValue.get(property); if (value !== currentValue) { - if (!this.isEnvSetup && isShreddable(value)) { - setNotShreddable(value); + if (!this.isEnvSetup && canDeleteProperties(value)) { + protectProperties(value); } if (currentValue) { this.leftovers.push({property, value: currentValue}); diff --git a/packages/jest-repl/src/cli/runtime-cli.ts b/packages/jest-repl/src/cli/runtime-cli.ts index 394528f1963e..9b52e1ae939f 100644 --- a/packages/jest-repl/src/cli/runtime-cli.ts +++ b/packages/jest-repl/src/cli/runtime-cli.ts @@ -99,9 +99,9 @@ export async function run( }, {console: customConsole, docblockPragmas: {}, testPath: filePath}, ); - setGlobal(environment.global, 'console', customConsole, false); - setGlobal(environment.global, 'jestProjectConfig', projectConfig, false); - setGlobal(environment.global, 'jestGlobalConfig', globalConfig, false); + setGlobal(environment.global, 'console', customConsole, 'retain'); + setGlobal(environment.global, 'jestProjectConfig', projectConfig, 'retain'); + setGlobal(environment.global, 'jestGlobalConfig', globalConfig, 'retain'); const runtime = new Runtime( projectConfig, diff --git a/packages/jest-runner/src/runTest.ts b/packages/jest-runner/src/runTest.ts index 9ebade6be07d..4bdea3cc34b6 100644 --- a/packages/jest-runner/src/runTest.ts +++ b/packages/jest-runner/src/runTest.ts @@ -184,7 +184,7 @@ async function runTestInternal( ? new LeakDetector(environment) : null; - setGlobal(environment.global, 'console', testConsole, false); + setGlobal(environment.global, 'console', testConsole, 'retain'); const runtime = new Runtime( projectConfig, diff --git a/packages/jest-util/src/shredder.ts b/packages/jest-util/src/garbage-collection-utils.ts similarity index 59% rename from packages/jest-util/src/shredder.ts rename to packages/jest-util/src/garbage-collection-utils.ts index 209c6c4a71c7..eb109a52447c 100644 --- a/packages/jest-util/src/shredder.ts +++ b/packages/jest-util/src/garbage-collection-utils.ts @@ -5,17 +5,17 @@ * LICENSE file in the root directory of this source tree. */ -const NO_SHRED_AFTER_TEARDOWN = Symbol.for('$$jest-no-shred'); +const PROTECT_PROPERTY = Symbol.for('$$jest-protect-from-deletion'); /** * Deletes all the properties from the given value (if it's an object), - * unless the value was protected via {@link #setNotShreddable}. + * unless the value was protected via {@link #protectProperties}. * * @param value the given value. */ -export function shred(value: unknown): void { - if (isShreddable(value)) { - const protectedProperties = Reflect.get(value, NO_SHRED_AFTER_TEARDOWN); +export function deleteProperties(value: unknown): void { + if (canDeleteProperties(value)) { + const protectedProperties = Reflect.get(value, PROTECT_PROPERTY); if (!Array.isArray(protectedProperties) || protectedProperties.length > 0) { for (const key of Reflect.ownKeys(value)) { if (!protectedProperties?.includes(key)) { @@ -27,28 +27,28 @@ export function shred(value: unknown): void { } /** - * Protects the given value from being shredded by {@link #shred}. + * Protects the given value from being deleted by {@link #deleteProperties}. * * @param value The given value. * @param properties If the array contains any property, * then only these properties will not be deleted; otherwise if the array is empty, * all properties will not be deleted. */ -export function setNotShreddable( +export function protectProperties( value: T, properties: Array = [], ): boolean { - if (isShreddable(value)) { - return Reflect.set(value, NO_SHRED_AFTER_TEARDOWN, properties); + if (canDeleteProperties(value)) { + return Reflect.set(value, PROTECT_PROPERTY, properties); } return false; } /** - * Whether the given value is possible to be shredded. + * Whether the given value has properties that can be deleted (regardless of protection). * * @param value The given value. */ -export function isShreddable(value: unknown): value is object { +export function canDeleteProperties(value: unknown): value is object { return value !== null && ['object', 'function'].includes(typeof value); } diff --git a/packages/jest-util/src/index.ts b/packages/jest-util/src/index.ts index e82663821315..c40b727cc113 100644 --- a/packages/jest-util/src/index.ts +++ b/packages/jest-util/src/index.ts @@ -29,4 +29,8 @@ export {default as tryRealpath} from './tryRealpath'; export {default as requireOrImportModule} from './requireOrImportModule'; export {default as invariant} from './invariant'; export {default as isNonNullable} from './isNonNullable'; -export {isShreddable, setNotShreddable, shred} from './shredder'; +export { + canDeleteProperties, + protectProperties, + deleteProperties, +} from './garbage-collection-utils'; diff --git a/packages/jest-util/src/setGlobal.ts b/packages/jest-util/src/setGlobal.ts index ece978243302..a7a65be3e18c 100644 --- a/packages/jest-util/src/setGlobal.ts +++ b/packages/jest-util/src/setGlobal.ts @@ -6,16 +6,19 @@ */ import type {Global} from '@jest/types'; -import {isShreddable, setNotShreddable} from './shredder'; +import { + canDeleteProperties, + protectProperties, +} from './garbage-collection-utils'; export default function setGlobal( globalToMutate: typeof globalThis | Global.Global, key: string | symbol, value: unknown, - shredAfterTeardown = true, + afterTeardown: 'clean' | 'retain' = 'clean', ): void { Reflect.set(globalToMutate, key, value); - if (!shredAfterTeardown && isShreddable(value)) { - setNotShreddable(value); + if (afterTeardown === 'retain' && canDeleteProperties(value)) { + protectProperties(value); } } From 53ea04f92ffdcb77335cb56e9d4fd0d729258681 Mon Sep 17 00:00:00 2001 From: Eyal Roth Date: Sat, 24 May 2025 18:15:14 +0300 Subject: [PATCH 3/3] #15215 Fix handling of globally shared circus event handlers --- packages/jest-circus/src/state.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/jest-circus/src/state.ts b/packages/jest-circus/src/state.ts index 45a8192627c9..839c4b510121 100644 --- a/packages/jest-circus/src/state.ts +++ b/packages/jest-circus/src/state.ts @@ -14,9 +14,8 @@ import {makeDescribe} from './utils'; const handlers: Array = ((globalThis as Global.Global)[ EVENT_HANDLERS -] = ((globalThis as Global.Global)[ - EVENT_HANDLERS -] as Array) || [eventHandler, formatNodeAssertErrors]); +] as Array) || [eventHandler, formatNodeAssertErrors]; +setGlobal(globalThis, EVENT_HANDLERS, handlers, 'retain'); export const ROOT_DESCRIBE_BLOCK_NAME = 'ROOT_DESCRIBE_BLOCK';