Skip to content

Commit 48da335

Browse files
paul-tavarespaul-tavares
authored
[9.1] [Security Solution][Endpoint] Fix artifacts spaces migration (v9.1) to ensure all artifacts are processed (#238740) (#239009)
# Backport This will backport the following commits from `main` to `9.1`: - [[Security Solution][Endpoint] Fix artifacts spaces migration (`v9.1`) to ensure all artifacts are processed (#238740)](#238740) <!--- Backport version: 10.0.2 --> ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport) <!--BACKPORT [{"author":{"name":"Paul Tavares","email":"[email protected]"},"sourceCommit":{"committedDate":"2025-10-14T15:32:05Z","message":"[Security Solution][Endpoint] Fix artifacts spaces migration (`v9.1`) to ensure all artifacts are processed (#238740)\n\n## Summary\n\nFixes #238711\n\n- Fixes Endpoint artifact migration in support of spaces (introduced in\n`9.1.0`) to:\n- Ensure it processes all artifacts. The bug fix was to ensure that a\n`perPage` value is used when creating the SO client Point-in-Time\ninterface. Without it, only the first 20 artifacts were being processed\n- Fix the filter query to ensure that only artifacts missing the\n`spaceOwnerId` tag are matched when retrieving items that need to be\nmigrated\n- Add logic to ensure that this endpoint artifacts migration is re-run\n\n\n\n### Checklist\n\n- [x] [Unit or functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere updated or added to match the most common scenarios","sha":"8a3cc7c31641c31c409ffffbf1cbd0262a20d6f6","branchLabelMapping":{"^v9.3.0$":"main","^v(\\d+).(\\d+).\\d+$":"$1.$2"}},"sourcePullRequest":{"labels":["release_note:fix","Team:Defend Workflows","backport:version","v9.2.0","v9.3.0","v9.1.6"],"title":"[Security Solution][Endpoint] Fix artifacts spaces migration (`v9.1`) to ensure all artifacts are processed","number":238740,"url":"https://github.com/elastic/kibana/pull/238740","mergeCommit":{"message":"[Security Solution][Endpoint] Fix artifacts spaces migration (`v9.1`) to ensure all artifacts are processed (#238740)\n\n## Summary\n\nFixes #238711\n\n- Fixes Endpoint artifact migration in support of spaces (introduced in\n`9.1.0`) to:\n- Ensure it processes all artifacts. The bug fix was to ensure that a\n`perPage` value is used when creating the SO client Point-in-Time\ninterface. Without it, only the first 20 artifacts were being processed\n- Fix the filter query to ensure that only artifacts missing the\n`spaceOwnerId` tag are matched when retrieving items that need to be\nmigrated\n- Add logic to ensure that this endpoint artifacts migration is re-run\n\n\n\n### Checklist\n\n- [x] [Unit or functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere updated or added to match the most common scenarios","sha":"8a3cc7c31641c31c409ffffbf1cbd0262a20d6f6"}},"sourceBranch":"main","suggestedTargetBranches":["9.2","9.1"],"targetPullRequestStates":[{"branch":"9.2","label":"v9.2.0","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"},{"branch":"main","label":"v9.3.0","branchLabelMappingKey":"^v9.3.0$","isSourceBranch":true,"state":"MERGED","url":"https://github.com/elastic/kibana/pull/238740","number":238740,"mergeCommit":{"message":"[Security Solution][Endpoint] Fix artifacts spaces migration (`v9.1`) to ensure all artifacts are processed (#238740)\n\n## Summary\n\nFixes #238711\n\n- Fixes Endpoint artifact migration in support of spaces (introduced in\n`9.1.0`) to:\n- Ensure it processes all artifacts. The bug fix was to ensure that a\n`perPage` value is used when creating the SO client Point-in-Time\ninterface. Without it, only the first 20 artifacts were being processed\n- Fix the filter query to ensure that only artifacts missing the\n`spaceOwnerId` tag are matched when retrieving items that need to be\nmigrated\n- Add logic to ensure that this endpoint artifacts migration is re-run\n\n\n\n### Checklist\n\n- [x] [Unit or functional\ntests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)\nwere updated or added to match the most common scenarios","sha":"8a3cc7c31641c31c409ffffbf1cbd0262a20d6f6"}},{"branch":"9.1","label":"v9.1.6","branchLabelMappingKey":"^v(\\d+).(\\d+).\\d+$","isSourceBranch":false,"state":"NOT_CREATED"}]}] BACKPORT-->
1 parent 6ea6c35 commit 48da335

File tree

3 files changed

+75
-13
lines changed

3 files changed

+75
-13
lines changed

‎x-pack/solutions/security/plugins/security_solution/server/endpoint/lib/reference_data/types.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,14 @@ export interface MigrationMetadata {
4141
started: string;
4242
finished: string;
4343
status: 'not-started' | 'complete' | 'pending';
44+
/**
45+
* Any data that needs to be persisted with the migration
46+
*/
4447
data?: unknown;
48+
/**
49+
* Version may be helpful in cases where the migration code might have had issues and we need to re-run it
50+
*/
51+
version?: number;
4552
}
4653

4754
export interface OrphanResponseActionsMetadata {

‎x-pack/solutions/security/plugins/security_solution/server/endpoint/migrations/space_awareness_migration.test.ts‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ describe('Space awareness migration', () => {
5555
started: '',
5656
finished: '',
5757
status: 'not-started',
58+
version: 2,
5859
},
5960
},
6061
[RESPONSE_ACTIONS_MIGRATION_REF_DATA_ID]: {
@@ -154,11 +155,11 @@ describe('Space awareness migration', () => {
154155
],
155156
namespaceType: ['agnostic', 'agnostic', 'agnostic', 'agnostic', 'agnostic'],
156157
filter: [
157-
`NOT exception-list-agnostic.attributes.tags:"${buildSpaceOwnerIdTag('*')}"`,
158-
`NOT exception-list-agnostic.attributes.tags:"${buildSpaceOwnerIdTag('*')}"`,
159-
`NOT exception-list-agnostic.attributes.tags:"${buildSpaceOwnerIdTag('*')}"`,
160-
`NOT exception-list-agnostic.attributes.tags:"${buildSpaceOwnerIdTag('*')}"`,
161-
`NOT exception-list-agnostic.attributes.tags:"${buildSpaceOwnerIdTag('*')}"`,
158+
`NOT exception-list-agnostic.attributes.tags:(ownerSpaceId*)`,
159+
`NOT exception-list-agnostic.attributes.tags:(ownerSpaceId*)`,
160+
`NOT exception-list-agnostic.attributes.tags:(ownerSpaceId*)`,
161+
`NOT exception-list-agnostic.attributes.tags:(ownerSpaceId*)`,
162+
`NOT exception-list-agnostic.attributes.tags:(ownerSpaceId*)`,
162163
],
163164
})
164165
);
@@ -207,6 +208,21 @@ describe('Space awareness migration', () => {
207208
expect.objectContaining({ metadata: expect.objectContaining({ status: 'complete' }) })
208209
);
209210
});
211+
212+
it('should re-run migration if version is less than 2', async () => {
213+
artifactMigrationState.metadata.status = 'complete';
214+
artifactMigrationState.metadata.version = 1;
215+
216+
await expect(
217+
migrateEndpointDataToSupportSpaces(endpointServiceMock)
218+
).resolves.toBeUndefined();
219+
expect(endpointServiceMock.getReferenceDataClient().update).toHaveBeenCalledWith(
220+
ARTIFACTS_MIGRATION_REF_DATA_ID,
221+
expect.objectContaining({
222+
metadata: expect.objectContaining({ status: 'complete', version: 2 }),
223+
})
224+
);
225+
});
210226
});
211227

212228
describe('for Response Actions', () => {
@@ -280,7 +296,7 @@ describe('Space awareness migration', () => {
280296
await expect(
281297
migrateEndpointDataToSupportSpaces(endpointServiceMock)
282298
).resolves.toBeUndefined();
283-
expect(endpointServiceMock.getExceptionListsClient).not.toHaveBeenCalled();
299+
expect(endpointServiceMock.getInternalEsClient().indices.exists).not.toHaveBeenCalled();
284300
}
285301
);
286302

‎x-pack/solutions/security/plugins/security_solution/server/endpoint/migrations/space_awareness_migration.ts‎

Lines changed: 46 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import type {
1818
SearchTotalHits,
1919
} from '@elastic/elasticsearch/lib/api/types';
2020
import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common';
21+
import { clone, pick } from 'lodash';
2122
import { ALLOWED_ACTION_REQUEST_TAGS } from '../services/actions/constants';
2223
import { GLOBAL_ARTIFACT_TAG } from '../../../common/endpoint/service/artifacts';
2324
import { ensureActionRequestsIndexIsConfigured } from '../services';
@@ -97,17 +98,34 @@ const migrateArtifactsToSpaceAware = async (
9798
const refDataClient = endpointService.getReferenceDataClient();
9899
const migrationState = await getMigrationState(refDataClient, ARTIFACTS_MIGRATION_REF_DATA_ID);
99100

100-
if (migrationState.metadata.status !== 'not-started') {
101+
// If migration has already been run and the version was 2 or higher -or- the status is pending (already running),
102+
// then we don't need to run it again
103+
if (
104+
(migrationState.metadata.status === 'complete' &&
105+
(migrationState.metadata.version ?? 0) >= 2) ||
106+
migrationState.metadata.status === 'pending'
107+
) {
101108
logger.debug(
102-
`Migration for endpoint artifacts in support of spaces has a status of [${migrationState.metadata.status}]. Nothing to do.`
109+
`Migration (v2) for endpoint artifacts in support of spaces has a status of [${migrationState.metadata.status}], version [${migrationState.metadata.version}]. Nothing to do.`
103110
);
104111
return;
105112
}
106113

107-
logger.info(`starting migration of endpoint artifacts in support of spaces`);
114+
logger.debug(`starting migration (v2) of endpoint artifacts in support of spaces`);
115+
116+
// If there are statuses about a prior run, then store a clone of it in the migration object for reference
117+
const priorRunData =
118+
migrationState.metadata.status === 'complete' ? clone(migrationState.metadata) : undefined;
108119

120+
// Migration version history:
121+
// V1: original migration of artifacts with release 9.1.0
122+
// v2: fixes for migration issue: https://github.com/elastic/kibana/issues/238711
123+
migrationState.metadata.version = 2;
109124
migrationState.metadata.status = 'pending';
110125
migrationState.metadata.started = new Date().toISOString();
126+
migrationState.metadata.finished = '';
127+
migrationState.metadata.data = undefined;
128+
111129
await updateMigrationState(refDataClient, ARTIFACTS_MIGRATION_REF_DATA_ID, migrationState);
112130

113131
const exceptionsClient = endpointService.getExceptionListsClient();
@@ -130,15 +148,36 @@ const migrateArtifactsToSpaceAware = async (
130148

131149
return acc;
132150
}, {} as Record<string, { success: number; failed: number; errors: string[] }>),
151+
priorRuns: [] as Array<typeof migrationState.metadata>,
133152
};
134153

135154
migrationState.metadata.data = migrationStats;
136155

156+
if (priorRunData) {
157+
migrationStats.priorRuns.push(
158+
...((priorRunData.data as typeof migrationStats)?.priorRuns ?? [])
159+
);
160+
161+
migrationStats.priorRuns.push({
162+
...(pick(priorRunData, [
163+
'status',
164+
'started',
165+
'finished',
166+
'version',
167+
]) as typeof migrationState.metadata),
168+
data: pick(priorRunData.data as typeof migrationStats, [
169+
'totalItems',
170+
'itemsNeedingUpdates',
171+
'successUpdates',
172+
'failedUpdates',
173+
'artifacts',
174+
]),
175+
});
176+
}
177+
137178
const updateProcessor = new QueueProcessor<UpdateExceptionListItemOptions & { listId: string }>({
138179
batchSize: 50,
139180
batchHandler: async ({ data: artifactUpdates }) => {
140-
// TODO:PT add a `bulkUpdate()` to the exceptionsListClient
141-
142181
migrationStats.itemsNeedingUpdates += artifactUpdates.length;
143182

144183
await pMap(
@@ -172,9 +211,9 @@ const migrateArtifactsToSpaceAware = async (
172211
namespaceType: listIds.map(() => 'agnostic'),
173212
filter: listIds.map(
174213
// Find all artifacts that do NOT have a space owner id tag
175-
() => `NOT exception-list-agnostic.attributes.tags:"${buildSpaceOwnerIdTag('*')}"`
214+
() => `NOT exception-list-agnostic.attributes.tags:(ownerSpaceId*)`
176215
),
177-
perPage: undefined,
216+
perPage: 1_000,
178217
sortField: undefined,
179218
sortOrder: undefined,
180219
maxSize: undefined,

0 commit comments

Comments
 (0)