[9.1] [Security Solution][Endpoint] Fix artifacts spaces migration (v9.1) to ensure all artifacts are processed (#238740)

x-pack/solutions/security/plugins/security_solution/server/endpoint/lib/reference_data/types.ts

@@ -41,7 +41,14 @@ export interface MigrationMetadata {
   started: string;
   finished: string;
   status: 'not-started' | 'complete' | 'pending';
+  /**
+   * Any data that needs to be persisted with the migration
+   */
   data?: unknown;
+  /**
+   * Version may be helpful in cases where the migration code might have had issues and we need to re-run it
+   */
+  version?: number;
 }
 
 export interface OrphanResponseActionsMetadata {

x-pack/solutions/security/plugins/security_solution/server/endpoint/migrations/space_awareness_migration.test.ts

@@ -55,6 +55,7 @@ describe('Space awareness migration', () => {
           started: '',
           finished: '',
           status: 'not-started',
+          version: 2,
         },
       },
       [RESPONSE_ACTIONS_MIGRATION_REF_DATA_ID]: {
@@ -154,11 +155,11 @@ describe('Space awareness migration', () => {
           ],
           namespaceType: ['agnostic', 'agnostic', 'agnostic', 'agnostic', 'agnostic'],
           filter: [
-            `NOT exception-list-agnostic.attributes.tags:"${buildSpaceOwnerIdTag('*')}"`,
-            `NOT exception-list-agnostic.attributes.tags:"${buildSpaceOwnerIdTag('*')}"`,
-            `NOT exception-list-agnostic.attributes.tags:"${buildSpaceOwnerIdTag('*')}"`,
-            `NOT exception-list-agnostic.attributes.tags:"${buildSpaceOwnerIdTag('*')}"`,
-            `NOT exception-list-agnostic.attributes.tags:"${buildSpaceOwnerIdTag('*')}"`,
+            `NOT exception-list-agnostic.attributes.tags:(ownerSpaceId*)`,
+            `NOT exception-list-agnostic.attributes.tags:(ownerSpaceId*)`,
+            `NOT exception-list-agnostic.attributes.tags:(ownerSpaceId*)`,
+            `NOT exception-list-agnostic.attributes.tags:(ownerSpaceId*)`,
+            `NOT exception-list-agnostic.attributes.tags:(ownerSpaceId*)`,
           ],
         })
       );
@@ -207,6 +208,21 @@ describe('Space awareness migration', () => {
         expect.objectContaining({ metadata: expect.objectContaining({ status: 'complete' }) })
       );
     });
+
+    it('should re-run migration if version is less than 2', async () => {
+      artifactMigrationState.metadata.status = 'complete';
+      artifactMigrationState.metadata.version = 1;
+
+      await expect(
+        migrateEndpointDataToSupportSpaces(endpointServiceMock)
+      ).resolves.toBeUndefined();
+      expect(endpointServiceMock.getReferenceDataClient().update).toHaveBeenCalledWith(
+        ARTIFACTS_MIGRATION_REF_DATA_ID,
+        expect.objectContaining({
+          metadata: expect.objectContaining({ status: 'complete', version: 2 }),
+        })
+      );
+    });
   });
 
   describe('for Response Actions', () => {
@@ -280,7 +296,7 @@ describe('Space awareness migration', () => {
         await expect(
           migrateEndpointDataToSupportSpaces(endpointServiceMock)
         ).resolves.toBeUndefined();
-        expect(endpointServiceMock.getExceptionListsClient).not.toHaveBeenCalled();
+        expect(endpointServiceMock.getInternalEsClient().indices.exists).not.toHaveBeenCalled();
       }
     );
 

x-pack/solutions/security/plugins/security_solution/server/endpoint/migrations/space_awareness_migration.ts

@@ -18,6 +18,7 @@ import type {
   SearchTotalHits,
 } from '@elastic/elasticsearch/lib/api/types';
 import { PACKAGE_POLICY_SAVED_OBJECT_TYPE } from '@kbn/fleet-plugin/common';
+import { clone, pick } from 'lodash';
 import { ALLOWED_ACTION_REQUEST_TAGS } from '../services/actions/constants';
 import { GLOBAL_ARTIFACT_TAG } from '../../../common/endpoint/service/artifacts';
 import { ensureActionRequestsIndexIsConfigured } from '../services';
@@ -97,17 +98,34 @@ const migrateArtifactsToSpaceAware = async (
   const refDataClient = endpointService.getReferenceDataClient();
   const migrationState = await getMigrationState(refDataClient, ARTIFACTS_MIGRATION_REF_DATA_ID);
 
-  if (migrationState.metadata.status !== 'not-started') {
+  // If migration has already been run and the version was 2 or higher -or- the status is pending (already running),
+  // then we don't need to run it again
+  if (
+    (migrationState.metadata.status === 'complete' &&
+      (migrationState.metadata.version ?? 0) >= 2) ||
+    migrationState.metadata.status === 'pending'
+  ) {
     logger.debug(
-      `Migration for endpoint artifacts in support of spaces has a status of [${migrationState.metadata.status}]. Nothing to do.`
+      `Migration (v2) for endpoint artifacts in support of spaces has a status of [${migrationState.metadata.status}], version [${migrationState.metadata.version}]. Nothing to do.`
     );
     return;
   }
 
-  logger.info(`starting migration of endpoint artifacts in support of spaces`);
+  logger.debug(`starting migration (v2) of endpoint artifacts in support of spaces`);
+
+  // If there are statuses about a prior run, then store a clone of it in the migration object for reference
+  const priorRunData =
+    migrationState.metadata.status === 'complete' ? clone(migrationState.metadata) : undefined;
 
+  // Migration version history:
+  // V1: original migration of artifacts with release 9.1.0
+  // v2: fixes for migration issue: https://github.com/elastic/kibana/issues/238711
+  migrationState.metadata.version = 2;
   migrationState.metadata.status = 'pending';
   migrationState.metadata.started = new Date().toISOString();
+  migrationState.metadata.finished = '';
+  migrationState.metadata.data = undefined;
+
   await updateMigrationState(refDataClient, ARTIFACTS_MIGRATION_REF_DATA_ID, migrationState);
 
   const exceptionsClient = endpointService.getExceptionListsClient();
@@ -130,15 +148,36 @@ const migrateArtifactsToSpaceAware = async (
 
       return acc;
     }, {} as Record<string, { success: number; failed: number; errors: string[] }>),
+    priorRuns: [] as Array<typeof migrationState.metadata>,
   };
 
   migrationState.metadata.data = migrationStats;
 
+  if (priorRunData) {
+    migrationStats.priorRuns.push(
+      ...((priorRunData.data as typeof migrationStats)?.priorRuns ?? [])
+    );
+
+    migrationStats.priorRuns.push({
+      ...(pick(priorRunData, [
+        'status',
+        'started',
+        'finished',
+        'version',
+      ]) as typeof migrationState.metadata),
+      data: pick(priorRunData.data as typeof migrationStats, [
+        'totalItems',
+        'itemsNeedingUpdates',
+        'successUpdates',
+        'failedUpdates',
+        'artifacts',
+      ]),
+    });
+  }
+
   const updateProcessor = new QueueProcessor<UpdateExceptionListItemOptions & { listId: string }>({
     batchSize: 50,
     batchHandler: async ({ data: artifactUpdates }) => {
-      // TODO:PT add a `bulkUpdate()` to the exceptionsListClient
-
       migrationStats.itemsNeedingUpdates += artifactUpdates.length;
 
       await pMap(
@@ -172,9 +211,9 @@ const migrateArtifactsToSpaceAware = async (
       namespaceType: listIds.map(() => 'agnostic'),
       filter: listIds.map(
         // Find all artifacts that do NOT have a space owner id tag
-        () => `NOT exception-list-agnostic.attributes.tags:"${buildSpaceOwnerIdTag('*')}"`
+        () => `NOT exception-list-agnostic.attributes.tags:(ownerSpaceId*)`
       ),
-      perPage: undefined,
+      perPage: 1_000,
       sortField: undefined,
       sortOrder: undefined,
       maxSize: undefined,