[Fleet] Add graceful error handling for invalid sortField on GET api/fleet/agents (elastic#239017)

## Summary

Closes elastic/ingest-dev#5992

- Added error handling for all fleet endpoints that use ES. ES errors
will now bubble up and be properly returned along with their status code
response from ES. This will stop errors from being `500` when they are
actually something else, we just werent properly handling them at the
top level.
- Also added additional test coverage to verify the handler will
throw/succeed when its supposed to, and added some simple integration
tests to verify the entire flow works as intended.

<img width="811" height="236" alt="image"
src="https://github.com/user-attachments/assets/dbc3b831-6984-427a-9762-399ad3b8e1b9"
/>



### Checklist

Check the PR satisfies following conditions. 

Reviewers should verify this PR satisfies this list as well.

- [ ] Any text added follows [EUI's writing
guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses
sentence case text and includes [i18n
support](https://github.com/elastic/kibana/blob/main/src/platform/packages/shared/kbn-i18n/README.md)
- [ ]
[Documentation](https://www.elastic.co/guide/en/kibana/master/development-documentation.html)
was added for features that require explanation or tutorials
- [ ] [Unit or functional
tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html)
were updated or added to match the most common scenarios
- [ ] If a plugin configuration key changed, check if it needs to be
allowlisted in the cloud and added to the [docker
list](https://github.com/elastic/kibana/blob/main/src/dev/build/tasks/os_packages/docker_generator/resources/base/bin/kibana-docker)
- [ ] This was checked for breaking HTTP API changes, and any breaking
changes have been approved by the breaking-change committee. The
`release_note:breaking` label should be applied in these situations.
- [ ] [Flaky Test
Runner](https://ci-stats.kibana.dev/trigger_flaky_test_runner/1) was
used on any tests changed
- [ ] The PR description includes the appropriate Release Notes section,
and the correct `release_note:*` label is applied per the
[guidelines](https://www.elastic.co/guide/en/kibana/master/contributing.html#kibana-release-notes-process)
- [ ] Review the [backport
guidelines](https://docs.google.com/document/d/1VyN5k91e5OVumlc0Gb9RPa3h1ewuPE705nRtioPiTvY/edit?usp=sharing)
and apply applicable `backport:*` labels.

### Identify risks

Does this PR introduce any risks? For example, consider risks like hard
to test bugs, performance regression, potential of data loss.

Describe the risk, its severity, and mitigation for each identified
risk. Invite stakeholders and evaluate how to proceed before merging.

- [ ] [See some risk
examples](https://github.com/elastic/kibana/blob/main/RISK_MATRIX.mdx)
- [ ] ...

---------

Co-authored-by: Elastic Machine <[email protected]>
Co-authored-by: kibanamachine <[email protected]>

Author: 3 people

x-pack/platform/plugins/shared/fleet/server/errors/handlers.ts

@@ -26,6 +26,8 @@ import {
   AgentPolicyNameExistsError,
   ConcurrentInstallOperationError,
   FleetError,
+  FleetElasticsearchValidationError,
+  isESClientError,
   PackageUnsupportedMediaTypeError,
   RegistryConnectionError,
   RegistryError,
@@ -231,6 +233,12 @@ export const defaultFleetErrorHandler: IngestErrorHandler = async ({
   error,
   response,
 }: IngestErrorHandlerParams): Promise<IKibanaResponse> => {
-  const options = fleetErrorToResponseOptions(error);
+  // Convert ALL Elasticsearch errors to Fleet errors (preserving original status codes)
+  let processedError = error;
+  if (isESClientError(error)) {
+    processedError = new FleetElasticsearchValidationError(error);
+  }
+
+  const options = fleetErrorToResponseOptions(processedError);
   return response.customError(options);
 };

x-pack/platform/plugins/shared/fleet/server/errors/index.ts

@@ -241,5 +241,20 @@ export class ArtifactsElasticsearchError extends FleetError {
   }
 }
 
+export class FleetElasticsearchValidationError extends FleetErrorWithStatusCode {
+  constructor(esError: Error) {
+    let statusCode: number | undefined;
+    const message = esError.message;
+
+    // Extract the original ES status code and ensure we have meta with statusCode
+    if (isESClientError(esError)) {
+      statusCode = esError.meta.statusCode;
+    }
+
+    // Pass through the ES error message, status code, and original error as meta
+    super(message, statusCode, esError);
+  }
+}
+
 export class FleetFilesClientError extends FleetError {}
 export class FleetFileNotFound extends FleetFilesClientError {}

x-pack/platform/plugins/shared/fleet/server/routes/agent/handlers.test.ts

@@ -6,10 +6,16 @@
  */
 
 import { coreMock, httpServerMock } from '@kbn/core/server/mocks';
+import { errors } from '@elastic/elasticsearch';
 
 import { getAgentStatusForAgentPolicy } from '../../services/agents/status';
+import { fetchAndAssignAgentMetrics } from '../../services/agents/agent_metrics';
 
-import { getAgentStatusForAgentPolicyHandler, getAvailableVersionsHandler } from './handlers';
+import {
+  getAgentStatusForAgentPolicyHandler,
+  getAvailableVersionsHandler,
+  getAgentsHandler,
+} from './handlers';
 
 jest.mock('../../services/agents/versions', () => {
   return {
@@ -30,7 +36,270 @@ jest.mock('../../services/agents/status', () => ({
   getAgentStatusForAgentPolicy: jest.fn(),
 }));
 
+jest.mock('../../services/agents/agent_metrics', () => ({
+  fetchAndAssignAgentMetrics: jest.fn(),
+}));
+
 describe('Handlers', () => {
+  // Helper function to create mock Elasticsearch errors
+  const createMockESError = (errorBody: any, statusCode: number = 400) => {
+    const error = new Error('ResponseError') as any;
+    error.meta = {
+      body: errorBody,
+      statusCode,
+    };
+    Object.setPrototypeOf(error, errors.ResponseError.prototype);
+    return error;
+  };
+
+  describe('getAgentsHandler', () => {
+    let mockAgentClient: any;
+    let mockContext: any;
+    let mockResponse: any;
+
+    beforeEach(() => {
+      mockAgentClient = {
+        asCurrentUser: {
+          listAgents: jest.fn(),
+        },
+      };
+
+      mockContext = {
+        core: Promise.resolve(coreMock.createRequestHandlerContext()),
+        fleet: Promise.resolve({
+          agentClient: mockAgentClient,
+        }),
+      };
+
+      mockResponse = httpServerMock.createResponseFactory();
+      (fetchAndAssignAgentMetrics as jest.Mock).mockClear();
+    });
+
+    it('should handle successful agent listing', async () => {
+      const mockAgents = [
+        { id: 'agent1', enrolled_at: '2023-01-01' },
+        { id: 'agent2', enrolled_at: '2023-01-02' },
+      ];
+
+      mockAgentClient.asCurrentUser.listAgents.mockResolvedValue({
+        agents: mockAgents,
+        total: 2,
+        page: 1,
+        perPage: 20,
+      });
+
+      const request = {
+        query: {
+          page: 1,
+          perPage: 20,
+          sortField: 'enrolled_at',
+          sortOrder: 'desc',
+        },
+      };
+
+      await getAgentsHandler(mockContext, request as any, mockResponse);
+
+      expect(mockResponse.ok).toHaveBeenCalledWith({
+        body: {
+          items: mockAgents,
+          total: 2,
+          page: 1,
+          perPage: 20,
+        },
+      });
+    });
+
+    it('should let ES parsing errors bubble up to global error handler', async () => {
+      const elasticsearchError = createMockESError({
+        error: {
+          type: 'parsing_exception',
+          reason: 'Invalid query syntax',
+        },
+      });
+
+      mockAgentClient.asCurrentUser.listAgents.mockRejectedValue(elasticsearchError);
+
+      const request = {
+        query: {
+          sortField: 'invalid_field',
+        },
+      };
+
+      await expect(getAgentsHandler(mockContext, request as any, mockResponse)).rejects.toEqual(
+        elasticsearchError
+      );
+    });
+
+    it('should let ES argument errors bubble up to global error handler', async () => {
+      const elasticsearchError = createMockESError({
+        error: {
+          type: 'illegal_argument_exception',
+          reason: 'No mapping found for [non_existent_field] in order to sort on',
+        },
+      });
+
+      mockAgentClient.asCurrentUser.listAgents.mockRejectedValue(elasticsearchError);
+
+      const request = {
+        query: {
+          sortField: 'non_existent_field',
+        },
+      };
+
+      await expect(getAgentsHandler(mockContext, request as any, mockResponse)).rejects.toEqual(
+        elasticsearchError
+      );
+    });
+
+    it('should let search_phase_execution_exception errors bubble up to global error handler', async () => {
+      const elasticsearchError = createMockESError({
+        error: {
+          type: 'search_phase_execution_exception',
+          reason: 'Unknown field [bad_field]',
+        },
+      });
+
+      mockAgentClient.asCurrentUser.listAgents.mockRejectedValue(elasticsearchError);
+
+      const request = {
+        query: {
+          sortField: 'bad_field',
+        },
+      };
+
+      await expect(getAgentsHandler(mockContext, request as any, mockResponse)).rejects.toEqual(
+        elasticsearchError
+      );
+    });
+
+    it('should let field mapping errors bubble up to global error handler', async () => {
+      const elasticsearchError = createMockESError({
+        error: {
+          type: 'some_error_type',
+          reason: 'No mapping found for field [invalid_field]',
+        },
+      });
+
+      mockAgentClient.asCurrentUser.listAgents.mockRejectedValue(elasticsearchError);
+
+      const request = {
+        query: {
+          sortField: 'invalid_field',
+        },
+      };
+
+      await expect(getAgentsHandler(mockContext, request as any, mockResponse)).rejects.toEqual(
+        elasticsearchError
+      );
+    });
+
+    it('should let unknown field errors bubble up to global error handler', async () => {
+      const elasticsearchError = createMockESError({
+        error: {
+          type: 'some_error_type',
+          reason: 'Unknown field [mystery_field] in sort criteria',
+        },
+      });
+
+      mockAgentClient.asCurrentUser.listAgents.mockRejectedValue(elasticsearchError);
+
+      const request = {
+        query: {
+          sortField: 'mystery_field',
+        },
+      };
+
+      await expect(getAgentsHandler(mockContext, request as any, mockResponse)).rejects.toEqual(
+        elasticsearchError
+      );
+    });
+
+    it('should re-throw non-validation errors', async () => {
+      const systemError = new Error('System error');
+
+      mockAgentClient.asCurrentUser.listAgents.mockRejectedValue(systemError);
+
+      const request = {
+        query: {
+          sortField: 'enrolled_at',
+        },
+      };
+
+      await expect(getAgentsHandler(mockContext, request as any, mockResponse)).rejects.toThrow(
+        'System error'
+      );
+    });
+
+    it('should re-throw elasticsearch errors that are not validation errors', async () => {
+      const elasticsearchError = createMockESError({
+        error: {
+          type: 'cluster_block_exception',
+          reason: 'Cluster is read-only',
+        },
+      });
+
+      mockAgentClient.asCurrentUser.listAgents.mockRejectedValue(elasticsearchError);
+
+      const request = {
+        query: {
+          sortField: 'enrolled_at',
+        },
+      };
+
+      await expect(getAgentsHandler(mockContext, request as any, mockResponse)).rejects.toEqual(
+        elasticsearchError
+      );
+    });
+
+    it('should let root_cause errors bubble up to global error handler', async () => {
+      const elasticsearchError = createMockESError({
+        error: {
+          type: 'search_phase_execution_exception',
+          reason: 'all shards failed',
+          root_cause: [
+            {
+              type: 'illegal_argument_exception',
+              reason: 'No mapping found for [hostname] in order to sort on',
+            },
+          ],
+        },
+      });
+
+      mockAgentClient.asCurrentUser.listAgents.mockRejectedValue(elasticsearchError);
+
+      const request = {
+        query: {
+          sortField: 'hostname',
+        },
+      };
+
+      await expect(getAgentsHandler(mockContext, request as any, mockResponse)).rejects.toEqual(
+        elasticsearchError
+      );
+    });
+
+    it('should let errors with missing reasons bubble up to global error handler', async () => {
+      const elasticsearchError = createMockESError({
+        error: {
+          type: 'parsing_exception',
+          // no reason provided
+        },
+      });
+
+      mockAgentClient.asCurrentUser.listAgents.mockRejectedValue(elasticsearchError);
+
+      const request = {
+        query: {
+          sortField: 'invalid_field',
+        },
+      };
+
+      await expect(getAgentsHandler(mockContext, request as any, mockResponse)).rejects.toEqual(
+        elasticsearchError
+      );
+    });
+  });
+
   describe('getAgentStatusForAgentPolicyHandler', () => {
     it.each([
       { requested: 'policy-id-1', called: ['policy-id-1'] },

x-pack/platform/test/fleet_api_integration/apis/agents/list.ts

@@ -101,6 +101,13 @@ export default function ({ getService }: FtrProviderContext) {
       expect(agent.access_api_key_id).to.eql('api-key-2');
     });
 
+    it('should return a 400 when given an invalid "sortField" value', async () => {
+      await supertest
+        .get(`/api/fleet/agents?sortField=invalid_field`)
+        .set('kbn-xsrf', 'xxxx')
+        .expect(400);
+    });
+
     it('should return a 200 when given sort options', async () => {
       const { body: apiResponse } = await supertest
         .get(`/api/fleet/agents?sortField=last_checkin&sortOrder=desc`)