test: Integration test for fiori MCP using promptfoo

.changeset/many-flies-dream.md

@@ -0,0 +1,6 @@
+---
+'@sap-ux/fiori-mcp-server': minor
+---
+
+- First integration tests using promptfoo
+- Updated input schema for 'execute-functionality' - sometimes input parameters was passed outside of `parameters` property

package.json

@@ -75,7 +75,7 @@
     "pnpm": {
         "overrides": {
             "router>path-to-regexp": "0.1.12",
-            "[email protected]>path-to-regexp": "8.2.0",
+            "router@^2.0.0>path-to-regexp": "8.2.0",
             "@storybook/manager-api>store2": "2.14.4",
             "mta-local": "1.0.4",
             "axios@<1.12.0": "^1.12.2"

packages/fiori-mcp-server/.gitignore

@@ -1,2 +1,5 @@
 dist
-data
+data
+reports
+test/test-data/copy
+test/integration/logs

packages/fiori-mcp-server/package.json

@@ -34,7 +34,10 @@
         "lint": "eslint . --ext .ts",
         "lint:fix": "eslint . --ext .ts --fix",
         "test": "jest --ci --forceExit --detectOpenHandles --colors",
-        "inspector": "npx @modelcontextprotocol/inspector node dist/index.js"
+        "inspector": "npx @modelcontextprotocol/inspector node dist/index.js",
+        "view:integration": "promptfoo view -y",
+        "test:integration:once": "promptfoo eval --config test/integration/scenarios/promptfooconfig.yaml --max-concurrency 1 --repeat 1 --output reports/integration.txt",
+        "test:integration:multiple": "npm run test:promptfoo -- --repeat 5"
     },
     "files": [
         "LICENSE",
@@ -58,6 +61,7 @@
         "@sap-ux/odata-annotation-core-types": "workspace:*",
         "@sap-ux/odata-entity-model": "workspace:*",
         "@sap-ux/text-document-utils": "workspace:*",
+        "@types/diff": "5.0.9",
         "@types/json-schema": "7.0.5",
         "@types/mem-fs": "1.1.2",
         "@types/mem-fs-editor": "7.0.1",
@@ -67,7 +71,12 @@
         "@sap-ux/telemetry": "workspace:*",
         "i18next": "25.3.0",
         "os-name": "4.0.1",
-        "zod": "4.1.5"
+        "zod": "4.1.5",
+        "@sap-ai-sdk/foundation-models": "2.0.0",
+        "@sap-ai-sdk/langchain": "2.0.0",
+        "promptfoo": "0.118.6",
+        "@langchain/mcp-adapters": "0.6.0",
+        "@langchain/core": "0.3.75"
     },
     "dependencies": {
         "@sap-ux/fiori-docs-embeddings": "*",

packages/fiori-mcp-server/src/types/input.ts

@@ -38,14 +38,21 @@ export const GetFunctionalityDetailsInputSchema = zod.object({
 /**
  * Input interface for the 'execute_functionality' functionality
  */
-export const ExecuteFunctionalityInputSchema = zod.object({
-    /** ID or array of IDs of the functionality(ies) to execute */
-    functionalityId: FunctionalityIdSchema.describe('The ID of the functionality to execute'),
-    /** Parameters for the functionality execution */
-    parameters: zod.record(zod.string(), zod.unknown()).describe('Parameters for the functionality execution'),
-    /** Path to the Fiori application */
-    appPath: zod.string().describe('Path to the Fiori application. Path should be an absolute path.')
-});
+export const ExecuteFunctionalityInputSchema = zod
+    .object({
+        /** ID or array of IDs of the functionality(ies) to execute */
+        functionalityId: FunctionalityIdSchema.describe('The ID of the functionality to execute'),
+        /** Parameters for the functionality execution */
+        parameters: zod.record(zod.string(), zod.unknown()).describe('Parameters for the functionality execution'),
+        /** Path to the Fiori application */
+        appPath: zod.string().describe('Path to the Fiori application. Path should be an absolute path.')
+    })
+    .describe(
+        'Input object for executing a functionality. ' +
+            'Only three top-level properties are allowed: "functionalityId", "parameters", and "appPath". ' +
+            'All other dynamic or functionality-specific inputs must be included inside the "parameters" object. ' +
+            'Do not place any additional fields at the root level.'
+    );
 
 export const DocSearchInputSchema = zod.object({
     query: zod

packages/fiori-mcp-server/test/integration/assertions/snapshots.ts

@@ -0,0 +1,272 @@
+import fs from 'fs/promises';
+import { existsSync } from 'fs';
+import { basename, join, dirname } from 'path';
+import type { AssertionValueFunctionContext, AssertionValueFunctionResult } from 'promptfoo';
+import { FOLDER_PATHS } from '../types';
+import assert from 'node:assert';
+
+interface SnapshotData {
+    content: {
+        snapshot: string;
+        source: string;
+    };
+    created: boolean;
+}
+
+interface SnapshotSegmentConfig {
+    path: Array<string>;
+    mode: string;
+}
+
+interface SnapshotConfiguration {
+    snapshot: string;
+    file: string;
+    segments?: SnapshotSegmentConfig[];
+}
+
+/**
+ * Compares the current test output against a stored snapshot to verify consistency.
+ * Loads the snapshot data based on the provided test context and configuration,
+ * validates it against the expected snapshot, and returns an assertion result.
+ *
+ * @param _output The string output generated by the test or process under validation.
+ * @param context The assertion context, containing test variables and optional configuration.
+ * @returns Result of assertion for promptfoo.
+ */
+export async function validate(
+    _output: string,
+    context: AssertionValueFunctionContext
+): Promise<AssertionValueFunctionResult> {
+    let reason = 'Unknown';
+    let pass = false;
+    const appPath = getAppPath(context.vars);
+    const config = context.config ? getConfiguration(context.config) : undefined;
+    if (appPath && config) {
+        try {
+            const snapshotData = await getSnapshotData(appPath, config.snapshot, config.file);
+            const compareResult = validateSnapshot(snapshotData, config);
+            pass = !compareResult;
+            if (!pass) {
+                console.log(`Snapshot mismatch for ${config.file}:\n${compareResult}`);
+            }
+            reason = pass ? 'Snapshot file matches' : `Snapshot file does not match: ${compareResult}`;
+        } catch (e) {
+            return {
+                pass: false,
+                score: 0,
+                reason: e.message
+            };
+        }
+    }
+
+    return {
+        pass,
+        score: pass ? 1 : 0,
+        reason
+    };
+}
+
+/**
+ * Retrieves the application path from a set of provided variables.
+ *
+ * @param vars Promptfoo context variables containing potential APP_PATH value.
+ * @returns The application path string if available and valid in passed variables, otherwise undefined.
+ */
+function getAppPath(vars: Record<string, string | object>): string | undefined {
+    return vars.APP_PATH && typeof vars.APP_PATH === 'string' ? vars.APP_PATH : undefined;
+}
+
+/**
+ * Parses and validates a raw configuration object into a normalized `SnapshotConfiguration`.
+ *
+ * @param config A raw configuration record, typically loaded from a test context.
+ * @returns A normalized `SnapshotConfiguration` object, or `undefined` if validation fails.
+ */
+function getConfiguration(config: Record<string, unknown>): SnapshotConfiguration | undefined {
+    if (
+        'file' in config &&
+        typeof config.file === 'string' &&
+        'snapshot' in config &&
+        typeof config.snapshot === 'string'
+    ) {
+        let segments: SnapshotSegmentConfig[] | undefined;
+        if ('segments' in config && Array.isArray(config.segments)) {
+            segments = config.segments.filter((segment) => typeof segment === 'object' && 'path' in segment);
+        }
+        return {
+            snapshot: config.snapshot,
+            file: config.file,
+            segments
+        };
+    }
+}
+
+/**
+ * Ensures the existence of a snapshot file for the given target and returns its data.
+ *
+ * @param projectPath The absolute path to the current project.
+ * @param key A key identifying the snapshot set or namespace.
+ * @param targetPath The relative path to the file being compared.
+ * @returns A `SnapshotData` object containing both source and snapshot contents.
+ */
+async function getSnapshotData(projectPath: string, key: string, targetPath: string): Promise<SnapshotData> {
+    let snapshotFolder = join(FOLDER_PATHS.snapshots, key);
+    const relativeFolder = dirname(join(targetPath));
+    if (relativeFolder) {
+        snapshotFolder = join(snapshotFolder, relativeFolder);
+    }
+    // Make sure snapshot folder exists
+    if (!existsSync(snapshotFolder)) {
+        await fs.mkdir(snapshotFolder, { recursive: true });
+    }
+    // Check target file
+    const filePath = join(projectPath, targetPath);
+    if (!existsSync(filePath)) {
+        throw new Error(`${filePath} does not exists`);
+    }
+    const fileName = basename(filePath);
+    const snapshotFile = join(snapshotFolder, fileName);
+    let created = false;
+    if (!existsSync(snapshotFile)) {
+        // Write snapshot
+        await fs.copyFile(filePath, snapshotFile);
+        created = true;
+    }
+    const sourceContent = await fs.readFile(filePath, 'utf8');
+    const snapshotContent = await fs.readFile(snapshotFile, 'utf8');
+    return {
+        content: {
+            snapshot: snapshotContent,
+            source: sourceContent
+        },
+        created
+    };
+}
+
+/**
+ * Validates that the provided source content matches its corresponding snapshot,
+ * optionally comparing specific JSON segments.
+ *
+ * @param snapshotData The snapshot and source contents to compare.
+ * @param config The snapshot configuration specifying comparison behavior.
+ * @returns A string describing the first detected difference or `undefined` if the snapshot matches the source.
+ */
+function validateSnapshot(snapshotData: SnapshotData, config: SnapshotConfiguration): string | undefined {
+    let compareResult: string | undefined;
+    if (config.segments && config.file.endsWith('.json')) {
+        const actual = JSON.parse(snapshotData.content.source);
+        const snapshot = JSON.parse(snapshotData.content.snapshot);
+        for (const segement of config.segments) {
+            const actualSegment = getByPath(actual, segement.path);
+            const snapshotSegment = getByPath(snapshot, segement.path);
+            if (
+                actualSegment !== null &&
+                snapshotSegment !== null &&
+                typeof actualSegment === 'object' &&
+                typeof snapshotSegment === 'object'
+            ) {
+                if (segement.mode === 'contains') {
+                    compareResult = deepContains(snapshotSegment, actualSegment);
+                } else {
+                    compareResult = compare(actualSegment, snapshotSegment);
+                }
+            } else {
+                compareResult =
+                    actualSegment === snapshotSegment ? undefined : `Value differs for ${segement.path.join('/')}`;
+            }
+            if (compareResult !== undefined) {
+                break;
+            }
+        }
+    } else {
+        compareResult = config.file.endsWith('.json')
+            ? compare(JSON.parse(snapshotData.content.source), JSON.parse(snapshotData.content.snapshot))
+            : compare(snapshotData.content.source, snapshotData.content.snapshot);
+    }
+    return compareResult;
+}
+
+/**
+ * Retrieves a nested value from an object or array using a sequence of keys.
+ *
+ * @param obj The object or array to traverse.
+ * @param path An array of keys or indices representing the access path.
+ * @returns The value found at the given path, or `undefined` if any part of the path is invalid.
+ */
+function getByPath(obj: unknown, path: (string | number)[]): unknown {
+    if (!Array.isArray(path)) {
+        return undefined;
+    }
+
+    let current: unknown = obj;
+
+    for (let i = 0; i < path.length; i++) {
+        const key = path[i];
+        // Ensure current is an object or array before trying to access properties
+        if (
+            current === null ||
+            typeof current !== 'object' ||
+            !(key in (current as Record<string | number, unknown>))
+        ) {
+            return undefined;
+        }
+
+        current = (current as Record<string | number, unknown>)[key];
+    }
+
+    return current;
+}
+
+/**
+ * Compares two values (objects or strings) for deep equality.
+ * Uses Node's built-in `assert.deepStrictEqual()` internally.
+ * If the values differ, it returns a human-readable diff string.
+ * If the values are deeply equal, it returns `undefined`.
+ *
+ * @param {string | object} value1 - The first object or string to compare.
+ * @param {string | object} value2 - The second object or string to compare.
+ * @returns {string | undefined} A string describing the differences if values differ,
+ *   or `undefined` if they are identical.
+ */
+function compare(value1: string | object, value2: string | object): string | undefined {
+    try {
+        assert.deepStrictEqual(value1, value2);
+    } catch (e) {
+        return e.message;
+    }
+}
+
+/**
+ * Recursively checks whether all properties and values of the `expected` object
+ * are contained within the `actual` object.
+ *
+ * @param expected The reference value or object to compare against.
+ * @param actual The object or value being tested for containment.
+ * @param path (Internal) The current object path used for detailed mismatch reporting.
+ * @returns A descriptive error string indicating the first mismatch or missing key,
+ *          or `undefined` if `actual` fully contains `expected`.
+ */
+function deepContains(expected: unknown, actual: unknown, path = ''): string | undefined {
+    // Handle primitive values (and null)
+    if (typeof expected !== 'object' || expected === null || typeof actual !== 'object' || actual === null) {
+        return Object.is(expected, actual) ? undefined : `Mismatch at ${path}: expected ${expected}, got ${actual}`;
+    }
+
+    // At this point both are non-null objects
+    const expectedObj = expected as Record<string, unknown>;
+    const actualObj = actual as Record<string, unknown>;
+
+    for (const key of Object.keys(expectedObj)) {
+        if (!(key in actualObj)) {
+            return `Missing key at ${path}.${key}`;
+        }
+
+        const result = deepContains(expectedObj[key], actualObj[key], `${path}.${key}`);
+        if (result) {
+            return result;
+        }
+    }
+
+    // Ignore extra keys in `actual`
+    return undefined;
+}