From 1c1ddbd7c39b0c5ea106285a7bd5c4ee58cba9b3 Mon Sep 17 00:00:00 2001 From: Ugaitz Urien Date: Wed, 8 Oct 2025 17:23:51 +0200 Subject: [PATCH] feat(aap): Extended request data collection via rules --- index.d.ts | 10 + packages/dd-trace/src/appsec/reporter.js | 91 ++++-- .../src/appsec/waf/waf_context_wrapper.js | 2 +- packages/dd-trace/src/config.js | 2 + packages/dd-trace/src/config_defaults.js | 2 + .../src/remote_config/capabilities.js | 1 + packages/dd-trace/src/remote_config/index.js | 2 + ...ded-data-collection.express.plugin.spec.js | 292 ++++++++++++++++++ .../extended-data-collection.rules.json | 74 +++++ .../dd-trace/test/appsec/reporter.spec.js | 174 +++++++---- .../dd-trace/test/appsec/waf/index.spec.js | 3 +- 11 files changed, 572 insertions(+), 81 deletions(-) create mode 100644 packages/dd-trace/test/appsec/extended-data-collection.express.plugin.spec.js create mode 100644 packages/dd-trace/test/appsec/extended-data-collection.rules.json diff --git a/index.d.ts b/index.d.ts index 6ab86b5c084..19bb065654f 100644 --- a/index.d.ts +++ b/index.d.ts @@ -786,6 +786,8 @@ declare namespace tracer { /** Whether to enable request body collection on RASP event * @default false + * + * @deprecated Use UI and Remote Configuration to enable extended data collection */ bodyCollection?: boolean }, @@ -810,20 +812,28 @@ declare namespace tracer { }, /** * Configuration for extended headers collection tied to security events + * + * @deprecated Use UI and Remote Configuration to enable extended data collection */ extendedHeadersCollection?: { /** Whether to enable extended headers collection * @default false + * + * @deprecated Use UI and Remote Configuration to enable extended data collection */ enabled: boolean, /** Whether to redact collected headers * @default true + * + * @deprecated Use UI and Remote Configuration to enable extended data collection */ redaction: boolean, /** Specifies the maximum number of headers collected. * @default 50 + * + * @deprecated Use UI and Remote Configuration to enable extended data collection */ maxHeaders: number, } diff --git a/packages/dd-trace/src/appsec/reporter.js b/packages/dd-trace/src/appsec/reporter.js index af806041e43..d9604e7eded 100644 --- a/packages/dd-trace/src/appsec/reporter.js +++ b/packages/dd-trace/src/appsec/reporter.js @@ -38,14 +38,20 @@ const config = { const metricsQueue = new Map() +const extendedDataCollectionRequest = new WeakMap() + // following header lists are ordered in the same way the spec orders them, it doesn't matter but it's easier to compare const contentHeaderList = [ 'content-length', - 'content-type', 'content-encoding', 'content-language' ] +const responseHeaderList = [ + ...contentHeaderList, + 'content-type' +] + const identificationHeaders = [ 'x-amzn-trace-id', 'cloudfront-viewer-ja3-fingerprint', @@ -75,15 +81,27 @@ const requestHeadersList = [ ...identificationHeaders ] +const redactedHeadersList = [ + 'authorization', + 'proxy-authorization', + 'www-authenticate', + 'proxy-authenticate', + 'authentication-info', + 'proxy-authentication-info', + 'cookie', + 'set-cookie' +] + // these request headers are always collected - it breaks the expected spec orders const REQUEST_HEADERS_MAP = mapHeaderAndTags(requestHeadersList, REQUEST_HEADER_TAG_PREFIX) const EVENT_HEADERS_MAP = mapHeaderAndTags(eventHeadersList, REQUEST_HEADER_TAG_PREFIX) -const RESPONSE_HEADERS_MAP = mapHeaderAndTags(contentHeaderList, RESPONSE_HEADER_TAG_PREFIX) +const RESPONSE_HEADERS_MAP = mapHeaderAndTags(responseHeaderList, RESPONSE_HEADER_TAG_PREFIX) const NON_EXTENDED_REQUEST_HEADERS = new Set([...requestHeadersList, ...eventHeadersList]) -const NON_EXTENDED_RESPONSE_HEADERS = new Set(contentHeaderList) +const NON_EXTENDED_RESPONSE_HEADERS = new Set(responseHeaderList) +const REDACTED_HEADERS = new Set(redactedHeadersList) function init (_config) { config.headersExtendedCollectionEnabled = _config.extendedHeadersCollection.enabled @@ -132,7 +150,9 @@ function filterExtendedHeaders (headers, excludedHeaderNames, tagPrefix, limit = for (const [headerName, headerValue] of Object.entries(headers)) { if (counter >= limit) break if (!excludedHeaderNames.has(headerName)) { - result[getHeaderTag(tagPrefix, headerName)] = String(headerValue) + result[getHeaderTag(tagPrefix, headerName)] = REDACTED_HEADERS.has(headerName) + ? '' + : String(headerValue) counter++ } } @@ -140,7 +160,7 @@ function filterExtendedHeaders (headers, excludedHeaderNames, tagPrefix, limit = return result } -function getCollectedHeaders (req, res, shouldCollectEventHeaders, storedResponseHeaders = {}) { +function getCollectedHeaders (req, res, shouldCollectEventHeaders, storedResponseHeaders = {}, extendedDataCollection) { // Mandatory const mandatoryCollectedHeaders = filterHeaders(req.headers, REQUEST_HEADERS_MAP) @@ -154,7 +174,8 @@ function getCollectedHeaders (req, res, shouldCollectEventHeaders, storedRespons const requestEventCollectedHeaders = filterHeaders(req.headers, EVENT_HEADERS_MAP) const responseEventCollectedHeaders = filterHeaders(responseHeaders, RESPONSE_HEADERS_MAP) - if (!config.headersExtendedCollectionEnabled || config.headersRedaction) { + // TODO headersExtendedCollectionEnabled and headersRedaction properties are deprecated to delete in a major + if ((!config.headersExtendedCollectionEnabled || config.headersRedaction) && !extendedDataCollection) { // Standard collection return Object.assign( mandatoryCollectedHeaders, @@ -163,12 +184,15 @@ function getCollectedHeaders (req, res, shouldCollectEventHeaders, storedRespons ) } + // TODO config.maxHeadersCollected is deprecated to delete in a major + const maxHeadersCollected = extendedDataCollection?.max_collected_headers ?? config.maxHeadersCollected + // Extended collection - const requestExtendedHeadersAvailableCount = - config.maxHeadersCollected - - Object.keys(mandatoryCollectedHeaders).length - + const collectedHeadersCount = Object.keys(mandatoryCollectedHeaders).length + Object.keys(requestEventCollectedHeaders).length + const requestExtendedHeadersAvailableCount = maxHeadersCollected - collectedHeadersCount + const requestEventExtendedCollectedHeaders = filterExtendedHeaders( req.headers, @@ -178,7 +202,7 @@ function getCollectedHeaders (req, res, shouldCollectEventHeaders, storedRespons ) const responseExtendedHeadersAvailableCount = - config.maxHeadersCollected - + maxHeadersCollected - Object.keys(responseEventCollectedHeaders).length const responseEventExtendedCollectedHeaders = @@ -199,15 +223,15 @@ function getCollectedHeaders (req, res, shouldCollectEventHeaders, storedRespons // Check discarded headers const requestHeadersCount = Object.keys(req.headers).length - if (requestHeadersCount > config.maxHeadersCollected) { + if (requestHeadersCount > maxHeadersCollected) { headersTags['_dd.appsec.request.header_collection.discarded'] = - requestHeadersCount - config.maxHeadersCollected + requestHeadersCount - maxHeadersCollected } const responseHeadersCount = Object.keys(responseHeaders).length - if (responseHeadersCount > config.maxHeadersCollected) { + if (responseHeadersCount > maxHeadersCollected) { headersTags['_dd.appsec.response.header_collection.discarded'] = - responseHeadersCount - config.maxHeadersCollected + responseHeadersCount - maxHeadersCollected } return headersTags @@ -307,7 +331,7 @@ function reportTruncationMetrics (rootSpan, metrics) { } } -function reportAttack (attackData) { +function reportAttack ({ events: attackData, actions }) { const store = storage('legacy').getStore() const req = store?.req const rootSpan = web.root(req) @@ -338,8 +362,14 @@ function reportAttack (attackData) { rootSpan.addTags(newTags) + // TODO this should be deleted in a major if (config.raspBodyCollection && isRaspAttack(attackData)) { - reportRequestBody(rootSpan, req.body) + reportRequestBody(rootSpan, req.body, true) + } + + const extendedDataCollection = actions?.extended_data_collection + if (extendedDataCollection) { + extendedDataCollectionRequest.set(req, extendedDataCollection) } } @@ -398,18 +428,29 @@ function truncateRequestBody (target, depth = 0) { } } -function reportRequestBody (rootSpan, requestBody) { - if (!requestBody) return +function reportRequestBody (rootSpan, requestBody, comesFromRaspAction = false) { + if (!requestBody || Object.keys(requestBody).length === 0) return if (!rootSpan.meta_struct) { rootSpan.meta_struct = {} } - if (!rootSpan.meta_struct['http.request.body']) { + if (rootSpan.meta_struct['http.request.body']) { + // If the rasp.exceed metric exists, set also the same for the new tag + const currentTags = rootSpan.context()._tags + const sizeExceedTagValue = currentTags['_dd.appsec.rasp.request_body_size.exceeded'] + + if (sizeExceedTagValue) { + rootSpan.setTag('_dd.appsec.request_body_size.exceeded', sizeExceedTagValue) + } + } else { const { truncated, value } = truncateRequestBody(requestBody) rootSpan.meta_struct['http.request.body'] = value if (truncated) { - rootSpan.setTag('_dd.appsec.rasp.request_body_size.exceeded', 'true') + const sizeExceedTagKey = comesFromRaspAction + ? '_dd.appsec.rasp.request_body_size.exceeded' // TODO old metric to delete in a major + : '_dd.appsec.request_body_size.exceeded' + rootSpan.setTag(sizeExceedTagKey, 'true') } } } @@ -496,7 +537,15 @@ function finishRequest (req, res, storedResponseHeaders) { const tags = rootSpan.context()._tags - const newTags = getCollectedHeaders(req, res, shouldCollectEventHeaders(tags), storedResponseHeaders) + const extendedDataCollection = extendedDataCollectionRequest.get(req) + const newTags = getCollectedHeaders( + req, res, shouldCollectEventHeaders(tags), storedResponseHeaders, extendedDataCollection + ) + + if (extendedDataCollection) { + // TODO add support for fastify, req.body is not available in fastify + reportRequestBody(rootSpan, req.body) + } if (tags['appsec.event'] === 'true' && typeof req.route?.path === 'string') { newTags['http.endpoint'] = req.route.path diff --git a/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js b/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js index d9d8856982f..f684b4b703c 100644 --- a/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js +++ b/packages/dd-trace/src/appsec/waf/waf_context_wrapper.js @@ -141,7 +141,7 @@ class WAFContextWrapper { metrics.wafTimeout = result.timeout if (ruleTriggered) { - Reporter.reportAttack(result.events) + Reporter.reportAttack(result) } Reporter.reportAttributes(result.attributes) diff --git a/packages/dd-trace/src/config.js b/packages/dd-trace/src/config.js index 2195b86ed60..a8b22799d3c 100644 --- a/packages/dd-trace/src/config.js +++ b/packages/dd-trace/src/config.js @@ -668,6 +668,7 @@ class Config { this._envUnprocessed['appsec.blockedTemplateJson'] = DD_APPSEC_HTTP_BLOCKED_TEMPLATE_JSON this._setBoolean(env, 'appsec.enabled', DD_APPSEC_ENABLED) this._setString(env, 'appsec.eventTracking.mode', DD_APPSEC_AUTO_USER_INSTRUMENTATION_MODE) + // TODO appsec.extendedHeadersCollection are deprecated, to delete in a major this._setBoolean(env, 'appsec.extendedHeadersCollection.enabled', DD_APPSEC_COLLECT_ALL_HEADERS) this._setBoolean( env, @@ -679,6 +680,7 @@ class Config { this._setString(env, 'appsec.obfuscatorKeyRegex', DD_APPSEC_OBFUSCATION_PARAMETER_KEY_REGEXP) this._setString(env, 'appsec.obfuscatorValueRegex', DD_APPSEC_OBFUSCATION_PARAMETER_VALUE_REGEXP) this._setBoolean(env, 'appsec.rasp.enabled', DD_APPSEC_RASP_ENABLED) + // TODO Deprecated, to delete in a major this._setBoolean(env, 'appsec.rasp.bodyCollection', DD_APPSEC_RASP_COLLECT_REQUEST_BODY) env['appsec.rateLimit'] = maybeInt(DD_APPSEC_TRACE_RATE_LIMIT) this._envUnprocessed['appsec.rateLimit'] = DD_APPSEC_TRACE_RATE_LIMIT diff --git a/packages/dd-trace/src/config_defaults.js b/packages/dd-trace/src/config_defaults.js index 3054bb46fe8..acb6bb1cce3 100644 --- a/packages/dd-trace/src/config_defaults.js +++ b/packages/dd-trace/src/config_defaults.js @@ -36,12 +36,14 @@ module.exports = { 'appsec.blockedTemplateJson': undefined, 'appsec.enabled': undefined, 'appsec.eventTracking.mode': 'identification', + // TODO appsec.extendedHeadersCollection is deprecated, to delete in a major 'appsec.extendedHeadersCollection.enabled': false, 'appsec.extendedHeadersCollection.redaction': true, 'appsec.extendedHeadersCollection.maxHeaders': 50, 'appsec.obfuscatorKeyRegex': defaultWafObfuscatorKeyRegex, 'appsec.obfuscatorValueRegex': defaultWafObfuscatorValueRegex, 'appsec.rasp.enabled': true, + // TODO Deprecated, to delete in a major 'appsec.rasp.bodyCollection': false, 'appsec.rateLimit': 100, 'appsec.rules': undefined, diff --git a/packages/dd-trace/src/remote_config/capabilities.js b/packages/dd-trace/src/remote_config/capabilities.js index b46755bbc86..e8f65cf30d0 100644 --- a/packages/dd-trace/src/remote_config/capabilities.js +++ b/packages/dd-trace/src/remote_config/capabilities.js @@ -31,6 +31,7 @@ module.exports = { ASM_RASP_CMDI: 1n << 37n, ASM_DD_MULTICONFIG: 1n << 42n, ASM_TRACE_TAGGING_RULES: 1n << 43n, + ASM_EXTENDED_DATA_COLLECTION: 1n << 44n, /* DO NOT ADD ARBITRARY CAPABILITIES IN YOUR CODE UNLESS THEY ARE ALREADY DEFINED IN THE BACKEND SOURCE OF TRUTH diff --git a/packages/dd-trace/src/remote_config/index.js b/packages/dd-trace/src/remote_config/index.js index 813d7ed577c..f5c491c8668 100644 --- a/packages/dd-trace/src/remote_config/index.js +++ b/packages/dd-trace/src/remote_config/index.js @@ -96,6 +96,7 @@ function enableWafUpdate (appsecConfig) { rc.updateCapabilities(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, true) rc.updateCapabilities(RemoteConfigCapabilities.ASM_DD_MULTICONFIG, true) rc.updateCapabilities(RemoteConfigCapabilities.ASM_TRACE_TAGGING_RULES, true) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_EXTENDED_DATA_COLLECTION, true) if (appsecConfig.rasp?.enabled) { rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SQLI, true) @@ -134,6 +135,7 @@ function disableWafUpdate () { rc.updateCapabilities(RemoteConfigCapabilities.ASM_HEADER_FINGERPRINT, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_DD_MULTICONFIG, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_TRACE_TAGGING_RULES, false) + rc.updateCapabilities(RemoteConfigCapabilities.ASM_EXTENDED_DATA_COLLECTION, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SQLI, false) rc.updateCapabilities(RemoteConfigCapabilities.ASM_RASP_SSRF, false) diff --git a/packages/dd-trace/test/appsec/extended-data-collection.express.plugin.spec.js b/packages/dd-trace/test/appsec/extended-data-collection.express.plugin.spec.js new file mode 100644 index 00000000000..a100ce5d82f --- /dev/null +++ b/packages/dd-trace/test/appsec/extended-data-collection.express.plugin.spec.js @@ -0,0 +1,292 @@ +'use strict' + +const Config = require('../../src/config') +const path = require('path') +const { withVersions } = require('../setup/mocha') +const agent = require('../plugins/agent') +const appsec = require('../../src/appsec') +const axios = require('axios') +const assert = require('assert') +const msgpack = require('@msgpack/msgpack') + +function createDeepObject (sheetValue, currentLevel = 1, max = 20) { + if (currentLevel === max) { + return { + [`s-${currentLevel}`]: `s-${currentLevel}`, + [`o-${currentLevel}`]: sheetValue + } + } + + return { + [`s-${currentLevel}`]: `s-${currentLevel}`, + [`o-${currentLevel}`]: createDeepObject(sheetValue, currentLevel + 1, max) + } +} + +describe('extended data collection', () => { + before(() => { + return agent.load(['express', 'http'], { client: false }) + }) + + after(() => { + return agent.close({ ritmReset: false }) + }) + + withVersions('express', 'express', expressVersion => { + let port, server + + before((done) => { + const express = require(`../../../../versions/express@${expressVersion}`).get() + const bodyParser = require('../../../../versions/body-parser').get() + + const app = express() + app.use(bodyParser.json()) + + app.post('/', (req, res) => { + res.setHeader('custom-response-header-1', 'custom-response-header-value-1') + res.setHeader('custom-response-header-2', 'custom-response-header-value-2') + res.setHeader('custom-response-header-3', 'custom-response-header-value-3') + res.setHeader('custom-response-header-4', 'custom-response-header-value-4') + res.setHeader('custom-response-header-5', 'custom-response-header-value-5') + res.setHeader('custom-response-header-6', 'custom-response-header-value-6') + res.setHeader('custom-response-header-7', 'custom-response-header-value-7') + res.setHeader('custom-response-header-8', 'custom-response-header-value-8') + res.setHeader('custom-response-header-9', 'custom-response-header-value-9') + res.setHeader('custom-response-header-10', 'custom-response-header-value-10') + + res.end('DONE') + }) + + app.post('/redacted-headers', (req, res) => { + res.setHeader('authorization', 'header-value-1') + res.setHeader('proxy-authorization', 'header-value-2') + res.setHeader('www-authenticate', 'header-value-4') + res.setHeader('proxy-authenticate', 'header-value-5') + res.setHeader('authentication-info', 'header-value-6') + res.setHeader('proxy-authentication-info', 'header-value-7') + res.setHeader('cookie', 'header-value-8') + res.setHeader('set-cookie', 'header-value-9') + + res.end('DONE') + }) + + server = app.listen(port, () => { + port = server.address().port + done() + }) + }) + + after(() => { + server.close() + }) + + beforeEach(() => { + appsec.enable(new Config( + { + appsec: { + enabled: true, + rules: path.join(__dirname, './extended-data-collection.rules.json') + } + } + )) + }) + + afterEach(() => { + appsec.disable() + }) + + it('Should collect nothing when no extended_data_collection is triggered', async () => { + const requestBody = { + other: 'other', + chained: { + child: 'one', + child2: 2 + } + } + await axios.post( + `http://localhost:${port}/`, + requestBody, + { + headers: { + 'custom-header-key-1': 'custom-header-value-1', + 'custom-header-key-2': 'custom-header-value-2', + 'custom-header-key-3': 'custom-header-value-3' + } + } + ) + + await agent.assertSomeTraces((traces) => { + const span = traces[0][0] + assert.strictEqual(span.type, 'web') + + assert.strictEqual(span.meta['http.request.headers.custom-request-header-1'], undefined) + assert.strictEqual(span.meta['http.request.headers.custom-request-header-2'], undefined) + assert.strictEqual(span.meta['http.request.headers.custom-request-header-3'], undefined) + + assert.strictEqual(span.meta['http.response.headers.custom-response-header-1'], undefined) + assert.strictEqual(span.meta['http.response.headers.custom-response-header-2'], undefined) + assert.strictEqual(span.meta['http.response.headers.custom-response-header-3'], undefined) + + const rawMetaStructBody = span.meta_struct?.['http.request.body'] + assert.strictEqual(rawMetaStructBody, undefined) + }) + }) + + it('Should redact request/response headers', async () => { + const requestBody = { + bodyParam: 'collect-standard' + } + await axios.post( + `http://localhost:${port}/redacted-headers`, + requestBody, + { + headers: { + authorization: 'header-value-1', + 'proxy-authorization': 'header-value-2', + 'www-authenticate': 'header-value-3', + 'proxy-authenticate': 'header-value-4', + 'authentication-info': 'header-value-5', + 'proxy-authentication-info': 'header-value-6', + cookie: 'header-value-7', + 'set-cookie': 'header-value-8' + } + } + ) + + await agent.assertSomeTraces((traces) => { + const span = traces[0][0] + assert.strictEqual(span.type, 'web') + assert.strictEqual(span.meta['http.request.headers.authorization'], '') + assert.strictEqual(span.meta['http.request.headers.proxy-authorization'], '') + assert.strictEqual(span.meta['http.request.headers.www-authenticate'], '') + assert.strictEqual(span.meta['http.request.headers.proxy-authenticate'], '') + assert.strictEqual(span.meta['http.request.headers.authentication-info'], '') + assert.strictEqual(span.meta['http.request.headers.proxy-authentication-info'], '') + assert.strictEqual(span.meta['http.request.headers.cookie'], '') + assert.strictEqual(span.meta['http.request.headers.set-cookie'], '') + + assert.strictEqual(span.meta['http.response.headers.authorization'], '') + assert.strictEqual(span.meta['http.response.headers.proxy-authorization'], '') + assert.strictEqual(span.meta['http.response.headers.www-authenticate'], '') + assert.strictEqual(span.meta['http.response.headers.proxy-authenticate'], '') + assert.strictEqual(span.meta['http.response.headers.authentication-info'], '') + assert.strictEqual(span.meta['http.response.headers.proxy-authentication-info'], '') + assert.strictEqual(span.meta['http.response.headers.cookie'], '') + assert.strictEqual(span.meta['http.response.headers.set-cookie'], '') + }) + }) + + it('Should collect request body and request/response with a max of 8 headers', async () => { + const requestBody = { + bodyParam: 'collect-few-headers', + other: 'other', + chained: { + child: 'one', + child2: 2 + } + } + await axios.post( + `http://localhost:${port}/`, + requestBody, + { + headers: { + 'custom-request-header-1': 'custom-request-header-value-1', + 'custom-request-header-2': 'custom-request-header-value-2', + 'custom-request-header-3': 'custom-request-header-value-3', + 'custom-request-header-4': 'custom-request-header-value-4', + 'custom-request-header-5': 'custom-request-header-value-5', + 'custom-request-header-6': 'custom-request-header-value-6', + 'custom-request-header-7': 'custom-request-header-value-7', + 'custom-request-header-8': 'custom-request-header-value-8', + 'custom-request-header-9': 'custom-request-header-value-9', + 'custom-request-header-10': 'custom-request-header-value-10' + } + } + ) + + await agent.assertSomeTraces((traces) => { + const span = traces[0][0] + assert.strictEqual(span.type, 'web') + const collectedRequestHeaders = Object.keys(span.meta) + .filter(metaKey => metaKey.startsWith('http.request.headers.')).length + const collectedResponseHeaders = Object.keys(span.meta) + .filter(metaKey => metaKey.startsWith('http.response.headers.')).length + assert.strictEqual(collectedRequestHeaders, 8) + assert.strictEqual(collectedResponseHeaders, 8) + + assert.ok(span.metrics['_dd.appsec.request.header_collection.discarded'] > 2) + assert.ok(span.metrics['_dd.appsec.response.header_collection.discarded'] > 2) + + const metaStructBody = msgpack.decode(span.meta_struct['http.request.body']) + assert.deepEqual(metaStructBody, requestBody) + }) + }) + + it('Should truncate the request body when depth is more than 20 levels', async () => { + const deepObject = createDeepObject('sheet') + + const requestBody = { + bodyParam: 'collect-standard', + deepObject + } + + const expectedDeepTruncatedObject = createDeepObject({ 's-19': 's-19' }, 1, 18) + const expectedRequestBody = { + bodyParam: 'collect-standard', + deepObject: expectedDeepTruncatedObject + } + await axios.post(`http://localhost:${port}/`, requestBody) + + await agent.assertSomeTraces((traces) => { + const span = traces[0][0] + assert.strictEqual(span.type, 'web') + + const metaStructBody = msgpack.decode(span.meta_struct['http.request.body']) + assert.deepEqual(metaStructBody, expectedRequestBody) + }) + }) + + it('Should truncate the request body when string length is more than 4096 characters', async () => { + const requestBody = { + bodyParam: 'collect-standard', + longValue: Array(5000).fill('A').join('') + } + + const expectedRequestBody = { + bodyParam: 'collect-standard', + longValue: Array(4096).fill('A').join('') + } + await axios.post(`http://localhost:${port}/`, requestBody) + + await agent.assertSomeTraces((traces) => { + const span = traces[0][0] + assert.strictEqual(span.type, 'web') + + const metaStructBody = msgpack.decode(span.meta_struct['http.request.body']) + assert.deepEqual(metaStructBody, expectedRequestBody) + }) + }) + + it('Should truncate the request body when a node has more than 256 elements', async () => { + const children = Array(300).fill('item') + const requestBody = { + bodyParam: 'collect-standard', + children + } + + const expectedRequestBody = { + bodyParam: 'collect-standard', + children: children.slice(0, 256) + } + await axios.post(`http://localhost:${port}/`, requestBody) + + await agent.assertSomeTraces((traces) => { + const span = traces[0][0] + assert.strictEqual(span.type, 'web') + + const metaStructBody = msgpack.decode(span.meta_struct['http.request.body']) + assert.deepEqual(metaStructBody, expectedRequestBody) + }) + }) + }) +}) diff --git a/packages/dd-trace/test/appsec/extended-data-collection.rules.json b/packages/dd-trace/test/appsec/extended-data-collection.rules.json new file mode 100644 index 00000000000..fa45346ff85 --- /dev/null +++ b/packages/dd-trace/test/appsec/extended-data-collection.rules.json @@ -0,0 +1,74 @@ +{ + "version": "2.2", + "metadata": { + "rules_version": "1.5.0" + }, + "rules": [ + { + "id": "test-rule-id-1", + "name": "test-rule-name-1", + "tags": { + "type": "a", + "category": "custom" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.body" + } + ], + "list": [ + "collect-standard" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": ["lowercase"], + "on_match": ["extended_data_collection_standard"] + }, + { + "id": "test-rule-id-2", + "name": "test-rule-name-2", + "tags": { + "type": "a", + "category": "custom" + }, + "conditions": [ + { + "parameters": { + "inputs": [ + { + "address": "server.request.body" + } + ], + "list": [ + "collect-few-headers" + ] + }, + "operator": "phrase_match" + } + ], + "transformers": ["lowercase"], + "on_match": ["extended_data_collection_few_headers"] + } + ], + "actions": [ + { + "id": "extended_data_collection_standard", + "parameters": { + "max_collected_headers": 50 + }, + "type": "extended_data_collection" + }, + { + "id": "extended_data_collection_few_headers", + "parameters": { + "max_collected_headers": 8 + }, + "type": "extended_data_collection" + } + ] +} diff --git a/packages/dd-trace/test/appsec/reporter.spec.js b/packages/dd-trace/test/appsec/reporter.spec.js index 2a384c9f1db..b175869cd12 100644 --- a/packages/dd-trace/test/appsec/reporter.spec.js +++ b/packages/dd-trace/test/appsec/reporter.spec.js @@ -384,12 +384,14 @@ describe('reporter', () => { it('should add tags to request span when socket is not there', () => { delete req.socket - Reporter.reportAttack([ - { - rule: {}, - rule_matches: [{}] - } - ]) + Reporter.reportAttack({ + events: [ + { + rule: {}, + rule_matches: [{}] + } + ] + }) expect(web.root).to.have.been.calledOnceWith(req) expect(span.addTags).to.have.been.calledOnceWithExactly({ @@ -400,12 +402,14 @@ describe('reporter', () => { }) it('should add tags to request span', () => { - Reporter.reportAttack([ - { - rule: {}, - rule_matches: [{}] - } - ]) + Reporter.reportAttack({ + events: [ + { + rule: {}, + rule_matches: [{}] + } + ] + }) expect(web.root).to.have.been.calledOnceWith(req) expect(span.addTags).to.have.been.calledOnceWithExactly({ @@ -419,7 +423,7 @@ describe('reporter', () => { it('should not overwrite origin tag', () => { span.context()._tags = { '_dd.origin': 'tracer' } - Reporter.reportAttack([]) + Reporter.reportAttack({ events: [] }) expect(web.root).to.have.been.calledOnceWith(req) expect(span.addTags).to.have.been.calledOnceWithExactly({ @@ -432,15 +436,17 @@ describe('reporter', () => { it('should merge attacks json', () => { span.context()._tags = { '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]}]}' } - Reporter.reportAttack([ - { - rule: {} - }, - { - rule: {}, - rule_matches: [{}] - } - ]) + Reporter.reportAttack({ + events: [ + { + rule: {} + }, + { + rule: {}, + rule_matches: [{}] + } + ] + }) expect(web.root).to.have.been.calledOnceWith(req) expect(span.addTags).to.have.been.calledOnceWithExactly({ @@ -454,15 +460,17 @@ describe('reporter', () => { it('should call standalone sample', () => { span.context()._tags = { '_dd.appsec.json': '{"triggers":[{"rule":{},"rule_matches":[{}]}]}' } - Reporter.reportAttack([ - { - rule: {} - }, - { - rule: {}, - rule_matches: [{}] - } - ]) + Reporter.reportAttack({ + events: [ + { + rule: {} + }, + { + rule: {}, + rule_matches: [{}] + } + ] + }) expect(web.root).to.have.been.calledOnceWith(req) expect(span.addTags).to.have.been.calledOnceWithExactly({ @@ -505,16 +513,18 @@ describe('reporter', () => { } ) - Reporter.reportAttack([ - { - rule: { - tags: { - module: 'rasp' - } - }, - rule_matches: [{}] - } - ]) + Reporter.reportAttack({ + events: [ + { + rule: { + tags: { + module: 'rasp' + } + }, + rule_matches: [{}] + } + ] + }) expect(span.meta_struct['http.request.body']).to.be.deep.equal(expectedBody) }) @@ -534,16 +544,18 @@ describe('reporter', () => { } ) - Reporter.reportAttack([ - { - rule: { - tags: { - module: 'rasp' - } - }, - rule_matches: [{}] - } - ]) + Reporter.reportAttack({ + events: [ + { + rule: { + tags: { + module: 'rasp' + } + }, + rule_matches: [{}] + } + ] + }) expect(span.meta_struct?.['http.request.body']).to.be.undefined }) @@ -616,18 +628,64 @@ describe('reporter', () => { req.body = requestBody - Reporter.reportAttack([ + Reporter.reportAttack({ + events: [ + { + rule: { + tags: { + module: 'rasp' + } + }, + rule_matches: [{}] + } + ] + }) + + expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.rasp.request_body_size.exceeded', 'true') + }) + + it('should set request body size exceeded metric for old and new approaches when both events happen', () => { + Reporter.init( { - rule: { - tags: { - module: 'rasp' - } + rateLimit: 100, + extendedHeadersCollection: { + enabled: false, + redaction: true, + maxHeaders: 50 }, - rule_matches: [{}] + rasp: { + bodyCollection: true + } + } + ) + + req.body = requestBody + + Reporter.reportAttack({ + events: [ + { + rule: { + tags: { + module: 'rasp' + } + }, + rule_matches: [{}] + } + ], + actions: { + extended_data_collection: { + max_collected_headers: 10 + } } - ]) + }) expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.rasp.request_body_size.exceeded', 'true') + span.context()._tags = { '_dd.appsec.rasp.request_body_size.exceeded': 'true' } + + const res = {} + Reporter.finishRequest(req, res, {}) + + expect(span.setTag).to.have.been.calledWithExactly('_dd.appsec.request_body_size.exceeded', 'true') }) }) }) diff --git a/packages/dd-trace/test/appsec/waf/index.spec.js b/packages/dd-trace/test/appsec/waf/index.spec.js index c273decd041..7d2194d9fdd 100644 --- a/packages/dd-trace/test/appsec/waf/index.spec.js +++ b/packages/dd-trace/test/appsec/waf/index.spec.js @@ -8,6 +8,7 @@ const proxyquire = require('proxyquire') const Config = require('../../../src/config') const rules = require('../../../src/appsec/recommended.json') const Reporter = require('../../../src/appsec/reporter') +const { match } = require('sinon') describe('WAF Manager', () => { const knownAddresses = new Set([ @@ -458,7 +459,7 @@ describe('WAF Manager', () => { wafContextWrapper.run(params) - expect(Reporter.reportAttack).to.be.calledOnceWithExactly(['ATTACK DATA']) + expect(Reporter.reportAttack).to.be.calledOnceWith(match({ events: ['ATTACK DATA'] })) }) it('should report if rule is triggered', () => {