diff --git a/.changeset/chilled-chairs-design.md b/.changeset/chilled-chairs-design.md new file mode 100644 index 000000000..e1b27cafd --- /dev/null +++ b/.changeset/chilled-chairs-design.md @@ -0,0 +1,5 @@ +--- +"@hyperdx/common-utils": minor +--- + +Add support for visualizing histogram counts diff --git a/.gitignore b/.gitignore index 180f8b26f..a0c525a78 100644 --- a/.gitignore +++ b/.gitignore @@ -72,4 +72,7 @@ docker-compose.prod.yml .volumes # NX -.nx/ \ No newline at end of file +.nx/ + +# webstorm +.idea \ No newline at end of file diff --git a/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap b/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap index d4cc2c1ee..b7424e1ed 100644 --- a/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap +++ b/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap @@ -18,7 +18,103 @@ exports[`renderChartConfig containing CTE clauses should render a ChSql CTE conf exports[`renderChartConfig containing CTE clauses should render a chart config CTE configuration correctly 1`] = `"WITH Parts AS (SELECT _part, _part_offset FROM default.some_table WHERE ((FieldA = 'test')) ORDER BY rand() DESC LIMIT 1000) SELECT * FROM Parts WHERE ((FieldA = 'test') AND (indexHint((_part, _part_offset) IN (SELECT tuple(_part, _part_offset) FROM Parts)))) ORDER BY rand() DESC LIMIT 1000"`; -exports[`renderChartConfig histogram metric queries should generate a query with grouping and time bucketing 1`] = ` +exports[`renderChartConfig histogram metric queries count should generate a count query with grouping and time bucketing 1`] = ` +"WITH grouped_data AS ( + SELECT + toStartOfInterval(toDateTime(TimeUnix), INTERVAL 2 minute) AS \`__hdx_time_bucket\`, + [ResourceAttributes['host']] as group, + toInt64(sum(Count)) as cumulative_count, + AggregationTemporality + FROM default.otel_metrics_histogram + WHERE (TimeUnix >= toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 2 minute) - INTERVAL 2 minute AND TimeUnix <= toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 2 minute) + INTERVAL 2 minute) AND ((MetricName = 'http.server.duration')) + GROUP BY group, \`__hdx_time_bucket\`, AggregationTemporality + ),metrics AS ( + SELECT + \`__hdx_time_bucket\`, + group, + CASE + WHEN AggregationTemporality = 1 THEN cumulative_count + WHEN AggregationTemporality = 2 THEN + GREATEST(0, + cumulative_count - COALESCE( + lag(cumulative_count) OVER ( + PARTITION BY group, AggregationTemporality + ORDER BY \`__hdx_time_bucket\` + ), + 0 + ) + ) + ELSE 0 + END as \\"Value\\" + FROM grouped_data + ) SELECT \`__hdx_time_bucket\`, group, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" +`; + +exports[`renderChartConfig histogram metric queries count should generate a count query without grouping but time bucketing 1`] = ` +"WITH grouped_data AS ( + SELECT + toStartOfInterval(toDateTime(TimeUnix), INTERVAL 2 minute) AS \`__hdx_time_bucket\`, + + toInt64(sum(Count)) as cumulative_count, + AggregationTemporality + FROM default.otel_metrics_histogram + WHERE (TimeUnix >= toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 2 minute) - INTERVAL 2 minute AND TimeUnix <= toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 2 minute) + INTERVAL 2 minute) AND ((MetricName = 'http.server.duration')) + GROUP BY \`__hdx_time_bucket\`, AggregationTemporality + ),metrics AS ( + SELECT + \`__hdx_time_bucket\`, + + CASE + WHEN AggregationTemporality = 1 THEN cumulative_count + WHEN AggregationTemporality = 2 THEN + GREATEST(0, + cumulative_count - COALESCE( + lag(cumulative_count) OVER ( + PARTITION BY AggregationTemporality + ORDER BY \`__hdx_time_bucket\` + ), + 0 + ) + ) + ELSE 0 + END as \\"Value\\" + FROM grouped_data + ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" +`; + +exports[`renderChartConfig histogram metric queries count should generate a count query without grouping or time bucketing 1`] = ` +"WITH grouped_data AS ( + SELECT + TimeUnix AS \`__hdx_time_bucket\`, + + toInt64(sum(Count)) as cumulative_count, + AggregationTemporality + FROM default.otel_metrics_histogram + WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND ((MetricName = 'http.server.duration')) + GROUP BY \`__hdx_time_bucket\`, AggregationTemporality + ),metrics AS ( + SELECT + \`__hdx_time_bucket\`, + + CASE + WHEN AggregationTemporality = 1 THEN cumulative_count + WHEN AggregationTemporality = 2 THEN + GREATEST(0, + cumulative_count - COALESCE( + lag(cumulative_count) OVER ( + PARTITION BY AggregationTemporality + ORDER BY \`__hdx_time_bucket\` + ), + 0 + ) + ) + ELSE 0 + END as \\"Value\\" + FROM grouped_data + ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" +`; + +exports[`renderChartConfig histogram metric queries quantile should generate a query with grouping and time bucketing 1`] = ` "WITH source AS ( SELECT MetricName, @@ -102,7 +198,7 @@ exports[`renderChartConfig histogram metric queries should generate a query with ) SELECT \`__hdx_time_bucket\`, group, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" `; -exports[`renderChartConfig histogram metric queries should generate a query without grouping but time bucketing 1`] = ` +exports[`renderChartConfig histogram metric queries quantile should generate a query without grouping but time bucketing 1`] = ` "WITH source AS ( SELECT MetricName, @@ -186,12 +282,12 @@ exports[`renderChartConfig histogram metric queries should generate a query with ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable'" `; -exports[`renderChartConfig histogram metric queries should generate a query without grouping or time bucketing 1`] = ` +exports[`renderChartConfig histogram metric queries quantile should generate a query without grouping or time bucketing 1`] = ` "WITH source AS ( SELECT MetricName, ExplicitBounds, - TimeUnix AS \`__hdx_time_bucket\` + TimeUnix AS \`__hdx_time_bucket\`, sumForEach(deltas) as rates FROM ( diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 89dd1f8bf..a7d0afa10 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -157,115 +157,227 @@ describe('renderChartConfig', () => { }); describe('histogram metric queries', () => { - it('should generate a query without grouping or time bucketing', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Line, - connection: 'test-connection', - metricTables: { - gauge: 'otel_metrics_gauge', - histogram: 'otel_metrics_histogram', - sum: 'otel_metrics_sum', - summary: 'otel_metrics_summary', - 'exponential histogram': 'otel_metrics_exponential_histogram', - }, - from: { - databaseName: 'default', - tableName: '', - }, - select: [ - { - aggFn: 'quantile', - level: 0.5, - valueExpression: 'Value', - metricName: 'http.server.duration', - metricType: MetricsDataType.Histogram, + describe('quantile', () => { + it('should generate a query without grouping or time bucketing', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', }, - ], - where: '', - whereLanguage: 'sql', - timestampValueExpression: 'TimeUnix', - dateRange: [new Date('2025-02-12'), new Date('2025-12-14')], - limit: { limit: 10 }, - }; + from: { + databaseName: 'default', + tableName: '', + }, + select: [ + { + aggFn: 'quantile', + level: 0.5, + valueExpression: 'Value', + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + }, + ], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-02-12'), new Date('2025-12-14')], + limit: { limit: 10 }, + }; - const generatedSql = await renderChartConfig(config, mockMetadata); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toMatchSnapshot(); - }); + const generatedSql = await renderChartConfig(config, mockMetadata); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toMatchSnapshot(); + }); - it('should generate a query without grouping but time bucketing', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Line, - connection: 'test-connection', - metricTables: { - gauge: 'otel_metrics_gauge', - histogram: 'otel_metrics_histogram', - sum: 'otel_metrics_sum', - summary: 'otel_metrics_summary', - 'exponential histogram': 'otel_metrics_exponential_histogram', - }, - from: { - databaseName: 'default', - tableName: '', - }, - select: [ - { - aggFn: 'quantile', - level: 0.5, - valueExpression: 'Value', - metricName: 'http.server.duration', - metricType: MetricsDataType.Histogram, + it('should generate a query without grouping but time bucketing', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', }, - ], - where: '', - whereLanguage: 'sql', - timestampValueExpression: 'TimeUnix', - dateRange: [new Date('2025-02-12'), new Date('2025-12-14')], - granularity: '2 minute', - limit: { limit: 10 }, - }; + from: { + databaseName: 'default', + tableName: '', + }, + select: [ + { + aggFn: 'quantile', + level: 0.5, + valueExpression: 'Value', + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + }, + ], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-02-12'), new Date('2025-12-14')], + granularity: '2 minute', + limit: { limit: 10 }, + }; - const generatedSql = await renderChartConfig(config, mockMetadata); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toMatchSnapshot(); + const generatedSql = await renderChartConfig(config, mockMetadata); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toMatchSnapshot(); + }); + + it('should generate a query with grouping and time bucketing', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }, + from: { + databaseName: 'default', + tableName: '', + }, + select: [ + { + aggFn: 'quantile', + level: 0.5, + valueExpression: 'Value', + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + }, + ], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-02-12'), new Date('2025-12-14')], + granularity: '2 minute', + groupBy: `ResourceAttributes['host']`, + limit: { limit: 10 }, + }; + + const generatedSql = await renderChartConfig(config, mockMetadata); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toMatchSnapshot(); + }); }); - it('should generate a query with grouping and time bucketing', async () => { - const config: ChartConfigWithOptDateRange = { - displayType: DisplayType.Line, - connection: 'test-connection', - metricTables: { - gauge: 'otel_metrics_gauge', - histogram: 'otel_metrics_histogram', - sum: 'otel_metrics_sum', - summary: 'otel_metrics_summary', - 'exponential histogram': 'otel_metrics_exponential_histogram', - }, - from: { - databaseName: 'default', - tableName: '', - }, - select: [ - { - aggFn: 'quantile', - level: 0.5, - valueExpression: 'Value', - metricName: 'http.server.duration', - metricType: MetricsDataType.Histogram, + describe('count', () => { + it('should generate a count query without grouping or time bucketing', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', }, - ], - where: '', - whereLanguage: 'sql', - timestampValueExpression: 'TimeUnix', - dateRange: [new Date('2025-02-12'), new Date('2025-12-14')], - granularity: '2 minute', - groupBy: `ResourceAttributes['host']`, - limit: { limit: 10 }, - }; + from: { + databaseName: 'default', + tableName: '', + }, + select: [ + { + aggFn: 'count', + valueExpression: 'Value', + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + }, + ], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-02-12'), new Date('2025-12-14')], + limit: { limit: 10 }, + }; - const generatedSql = await renderChartConfig(config, mockMetadata); - const actual = parameterizedQueryToSql(generatedSql); - expect(actual).toMatchSnapshot(); + const generatedSql = await renderChartConfig(config, mockMetadata); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toMatchSnapshot(); + }); + + it('should generate a count query without grouping but time bucketing', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }, + from: { + databaseName: 'default', + tableName: '', + }, + select: [ + { + aggFn: 'count', + valueExpression: 'Value', + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + }, + ], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-02-12'), new Date('2025-12-14')], + granularity: '2 minute', + limit: { limit: 10 }, + }; + + const generatedSql = await renderChartConfig(config, mockMetadata); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toMatchSnapshot(); + }); + + it('should generate a count query with grouping and time bucketing', async () => { + const config: ChartConfigWithOptDateRange = { + displayType: DisplayType.Line, + connection: 'test-connection', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }, + from: { + databaseName: 'default', + tableName: '', + }, + select: [ + { + aggFn: 'count', + valueExpression: 'Value', + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + }, + ], + where: '', + whereLanguage: 'sql', + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-02-12'), new Date('2025-12-14')], + granularity: '2 minute', + groupBy: `ResourceAttributes['host']`, + limit: { limit: 10 }, + }; + + const generatedSql = await renderChartConfig(config, mockMetadata); + const actual = parameterizedQueryToSql(generatedSql); + expect(actual).toMatchSnapshot(); + }); }); }); diff --git a/packages/common-utils/src/core/histogram.ts b/packages/common-utils/src/core/histogram.ts new file mode 100644 index 000000000..6d9ab6afe --- /dev/null +++ b/packages/common-utils/src/core/histogram.ts @@ -0,0 +1,190 @@ +import { ChSql, chSql } from '@/clickhouse'; +import { ChartConfig } from '@/types'; + +type WithClauses = ChartConfig['with']; +type TemplatedInput = ChSql | string; + +export const translateHistogram = ({ + select, + ...rest +}: { + select: Exclude[number]; + timeBucketSelect: TemplatedInput; + groupBy?: TemplatedInput; + from: TemplatedInput; + where: TemplatedInput; + valueAlias: TemplatedInput; +}) => { + if (select.aggFn === 'quantile') { + if (!('level' in select) || select.level === null) + throw new Error('quantile must have a level'); + return translateHistogramQuantile({ + ...rest, + level: select.level, + }); + } + if (select.aggFn === 'count') { + return translateHistogramCount(rest); + } + throw new Error(`${select.aggFn} is not supported for histograms currently`); +}; + +const translateHistogramCount = ({ + timeBucketSelect, + groupBy, + from, + where, + valueAlias, +}: { + timeBucketSelect: TemplatedInput; + groupBy?: TemplatedInput; + from: TemplatedInput; + where: TemplatedInput; + valueAlias: TemplatedInput; +}): WithClauses => [ + { + name: 'grouped_data', + sql: chSql` + SELECT + ${timeBucketSelect}, + ${groupBy ? chSql`[${groupBy}] as group,` : ''} + toInt64(sum(Count)) as cumulative_count, + AggregationTemporality + FROM ${from} + WHERE ${where} + GROUP BY ${groupBy ? 'group, ' : ''}\`__hdx_time_bucket\`, AggregationTemporality + `, + }, + { + name: 'metrics', + sql: chSql` + SELECT + \`__hdx_time_bucket\`, + ${groupBy ? 'group,' : ''} + CASE + WHEN AggregationTemporality = 1 THEN cumulative_count + WHEN AggregationTemporality = 2 THEN + GREATEST(0, + cumulative_count - COALESCE( + lag(cumulative_count) OVER ( + PARTITION BY ${groupBy ? 'group, ' : ''}AggregationTemporality + ORDER BY \`__hdx_time_bucket\` + ), + 0 + ) + ) + ELSE 0 + END as "${valueAlias}" + FROM grouped_data + `, + }, +]; + +const translateHistogramQuantile = ({ + timeBucketSelect, + groupBy, + from, + where, + valueAlias, + level, +}: { + timeBucketSelect: TemplatedInput; + groupBy?: TemplatedInput; + from: TemplatedInput; + where: TemplatedInput; + valueAlias: TemplatedInput; + level: number; +}): WithClauses => [ + { + name: 'source', + sql: chSql` + SELECT + MetricName, + ExplicitBounds, + ${timeBucketSelect}, + ${groupBy ? chSql`[${groupBy}] as group,` : ''} + sumForEach(deltas) as rates + FROM ( + SELECT + TimeUnix, + MetricName, + ResourceAttributes, + Attributes, + ExplicitBounds, + attr_hash, + any(attr_hash) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_attr_hash, + any(bounds_hash) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_bounds_hash, + any(counts) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_counts, + counts, + IF( + AggregationTemporality = 1 ${'' /* denotes a metric that is not monotonic e.g. already a delta */} + OR prev_attr_hash != attr_hash ${'' /* the attributes have changed so this is a different metric */} + OR bounds_hash != prev_bounds_hash ${'' /* the bucketing has changed so should be treated as different metric */} + OR arrayExists((x) -> x.2 < x.1, arrayZip(prev_counts, counts)), ${'' /* a data point has gone down, probably a reset event */} + counts, + counts - prev_counts + ) AS deltas + FROM ( + SELECT + TimeUnix, + MetricName, + AggregationTemporality, + ExplicitBounds, + ResourceAttributes, + Attributes, + cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS attr_hash, + cityHash64(ExplicitBounds) AS bounds_hash, + CAST(BucketCounts AS Array(Int64)) counts + FROM ${from} + WHERE ${where} + ORDER BY attr_hash, TimeUnix ASC + ) + ) + GROUP BY \`__hdx_time_bucket\`, MetricName, ${groupBy ? 'group, ' : ''}ExplicitBounds + ORDER BY \`__hdx_time_bucket\` + `, + }, + { + name: 'points', + sql: chSql` + SELECT + \`__hdx_time_bucket\`, + MetricName, + ${groupBy ? 'group,' : ''} + arrayZipUnaligned(arrayCumSum(rates), ExplicitBounds) as point, + length(point) as n + FROM source + `, + }, + { + name: 'metrics', + sql: chSql` + SELECT + \`__hdx_time_bucket\`, + MetricName, + ${groupBy ? 'group,' : ''} + point[n].1 AS total, + ${{ Float64: level }} * total AS rank, + arrayFirstIndex(x -> if(x.1 > rank, 1, 0), point) AS upper_idx, + point[upper_idx].1 AS upper_count, + ifNull(point[upper_idx].2, inf) AS upper_bound, + CASE + WHEN upper_idx > 1 THEN point[upper_idx - 1].2 + WHEN point[upper_idx].2 > 0 THEN 0 + ELSE inf + END AS lower_bound, + if ( + lower_bound = 0, + 0, + point[upper_idx - 1].1 + ) AS lower_count, + CASE + WHEN upper_bound = inf THEN point[upper_idx - 1].2 + WHEN lower_bound = inf THEN point[1].2 + ELSE lower_bound + (upper_bound - lower_bound) * ((rank - lower_count) / (upper_count - lower_count)) + END AS "${valueAlias}" + FROM points + WHERE length(point) > 1 AND total > 0 + `, + }, +]; diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index fb7c90382..fa2cc0d97 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -3,23 +3,8 @@ import * as SQLParser from 'node-sql-parser'; import SqlString from 'sqlstring'; import { ChSql, chSql, concatChSql, wrapChSqlIfNotEmpty } from '@/clickhouse'; +import { translateHistogram } from '@/core/histogram'; import { Metadata } from '@/core/metadata'; -import { CustomSchemaSQLSerializerV2, SearchQueryBuilder } from '@/queryParser'; - -/** - * Helper function to create a MetricName filter condition. - * Uses metricNameSql if available (which handles both old and new metric names via OR), - * otherwise falls back to a simple equality check. - */ -function createMetricNameFilter( - metricName: string, - metricNameSql?: string, -): string { - if (metricNameSql) { - return metricNameSql; - } - return SqlString.format('MetricName = ?', [metricName]); -} import { convertDateRangeToGranularityString, convertGranularityToSeconds, @@ -28,6 +13,7 @@ import { parseToStartOfFunction, splitAndTrimWithBracket, } from '@/core/utils'; +import { CustomSchemaSQLSerializerV2, SearchQueryBuilder } from '@/queryParser'; import { AggregateFunction, AggregateFunctionWithCombinators, @@ -47,6 +33,21 @@ import { SQLInterval, } from '@/types'; +/** + * Helper function to create a MetricName filter condition. + * Uses metricNameSql if available (which handles both old and new metric names via OR), + * otherwise falls back to a simple equality check. + */ +function createMetricNameFilter( + metricName: string, + metricNameSql?: string, +): string { + if (metricNameSql) { + return metricNameSql; + } + return SqlString.format('MetricName = ?', [metricName]); +} + /** The default maximum number of buckets setting when determining a bucket duration for 'auto' granularity */ export const DEFAULT_AUTO_GRANULARITY_MAX_BUCKETS = 60; @@ -1244,17 +1245,7 @@ async function translateMetricChartConfig( timestampValueExpression: `\`${timeBucketCol}\``, }; } else if (metricType === MetricsDataType.Histogram && metricName) { - // histograms are only valid for quantile selections - const { aggFn, level, alias, ..._selectRest } = _select as { - aggFn: string; - level?: number; - alias?: string; - }; - - if (aggFn !== 'quantile' || level == null) { - throw new Error('quantile must be specified for histogram metrics'); - } - + const { alias } = _select; // Use the alias from the select, defaulting to 'Value' for backwards compatibility const valueAlias = alias || 'Value'; @@ -1305,100 +1296,21 @@ async function translateMetricChartConfig( return { ...restChartConfig, - with: [ - { - name: 'source', - sql: chSql` - SELECT - MetricName, - ExplicitBounds, - ${timeBucketSelect.sql ? chSql`${timeBucketSelect},` : 'TimeUnix AS `__hdx_time_bucket`'} - ${groupBy ? chSql`[${groupBy}] as group,` : ''} - sumForEach(deltas) as rates - FROM ( - SELECT - TimeUnix, - MetricName, - ResourceAttributes, - Attributes, - ExplicitBounds, - attr_hash, - any(attr_hash) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_attr_hash, - any(bounds_hash) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_bounds_hash, - any(counts) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_counts, - counts, - IF( - AggregationTemporality = 1 ${'' /* denotes a metric that is not monotonic e.g. already a delta */} - OR prev_attr_hash != attr_hash ${'' /* the attributes have changed so this is a different metric */} - OR bounds_hash != prev_bounds_hash ${'' /* the bucketing has changed so should be treated as different metric */} - OR arrayExists((x) -> x.2 < x.1, arrayZip(prev_counts, counts)), ${'' /* a data point has gone down, probably a reset event */} - counts, - counts - prev_counts - ) AS deltas - FROM ( - SELECT - TimeUnix, - MetricName, - AggregationTemporality, - ExplicitBounds, - ResourceAttributes, - Attributes, - cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS attr_hash, - cityHash64(ExplicitBounds) AS bounds_hash, - CAST(BucketCounts AS Array(Int64)) counts - FROM ${renderFrom({ from: { ...from, tableName: metricTables[MetricsDataType.Histogram] } })} - WHERE ${where} - ORDER BY attr_hash, TimeUnix ASC - ) - ) - GROUP BY \`__hdx_time_bucket\`, MetricName, ${groupBy ? 'group, ' : ''}ExplicitBounds - ORDER BY \`__hdx_time_bucket\` - `, - }, - { - name: 'points', - sql: chSql` - SELECT - \`__hdx_time_bucket\`, - MetricName, - ${groupBy ? 'group,' : ''} - arrayZipUnaligned(arrayCumSum(rates), ExplicitBounds) as point, - length(point) as n - FROM source - `, - }, - { - name: 'metrics', - sql: chSql` - SELECT - \`__hdx_time_bucket\`, - MetricName, - ${groupBy ? 'group,' : ''} - point[n].1 AS total, - ${{ Float64: level }} * total AS rank, - arrayFirstIndex(x -> if(x.1 > rank, 1, 0), point) AS upper_idx, - point[upper_idx].1 AS upper_count, - ifNull(point[upper_idx].2, inf) AS upper_bound, - CASE - WHEN upper_idx > 1 THEN point[upper_idx - 1].2 - WHEN point[upper_idx].2 > 0 THEN 0 - ELSE inf - END AS lower_bound, - if ( - lower_bound = 0, - 0, - point[upper_idx - 1].1 - ) AS lower_count, - CASE - WHEN upper_bound = inf THEN point[upper_idx - 1].2 - WHEN lower_bound = inf THEN point[1].2 - ELSE lower_bound + (upper_bound - lower_bound) * ((rank - lower_count) / (upper_count - lower_count)) - END AS "${valueAlias}" - FROM points - WHERE length(point) > 1 AND total > 0 - `, - }, - ], + with: translateHistogram({ + select: _select, + timeBucketSelect: timeBucketSelect.sql + ? chSql`${timeBucketSelect}` + : 'TimeUnix AS `__hdx_time_bucket`', + groupBy, + from: renderFrom({ + from: { + ...from, + tableName: metricTables[MetricsDataType.Histogram], + }, + }), + where, + valueAlias, + }), select: `\`__hdx_time_bucket\`${groupBy ? ', group' : ''}, "${valueAlias}"`, from: { databaseName: '',