diff --git a/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap b/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap index 32e829056a..cdca6e3282 100644 --- a/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap +++ b/packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap @@ -22,6 +22,200 @@ exports[`renderChartConfig HAVING clause should render HAVING clause with multip ),count(),endpoint FROM default.metrics WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY endpoint HAVING avg(response_time) > 500 AND count(*) > 10 SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" `; +exports[`renderChartConfig JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED) should use toJSONString-based hash for gauge metric when Attributes column is JSON type 1`] = ` +"WITH Source AS ( + SELECT + *, + cityHash64(toJSONString(ScopeAttributes), toJSONString(ResourceAttributes), toJSONString(Attributes)) AS AttributesHash + FROM default.otel_metrics_gauge + WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND ((MetricName = 'system.cpu.utilization')) + ),Bucketed AS ( + SELECT + toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS \`__hdx_time_bucket2\`, + AttributesHash, + last_value(Value) AS LastValue, + any(ScopeAttributes) AS ScopeAttributes, + any(ResourceAttributes) AS ResourceAttributes, + any(Attributes) AS Attributes, + any(ResourceSchemaUrl) AS ResourceSchemaUrl, + any(ScopeName) AS ScopeName, + any(ScopeVersion) AS ScopeVersion, + any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount, + any(ScopeSchemaUrl) AS ScopeSchemaUrl, + any(ServiceName) AS ServiceName, + any(MetricDescription) AS MetricDescription, + any(MetricUnit) AS MetricUnit, + any(StartTimeUnix) AS StartTimeUnix, + any(Flags) AS Flags + FROM Source + GROUP BY AttributesHash, __hdx_time_bucket2 + ORDER BY AttributesHash, __hdx_time_bucket2 + ) SELECT avg( + toFloat64OrDefault(toString(LastValue)) + ),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" +`; + +exports[`renderChartConfig JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED) should use toJSONString-based hash for histogram (count) metric when Attributes column is JSON type 1`] = ` +"WITH source AS ( + SELECT + TimeUnix, + AggregationTemporality, + toStartOfInterval(toDateTime(TimeUnix), INTERVAL 2 minute) AS \`__hdx_time_bucket\`, + + cityHash64(toJSONString(ScopeAttributes), toJSONString(ResourceAttributes), toJSONString(Attributes)) AS attr_hash, + cityHash64(ExplicitBounds) AS bounds_hash, + toInt64(Count) AS current_count, + lagInFrame(toNullable(current_count), 1, NULL) OVER ( + PARTITION BY attr_hash, bounds_hash, AggregationTemporality + ORDER BY TimeUnix + ) AS prev_count, + CASE + WHEN AggregationTemporality = 1 THEN current_count + WHEN AggregationTemporality = 2 THEN greatest(0, current_count - coalesce(prev_count, 0)) + ELSE 0 + END AS delta + FROM default.otel_metrics_histogram + WHERE (TimeUnix >= toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 2 minute) - INTERVAL 2 minute AND TimeUnix <= toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 2 minute) + INTERVAL 2 minute) AND ((MetricName = 'http.server.request.count')) + ),metrics AS ( + SELECT + \`__hdx_time_bucket\`, + + sum(delta) AS \\"Value\\" + FROM source + GROUP BY \`__hdx_time_bucket\` + ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" +`; + +exports[`renderChartConfig JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED) should use toJSONString-based hash for histogram (quantile) metric when Attributes column is JSON type 1`] = ` +"WITH source AS ( + SELECT + MetricName, + ExplicitBounds, + toStartOfInterval(toDateTime(TimeUnix), INTERVAL 2 minute) AS \`__hdx_time_bucket\`, + + sumForEach(deltas) as rates + FROM ( + SELECT + TimeUnix, + MetricName, + ResourceAttributes, + Attributes, + ExplicitBounds, + attr_hash, + any(attr_hash) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_attr_hash, + any(bounds_hash) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_bounds_hash, + any(counts) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_counts, + counts, + IF( + AggregationTemporality = 1 + OR prev_attr_hash != attr_hash + OR bounds_hash != prev_bounds_hash + OR arrayExists((x) -> x.2 < x.1, arrayZip(prev_counts, counts)), + counts, + counts - prev_counts + ) AS deltas + FROM ( + SELECT + TimeUnix, + MetricName, + AggregationTemporality, + ExplicitBounds, + ResourceAttributes, + Attributes, + cityHash64(toJSONString(ScopeAttributes), toJSONString(ResourceAttributes), toJSONString(Attributes)) AS attr_hash, + cityHash64(ExplicitBounds) AS bounds_hash, + CAST(BucketCounts AS Array(Int64)) counts + FROM default.otel_metrics_histogram + WHERE (TimeUnix >= toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 2 minute) - INTERVAL 2 minute AND TimeUnix <= toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 2 minute) + INTERVAL 2 minute) AND ((MetricName = 'http.server.duration')) + ORDER BY attr_hash, TimeUnix ASC + ) + ) + GROUP BY \`__hdx_time_bucket\`, MetricName, ExplicitBounds + ORDER BY \`__hdx_time_bucket\` + ),points AS ( + SELECT + \`__hdx_time_bucket\`, + MetricName, + + arrayZipUnaligned(arrayCumSum(rates), ExplicitBounds) as point, + length(point) as n + FROM source + ),metrics AS ( + SELECT + \`__hdx_time_bucket\`, + MetricName, + + point[n].1 AS total, + 0.95 * total AS rank, + arrayFirstIndex(x -> if(x.1 > rank, 1, 0), point) AS upper_idx, + point[upper_idx].1 AS upper_count, + ifNull(point[upper_idx].2, inf) AS upper_bound, + CASE + WHEN upper_idx > 1 THEN point[upper_idx - 1].2 + WHEN point[upper_idx].2 > 0 THEN 0 + ELSE inf + END AS lower_bound, + if ( + lower_bound = 0, + 0, + point[upper_idx - 1].1 + ) AS lower_count, + CASE + WHEN upper_bound = inf THEN point[upper_idx - 1].2 + WHEN lower_bound = inf THEN point[1].2 + ELSE lower_bound + (upper_bound - lower_bound) * ((rank - lower_count) / (upper_count - lower_count)) + END AS \\"Value\\" + FROM points + WHERE length(point) > 1 AND total > 0 + ) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" +`; + +exports[`renderChartConfig JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED) should use toJSONString-based hash for sum metric when Attributes column is JSON type 1`] = ` +"WITH Source AS ( + SELECT + *, + cityHash64(toJSONString(ScopeAttributes), toJSONString(ResourceAttributes), toJSONString(Attributes)) AS AttributesHash, + IF(AggregationTemporality = 1, + SUM(Value) OVER (PARTITION BY AttributesHash ORDER BY AttributesHash, TimeUnix ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), + IF(IsMonotonic = 0, + Value, + deltaSum(Value) OVER (PARTITION BY AttributesHash ORDER BY AttributesHash, TimeUnix ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + ) + ) AS Rate, + IF(AggregationTemporality = 1, Rate, Value) AS Sum + FROM default.otel_metrics_sum + WHERE (TimeUnix >= toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 5 minute) - INTERVAL 5 minute AND TimeUnix <= toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 5 minute) + INTERVAL 5 minute) AND ((MetricName = 'db.client.connections.usage'))),Bucketed AS ( + SELECT + toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minute) AS \`__hdx_time_bucket2\`, + AttributesHash, + last_value(Source.Rate) AS \`__hdx_value_high\`, + any(\`__hdx_value_high\`) OVER(PARTITION BY AttributesHash ORDER BY \`__hdx_time_bucket2\` ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS \`__hdx_value_high_prev\`, + IF(IsMonotonic = 1, \`__hdx_value_high\` - \`__hdx_value_high_prev\`, \`__hdx_value_high\`) AS Rate, + last_value(Source.Sum) AS Sum, + any(ResourceAttributes) AS ResourceAttributes, + any(ResourceSchemaUrl) AS ResourceSchemaUrl, + any(ScopeName) AS ScopeName, + any(ScopeVersion) AS ScopeVersion, + any(ScopeAttributes) AS ScopeAttributes, + any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount, + any(ScopeSchemaUrl) AS ScopeSchemaUrl, + any(ServiceName) AS ServiceName, + any(MetricName) AS MetricName, + any(MetricDescription) AS MetricDescription, + any(MetricUnit) AS MetricUnit, + any(Attributes) AS Attributes, + any(StartTimeUnix) AS StartTimeUnix, + any(Flags) AS Flags, + any(AggregationTemporality) AS AggregationTemporality, + any(IsMonotonic) AS IsMonotonic + FROM Source + GROUP BY AttributesHash, \`__hdx_time_bucket2\` + ORDER BY AttributesHash, \`__hdx_time_bucket2\` + ) SELECT avg( + toFloat64OrDefault(toString(Rate)) + ) AS \\"Value\\",toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (\`__hdx_time_bucket2\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket2\` <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000" +`; + exports[`renderChartConfig SETTINGS clause should apply the "chart config" settings to the query 1`] = `"SELECT histogramMerge(20)(Duration),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; exports[`renderChartConfig SETTINGS clause should apply the "query settings" settings to the query 1`] = `"SELECT histogramMerge(20)(Duration),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`; diff --git a/packages/common-utils/src/__tests__/renderChartConfig.test.ts b/packages/common-utils/src/__tests__/renderChartConfig.test.ts index 6665180850..73aa3c534c 100644 --- a/packages/common-utils/src/__tests__/renderChartConfig.test.ts +++ b/packages/common-utils/src/__tests__/renderChartConfig.test.ts @@ -2398,4 +2398,207 @@ describe('renderChartConfig', () => { expect(actual).not.toContain('SampleRate'); }); }); + + describe('JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED)', () => { + // When the ClickHouse exporter uses json: true, attribute columns are JSON + // type instead of Map(String, String). mapConcat() fails on JSON columns, so + // the query generator must fall back to toJSONString()-based hashing. + + let jsonSchemaMockMetadata: jest.Mocked; + + beforeEach(() => { + const jsonSchemaColumns = [ + { name: 'TimeUnix', type: 'DateTime64(9)' }, + { name: 'MetricName', type: 'LowCardinality(String)' }, + { name: 'Attributes', type: 'JSON' }, + { name: 'ScopeAttributes', type: 'JSON' }, + { name: 'ResourceAttributes', type: 'JSON' }, + { name: 'Value', type: 'Float64' }, + { name: 'AggregationTemporality', type: 'Int32' }, + ]; + jsonSchemaMockMetadata = { + getColumns: jest.fn().mockResolvedValue(jsonSchemaColumns), + getMaterializedColumnsLookupTable: jest.fn().mockResolvedValue(null), + getColumn: jest + .fn() + .mockImplementation(async ({ column }: { column: string }) => + jsonSchemaColumns.find(col => col.name === column), + ), + getTableMetadata: jest + .fn() + .mockResolvedValue({ primary_key: 'TimeUnix' }), + getSkipIndices: jest.fn().mockResolvedValue([]), + getSetting: jest.fn().mockResolvedValue(undefined), + } as unknown as jest.Mocked; + }); + + const baseMetricConfig = { + displayType: DisplayType.Line, + connection: 'test-connection', + metricTables: { + gauge: 'otel_metrics_gauge', + histogram: 'otel_metrics_histogram', + sum: 'otel_metrics_sum', + summary: 'otel_metrics_summary', + 'exponential histogram': 'otel_metrics_exponential_histogram', + }, + from: { + databaseName: 'default', + tableName: '', + }, + where: '', + whereLanguage: 'sql' as const, + timestampValueExpression: 'TimeUnix', + dateRange: [new Date('2025-02-12'), new Date('2025-12-14')] as [ + Date, + Date, + ], + granularity: '1 minute' as const, + limit: { limit: 10 }, + }; + + it('should use toJSONString-based hash for gauge metric when Attributes column is JSON type', async () => { + const config: ChartConfigWithOptDateRange = { + ...baseMetricConfig, + select: [ + { + aggFn: 'avg', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + metricName: 'system.cpu.utilization', + metricType: MetricsDataType.Gauge, + }, + ], + }; + + const generatedSql = await renderChartConfig( + config, + jsonSchemaMockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + + expect(actual).toContain('toJSONString(ScopeAttributes)'); + expect(actual).toContain('toJSONString(ResourceAttributes)'); + expect(actual).toContain('toJSONString(Attributes)'); + expect(actual).not.toContain('mapConcat'); + expect(actual).toMatchSnapshot(); + }); + + it('should use toJSONString-based hash for sum metric when Attributes column is JSON type', async () => { + const config: ChartConfigWithOptDateRange = { + ...baseMetricConfig, + granularity: '5 minute', + select: [ + { + aggFn: 'avg', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + metricName: 'db.client.connections.usage', + metricType: MetricsDataType.Sum, + }, + ], + }; + + const generatedSql = await renderChartConfig( + config, + jsonSchemaMockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + + expect(actual).toContain('toJSONString(ScopeAttributes)'); + expect(actual).toContain('toJSONString(ResourceAttributes)'); + expect(actual).toContain('toJSONString(Attributes)'); + expect(actual).not.toContain('mapConcat'); + expect(actual).toMatchSnapshot(); + }); + + it('should use toJSONString-based hash for histogram (quantile) metric when Attributes column is JSON type', async () => { + const config: ChartConfigWithOptDateRange = { + ...baseMetricConfig, + granularity: '2 minute', + select: [ + { + aggFn: 'quantile', + level: 0.95, + valueExpression: 'Value', + metricName: 'http.server.duration', + metricType: MetricsDataType.Histogram, + }, + ], + }; + + const generatedSql = await renderChartConfig( + config, + jsonSchemaMockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + + expect(actual).toContain('toJSONString(ScopeAttributes)'); + expect(actual).toContain('toJSONString(ResourceAttributes)'); + expect(actual).toContain('toJSONString(Attributes)'); + expect(actual).not.toContain('mapConcat'); + expect(actual).toMatchSnapshot(); + }); + + it('should use toJSONString-based hash for histogram (count) metric when Attributes column is JSON type', async () => { + const config: ChartConfigWithOptDateRange = { + ...baseMetricConfig, + granularity: '2 minute', + select: [ + { + aggFn: 'count', + valueExpression: 'Value', + metricName: 'http.server.request.count', + metricType: MetricsDataType.Histogram, + }, + ], + }; + + const generatedSql = await renderChartConfig( + config, + jsonSchemaMockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + + expect(actual).toContain('toJSONString(ScopeAttributes)'); + expect(actual).toContain('toJSONString(ResourceAttributes)'); + expect(actual).toContain('toJSONString(Attributes)'); + expect(actual).not.toContain('mapConcat'); + expect(actual).toMatchSnapshot(); + }); + + it('should still use mapConcat when Attributes column is Map type (non-JSON schema)', async () => { + // Verify existing Map-schema behaviour is unchanged + const config: ChartConfigWithOptDateRange = { + ...baseMetricConfig, + select: [ + { + aggFn: 'avg', + aggCondition: '', + aggConditionLanguage: 'lucene', + valueExpression: 'Value', + metricName: 'system.cpu.utilization', + metricType: MetricsDataType.Gauge, + }, + ], + }; + + // mockMetadata returns Map-typed columns (default setup from beforeEach) + const generatedSql = await renderChartConfig( + config, + mockMetadata, + querySettings, + ); + const actual = parameterizedQueryToSql(generatedSql); + + expect(actual).toContain('mapConcat'); + expect(actual).not.toContain('toJSONString'); + }); + }); }); diff --git a/packages/common-utils/src/core/histogram.ts b/packages/common-utils/src/core/histogram.ts index d84453e8a1..d9bfabed71 100644 --- a/packages/common-utils/src/core/histogram.ts +++ b/packages/common-utils/src/core/histogram.ts @@ -4,8 +4,20 @@ import { BuilderChartConfig } from '@/types'; type WithClauses = BuilderChartConfig['with']; type TemplatedInput = ChSql | string; +// Returns the SQL expression for hashing metric attributes into a unique key. +// When using the JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED), the attribute +// columns are JSON type instead of Map(String, String), so mapConcat() would +// fail with "Function mapConcat requires at least one argument of type Map". +// In that case we fall back to hashing the JSON string representations instead. +export function attrHashExpr(isJsonSchema: boolean): string { + return isJsonSchema + ? 'cityHash64(toJSONString(ScopeAttributes), toJSONString(ResourceAttributes), toJSONString(Attributes))' + : 'cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes))'; +} + export const translateHistogram = ({ select, + isJsonSchema = false, ...rest }: { select: Exclude[number]; @@ -14,6 +26,7 @@ export const translateHistogram = ({ from: TemplatedInput; where: TemplatedInput; valueAlias: TemplatedInput; + isJsonSchema?: boolean; }) => { if (select.aggFn === 'quantile') { if (!('level' in select) || select.level === null) @@ -21,10 +34,11 @@ export const translateHistogram = ({ return translateHistogramQuantile({ ...rest, level: select.level, + isJsonSchema, }); } if (select.aggFn === 'count') { - return translateHistogramCount(rest); + return translateHistogramCount({ ...rest, isJsonSchema }); } throw new Error(`${select.aggFn} is not supported for histograms currently`); }; @@ -35,12 +49,14 @@ const translateHistogramCount = ({ from, where, valueAlias, + isJsonSchema = false, }: { timeBucketSelect: TemplatedInput; groupBy?: TemplatedInput; from: TemplatedInput; where: TemplatedInput; valueAlias: TemplatedInput; + isJsonSchema?: boolean; }): WithClauses => [ { name: 'source', @@ -50,7 +66,7 @@ const translateHistogramCount = ({ AggregationTemporality, ${timeBucketSelect}, ${groupBy ? chSql`[${groupBy}] AS group,` : ''} - cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS attr_hash, + ${attrHashExpr(isJsonSchema)} AS attr_hash, cityHash64(ExplicitBounds) AS bounds_hash, toInt64(Count) AS current_count, lagInFrame(toNullable(current_count), 1, NULL) OVER ( @@ -86,6 +102,7 @@ const translateHistogramQuantile = ({ where, valueAlias, level, + isJsonSchema = false, }: { timeBucketSelect: TemplatedInput; groupBy?: TemplatedInput; @@ -93,6 +110,7 @@ const translateHistogramQuantile = ({ where: TemplatedInput; valueAlias: TemplatedInput; level: number; + isJsonSchema?: boolean; }): WithClauses => [ { name: 'source', @@ -131,7 +149,7 @@ const translateHistogramQuantile = ({ ExplicitBounds, ResourceAttributes, Attributes, - cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS attr_hash, + ${attrHashExpr(isJsonSchema)} AS attr_hash, cityHash64(ExplicitBounds) AS bounds_hash, CAST(BucketCounts AS Array(Int64)) counts FROM ${from} diff --git a/packages/common-utils/src/core/renderChartConfig.ts b/packages/common-utils/src/core/renderChartConfig.ts index 72b24d5d5e..fa5c1eaaf4 100644 --- a/packages/common-utils/src/core/renderChartConfig.ts +++ b/packages/common-utils/src/core/renderChartConfig.ts @@ -2,8 +2,15 @@ import isPlainObject from 'lodash/isPlainObject'; import * as SQLParser from 'node-sql-parser'; import SqlString from 'sqlstring'; -import { ChSql, chSql, concatChSql, wrapChSqlIfNotEmpty } from '@/clickhouse'; -import { translateHistogram } from '@/core/histogram'; +import { + ChSql, + chSql, + concatChSql, + convertCHDataTypeToJSType, + JSDataType, + wrapChSqlIfNotEmpty, +} from '@/clickhouse'; +import { attrHashExpr, translateHistogram } from '@/core/histogram'; import { Metadata } from '@/core/metadata'; import { convertDateRangeToGranularityString, @@ -1213,6 +1220,43 @@ async function translateMetricChartConfig( } const { metricType, metricName, metricNameSql, ..._select } = select[0]; // Initial impl only supports one metric select per chart config + + // Detect whether the metric tables use the JSON schema + // (BETA_CH_OTEL_JSON_SCHEMA_ENABLED). When enabled, attribute columns + // (Attributes, ScopeAttributes, ResourceAttributes) are JSON type instead of + // Map(String, String), which means mapConcat() cannot be used. We detect this + // by inspecting the actual column type in ClickHouse. + let isJsonSchema = false; + const detectionTableName = + (metricType != null ? metricTables[metricType] : undefined) ?? + metricTables[MetricsDataType.Gauge] ?? + metricTables[MetricsDataType.Sum] ?? + metricTables[MetricsDataType.Histogram]; + if (detectionTableName && from.databaseName && chartConfig.connection) { + try { + const columns = await metadata.getColumns({ + databaseName: from.databaseName, + tableName: detectionTableName, + connectionId: chartConfig.connection, + }); + // We only check `Attributes` as a representative column — the OTel + // exporter sets all three attribute columns (Attributes, ScopeAttributes, + // ResourceAttributes) to the same type, so checking one is sufficient. + isJsonSchema = columns.some( + c => + c.name === 'Attributes' && + convertCHDataTypeToJSType(c.type) === JSDataType.JSON, + ); + } catch (e) { + // If column detection fails (e.g. table doesn't exist yet), fall back to + // the Map schema behaviour which was the original default. + console.warn( + 'Failed to detect metric table column types, falling back to Map schema', + e, + ); + } + } + if ( metricType === MetricsDataType.Gauge && metricName && @@ -1259,7 +1303,7 @@ async function translateMetricChartConfig( sql: chSql` SELECT *, - cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash + ${attrHashExpr(isJsonSchema)} AS AttributesHash FROM ${renderFrom({ from: { ...from, tableName: metricTables[MetricsDataType.Gauge] } })} WHERE ${where} `, @@ -1363,7 +1407,7 @@ async function translateMetricChartConfig( sql: chSql` SELECT *, - cityHash64(mapConcat(ScopeAttributes, ResourceAttributes, Attributes)) AS AttributesHash, + ${attrHashExpr(isJsonSchema)} AS AttributesHash, IF(AggregationTemporality = 1, SUM(Value) OVER (PARTITION BY AttributesHash ORDER BY AttributesHash, TimeUnix ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW), IF(IsMonotonic = 0, @@ -1501,6 +1545,7 @@ async function translateMetricChartConfig( }), where, valueAlias, + isJsonSchema, }), select: `\`__hdx_time_bucket\`${groupBy ? ', group' : ''}, "${valueAlias}"`, from: {