Skip to content

fix: use toJSONString for metric attribute hashing with JSON schema (#2087) #2118

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
doyong365 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: use toJSONString for metric attribute hashing with JSON schema (#2087) #2118

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
194 changes: 194 additions & 0 deletions packages/common-utils/src/__tests__/__snapshots__/renderChartConfig.test.ts.snap
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,200 @@ exports[`renderChartConfig HAVING clause should render HAVING clause with multip
),count(),endpoint FROM default.metrics WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY endpoint HAVING avg(response_time) > 500 AND count(*) > 10 SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"
`;

exports[`renderChartConfig JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED) should use toJSONString-based hash for gauge metric when Attributes column is JSON type 1`] = `
"WITH Source AS (
SELECT
*,
cityHash64(toJSONString(ScopeAttributes), toJSONString(ResourceAttributes), toJSONString(Attributes)) AS AttributesHash
FROM default.otel_metrics_gauge
WHERE (TimeUnix >= fromUnixTimestamp64Milli(1739318400000) AND TimeUnix <= fromUnixTimestamp64Milli(1765670400000)) AND ((MetricName = 'system.cpu.utilization'))
),Bucketed AS (
SELECT
toStartOfInterval(toDateTime(TimeUnix), INTERVAL 1 minute) AS \`__hdx_time_bucket2\`,
AttributesHash,
last_value(Value) AS LastValue,
any(ScopeAttributes) AS ScopeAttributes,
any(ResourceAttributes) AS ResourceAttributes,
any(Attributes) AS Attributes,
any(ResourceSchemaUrl) AS ResourceSchemaUrl,
any(ScopeName) AS ScopeName,
any(ScopeVersion) AS ScopeVersion,
any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount,
any(ScopeSchemaUrl) AS ScopeSchemaUrl,
any(ServiceName) AS ServiceName,
any(MetricDescription) AS MetricDescription,
any(MetricUnit) AS MetricUnit,
any(StartTimeUnix) AS StartTimeUnix,
any(Flags) AS Flags
FROM Source
GROUP BY AttributesHash, __hdx_time_bucket2
ORDER BY AttributesHash, __hdx_time_bucket2
) SELECT avg(
toFloat64OrDefault(toString(LastValue))
),toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (__hdx_time_bucket2 >= fromUnixTimestamp64Milli(1739318400000) AND __hdx_time_bucket2 <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(__hdx_time_bucket2), INTERVAL 1 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"
`;

exports[`renderChartConfig JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED) should use toJSONString-based hash for histogram (count) metric when Attributes column is JSON type 1`] = `
"WITH source AS (
SELECT
TimeUnix,
AggregationTemporality,
toStartOfInterval(toDateTime(TimeUnix), INTERVAL 2 minute) AS \`__hdx_time_bucket\`,

cityHash64(toJSONString(ScopeAttributes), toJSONString(ResourceAttributes), toJSONString(Attributes)) AS attr_hash,
cityHash64(ExplicitBounds) AS bounds_hash,
toInt64(Count) AS current_count,
lagInFrame(toNullable(current_count), 1, NULL) OVER (
PARTITION BY attr_hash, bounds_hash, AggregationTemporality
ORDER BY TimeUnix
) AS prev_count,
CASE
WHEN AggregationTemporality = 1 THEN current_count
WHEN AggregationTemporality = 2 THEN greatest(0, current_count - coalesce(prev_count, 0))
ELSE 0
END AS delta
FROM default.otel_metrics_histogram
WHERE (TimeUnix >= toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 2 minute) - INTERVAL 2 minute AND TimeUnix <= toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 2 minute) + INTERVAL 2 minute) AND ((MetricName = 'http.server.request.count'))
),metrics AS (
SELECT
\`__hdx_time_bucket\`,

sum(delta) AS \\"Value\\"
FROM source
GROUP BY \`__hdx_time_bucket\`
) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"
`;

exports[`renderChartConfig JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED) should use toJSONString-based hash for histogram (quantile) metric when Attributes column is JSON type 1`] = `
"WITH source AS (
SELECT
MetricName,
ExplicitBounds,
toStartOfInterval(toDateTime(TimeUnix), INTERVAL 2 minute) AS \`__hdx_time_bucket\`,

sumForEach(deltas) as rates
FROM (
SELECT
TimeUnix,
MetricName,
ResourceAttributes,
Attributes,
ExplicitBounds,
attr_hash,
any(attr_hash) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_attr_hash,
any(bounds_hash) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_bounds_hash,
any(counts) OVER (ROWS BETWEEN 1 preceding AND 1 preceding) AS prev_counts,
counts,
IF(
AggregationTemporality = 1
OR prev_attr_hash != attr_hash
OR bounds_hash != prev_bounds_hash
OR arrayExists((x) -> x.2 < x.1, arrayZip(prev_counts, counts)),
counts,
counts - prev_counts
) AS deltas
FROM (
SELECT
TimeUnix,
MetricName,
AggregationTemporality,
ExplicitBounds,
ResourceAttributes,
Attributes,
cityHash64(toJSONString(ScopeAttributes), toJSONString(ResourceAttributes), toJSONString(Attributes)) AS attr_hash,
cityHash64(ExplicitBounds) AS bounds_hash,
CAST(BucketCounts AS Array(Int64)) counts
FROM default.otel_metrics_histogram
WHERE (TimeUnix >= toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 2 minute) - INTERVAL 2 minute AND TimeUnix <= toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 2 minute) + INTERVAL 2 minute) AND ((MetricName = 'http.server.duration'))
ORDER BY attr_hash, TimeUnix ASC
)
)
GROUP BY \`__hdx_time_bucket\`, MetricName, ExplicitBounds
ORDER BY \`__hdx_time_bucket\`
),points AS (
SELECT
\`__hdx_time_bucket\`,
MetricName,

arrayZipUnaligned(arrayCumSum(rates), ExplicitBounds) as point,
length(point) as n
FROM source
),metrics AS (
SELECT
\`__hdx_time_bucket\`,
MetricName,

point[n].1 AS total,
0.95 * total AS rank,
arrayFirstIndex(x -> if(x.1 > rank, 1, 0), point) AS upper_idx,
point[upper_idx].1 AS upper_count,
ifNull(point[upper_idx].2, inf) AS upper_bound,
CASE
WHEN upper_idx > 1 THEN point[upper_idx - 1].2
WHEN point[upper_idx].2 > 0 THEN 0
ELSE inf
END AS lower_bound,
if (
lower_bound = 0,
0,
point[upper_idx - 1].1
) AS lower_count,
CASE
WHEN upper_bound = inf THEN point[upper_idx - 1].2
WHEN lower_bound = inf THEN point[1].2
ELSE lower_bound + (upper_bound - lower_bound) * ((rank - lower_count) / (upper_count - lower_count))
END AS \\"Value\\"
FROM points
WHERE length(point) > 1 AND total > 0
) SELECT \`__hdx_time_bucket\`, \\"Value\\" FROM metrics WHERE (\`__hdx_time_bucket\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket\` <= fromUnixTimestamp64Milli(1765670400000)) LIMIT 10 SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"
`;

exports[`renderChartConfig JSON schema (BETA_CH_OTEL_JSON_SCHEMA_ENABLED) should use toJSONString-based hash for sum metric when Attributes column is JSON type 1`] = `
"WITH Source AS (
SELECT
*,
cityHash64(toJSONString(ScopeAttributes), toJSONString(ResourceAttributes), toJSONString(Attributes)) AS AttributesHash,
IF(AggregationTemporality = 1,
SUM(Value) OVER (PARTITION BY AttributesHash ORDER BY AttributesHash, TimeUnix ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW),
IF(IsMonotonic = 0,
Value,
deltaSum(Value) OVER (PARTITION BY AttributesHash ORDER BY AttributesHash, TimeUnix ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)
)
) AS Rate,
IF(AggregationTemporality = 1, Rate, Value) AS Sum
FROM default.otel_metrics_sum
WHERE (TimeUnix >= toStartOfInterval(fromUnixTimestamp64Milli(1739318400000), INTERVAL 5 minute) - INTERVAL 5 minute AND TimeUnix <= toStartOfInterval(fromUnixTimestamp64Milli(1765670400000), INTERVAL 5 minute) + INTERVAL 5 minute) AND ((MetricName = 'db.client.connections.usage'))),Bucketed AS (
SELECT
toStartOfInterval(toDateTime(TimeUnix), INTERVAL 5 minute) AS \`__hdx_time_bucket2\`,
AttributesHash,
last_value(Source.Rate) AS \`__hdx_value_high\`,
any(\`__hdx_value_high\`) OVER(PARTITION BY AttributesHash ORDER BY \`__hdx_time_bucket2\` ROWS BETWEEN 1 PRECEDING AND 1 PRECEDING) AS \`__hdx_value_high_prev\`,
IF(IsMonotonic = 1, \`__hdx_value_high\` - \`__hdx_value_high_prev\`, \`__hdx_value_high\`) AS Rate,
last_value(Source.Sum) AS Sum,
any(ResourceAttributes) AS ResourceAttributes,
any(ResourceSchemaUrl) AS ResourceSchemaUrl,
any(ScopeName) AS ScopeName,
any(ScopeVersion) AS ScopeVersion,
any(ScopeAttributes) AS ScopeAttributes,
any(ScopeDroppedAttrCount) AS ScopeDroppedAttrCount,
any(ScopeSchemaUrl) AS ScopeSchemaUrl,
any(ServiceName) AS ServiceName,
any(MetricName) AS MetricName,
any(MetricDescription) AS MetricDescription,
any(MetricUnit) AS MetricUnit,
any(Attributes) AS Attributes,
any(StartTimeUnix) AS StartTimeUnix,
any(Flags) AS Flags,
any(AggregationTemporality) AS AggregationTemporality,
any(IsMonotonic) AS IsMonotonic
FROM Source
GROUP BY AttributesHash, \`__hdx_time_bucket2\`
ORDER BY AttributesHash, \`__hdx_time_bucket2\`
) SELECT avg(
toFloat64OrDefault(toString(Rate))
) AS \\"Value\\",toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` FROM Bucketed WHERE (\`__hdx_time_bucket2\` >= fromUnixTimestamp64Milli(1739318400000) AND \`__hdx_time_bucket2\` <= fromUnixTimestamp64Milli(1765670400000)) GROUP BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` ORDER BY toStartOfInterval(toDateTime(\`__hdx_time_bucket2\`), INTERVAL 5 minute) AS \`__hdx_time_bucket\` LIMIT 10 SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"
`;

exports[`renderChartConfig SETTINGS clause should apply the "chart config" settings to the query 1`] = `"SELECT histogramMerge(20)(Duration),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity SETTINGS short_circuit_function_evaluation = 'force_enable', optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`;

exports[`renderChartConfig SETTINGS clause should apply the "query settings" settings to the query 1`] = `"SELECT histogramMerge(20)(Duration),severity FROM default.logs WHERE (timestamp >= fromUnixTimestamp64Milli(1739318400000) AND timestamp <= fromUnixTimestamp64Milli(1739491200000)) GROUP BY severity SETTINGS optimize_read_in_order = 0, cast_keep_nullable = 1, additional_result_filter = 'x != 2', count_distinct_implementation = 'uniqCombined64', async_insert_busy_timeout_min_ms = 20000"`;
Expand Down
Loading
Loading