fix: filter logs with ID field

Author: karl-power

docker/otel-collector/schema/seed/00002_otel_logs.sql

@@ -25,6 +25,7 @@ CREATE TABLE IF NOT EXISTS ${DATABASE}.otel_logs
   `__hdx_materialized_k8s.pod.name` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.name'] CODEC(ZSTD(1)),
   `__hdx_materialized_k8s.pod.uid` LowCardinality(String) MATERIALIZED ResourceAttributes['k8s.pod.uid'] CODEC(ZSTD(1)),
   `__hdx_materialized_deployment.environment.name` LowCardinality(String) MATERIALIZED ResourceAttributes['deployment.environment.name'] CODEC(ZSTD(1)),
+  `__hdx_id` UInt16 MATERIALIZED toUInt16(rand()) CODEC(ZSTD(1)),
   INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
   INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
   INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,

packages/api/migrations/ch/000003_add_hdx_id_to_otel_logs.down.sql

@@ -0,0 +1 @@
+ALTER TABLE ${DATABASE}.otel_logs DROP COLUMN IF EXISTS `__hdx_id`;

packages/api/migrations/ch/000003_add_hdx_id_to_otel_logs.up.sql

@@ -0,0 +1,4 @@
+ALTER TABLE ${DATABASE}.otel_logs
+    ADD COLUMN IF NOT EXISTS `__hdx_id` UInt16
+    MATERIALIZED toUInt16(rand())
+    CODEC(ZSTD(1));

packages/app/src/components/DBRowTable.tsx

@@ -76,7 +76,7 @@ import {
   useRenderedSqlChartConfig,
 } from '@/hooks/useChartConfig';
 import { useCsvExport } from '@/hooks/useCsvExport';
-import { useTableMetadata } from '@/hooks/useMetadata';
+import { useColumns, useTableMetadata } from '@/hooks/useMetadata';
 import useOffsetPaginatedQuery from '@/hooks/useOffsetPaginatedQuery';
 import { useGroupedPatterns } from '@/hooks/usePatterns';
 import useRowWhere, {
@@ -1374,14 +1374,15 @@ export const RawLogTable = memo(
   },
 );
 
-export function appendSelectWithPrimaryAndPartitionKey(
+export function appendSelectWithAdditionalKeys(
   select: SelectList,
   primaryKeys: string,
   partitionKey: string,
+  extraKeys: string[],
 ): { select: SelectList; additionalKeysLength: number } {
   const partitionKeyArr = extractColumnReferencesFromKey(partitionKey);
   const primaryKeyArr = extractColumnReferencesFromKey(primaryKeys);
-  const allKeys = new Set([...partitionKeyArr, ...primaryKeyArr]);
+  const allKeys = new Set([...partitionKeyArr, ...primaryKeyArr, ...extraKeys]);
   if (typeof select === 'string') {
     const selectSplit = splitAndTrimWithBracket(select);
     const selectColumns = new Set(selectSplit);
@@ -1407,7 +1408,12 @@ function getSelectLength(select: SelectList): number {
   }
 }
 
-export function useConfigWithPrimaryAndPartitionKey(
+// Column name used to disambiguate rows with identical visible column values.
+// This is a MATERIALIZED column (random UInt16) added to otel_logs to ensure
+// the WHERE clause generated for row detail queries uniquely identifies a row.
+const HDX_ID_COLUMN = '__hdx_id';
+
+export function useConfigWithAdditionalSelect(
   config: BuilderChartConfigWithDateRange,
 ) {
   const { data: tableMetadata } = useTableMetadata({
@@ -1416,6 +1422,12 @@ export function useConfigWithPrimaryAndPartitionKey(
     connectionId: config.connection,
   });
 
+  const { data: columns } = useColumns({
+    databaseName: config.from.databaseName,
+    tableName: config.from.tableName,
+    connectionId: config.connection,
+  });
+
   const primaryKey = tableMetadata?.primary_key;
   const partitionKey = tableMetadata?.partition_key;
 
@@ -1424,14 +1436,17 @@ export function useConfigWithPrimaryAndPartitionKey(
       return undefined;
     }
 
-    const { select, additionalKeysLength } =
-      appendSelectWithPrimaryAndPartitionKey(
-        config.select,
-        primaryKey,
-        partitionKey,
-      );
+    const hasHdxIdColumn =
+      columns?.some(c => c.name === HDX_ID_COLUMN) ?? false;
+
+    const { select, additionalKeysLength } = appendSelectWithAdditionalKeys(
+      config.select,
+      primaryKey,
+      partitionKey,
+      hasHdxIdColumn ? [HDX_ID_COLUMN] : [],
+    );
     return { ...config, select, additionalKeysLength };
-  }, [primaryKey, partitionKey, config]);
+  }, [primaryKey, partitionKey, config, columns]);
 
   return mergedConfig;
 }
@@ -1564,7 +1579,7 @@ function DBSqlRowTableComponent({
     return base;
   }, [me, config, orderByArray]);
 
-  const mergedConfig = useConfigWithPrimaryAndPartitionKey(mergedConfigObj);
+  const mergedConfig = useConfigWithAdditionalSelect(mergedConfigObj);
 
   const { data, fetchNextPage, hasNextPage, isFetching, isError, error } =
     useOffsetPaginatedQuery(mergedConfig ?? config, {

packages/app/src/hooks/usePatterns.tsx

@@ -5,7 +5,7 @@ import { BuilderChartConfigWithDateRange } from '@hyperdx/common-utils/dist/type
 import { useQuery } from '@tanstack/react-query';
 
 import { timeBucketByGranularity, toStartOfInterval } from '@/ChartUtils';
-import { useConfigWithPrimaryAndPartitionKey } from '@/components/DBRowTable';
+import { useConfigWithAdditionalSelect } from '@/components/DBRowTable';
 import { useQueriedChartConfig } from '@/hooks/useChartConfig';
 import { getFirstTimestampValueExpression } from '@/source';
 
@@ -134,7 +134,7 @@ function usePatterns({
   statusCodeExpression?: string;
   enabled?: boolean;
 }) {
-  const configWithPrimaryAndPartitionKey = useConfigWithPrimaryAndPartitionKey({
+  const configWithPrimaryAndPartitionKey = useConfigWithAdditionalSelect({
     ...config,
     // TODO: User-configurable pattern columns and non-pattern/group by columns
     select: [