diff --git a/config/sample-config.yaml b/config/sample-config.yaml index b03ffb6a8..a32e60bce 100644 --- a/config/sample-config.yaml +++ b/config/sample-config.yaml @@ -697,11 +697,11 @@ frontend: filter: tls_cipher_suite width: 15 feature: tlsTracking - - id: TLSCurve + - id: TLSGroup group: Protocol Info - name: TLS Curve - field: TLSCurve - filter: tls_curve + name: TLS Group + field: TLSGroup + filter: tls_group width: 10 feature: tlsTracking - id: TLSTypes @@ -1411,11 +1411,11 @@ frontend: placeholder: 'E.g: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256' hint: Specify a TLS cipher suite. feature: tlsTracking - - id: tls_curve - name: TLS curve + - id: tls_group + name: TLS group component: text placeholder: 'E.g: X25519' - hint: Specify a TLS curve name. + hint: Specify a TLS group name. feature: tlsTracking - id: tls_types name: TLS packet type @@ -1837,9 +1837,9 @@ frontend: - name: TLSCipherSuite type: string description: TLS cipher suite - - name: TLSCurve + - name: TLSGroup type: string - description: TLS curve name + description: TLS group name - name: Dscp type: number description: Differentiated Services Code Point (DSCP) value diff --git a/pkg/loki/flow_query.go b/pkg/loki/flow_query.go index 777cc38a7..f6b2bd52b 100644 --- a/pkg/loki/flow_query.go +++ b/pkg/loki/flow_query.go @@ -126,25 +126,24 @@ func (q *FlowQueryBuilder) addLineFilters(filter filters.Match, values []string) return } - if q.config.IsArray(filter.Key) { - q.lineFilters = append(q.lineFilters, filters.ArrayLineFilter(filter.Key, values, filter.Not)) + var lf filters.LineFilter + var hasEmptyMatch bool + switch { + case q.config.IsArray(filter.Key): + lf, hasEmptyMatch = filters.ArrayLineFilter(filter.Key, values, filter.Not) + case q.config.IsNumeric(filter.Key): + lf, hasEmptyMatch = filters.NumericLineFilter(filter.Key, values, filter.Not, filter.MoreThanOrEqual) + case filter.Regex: + lf, hasEmptyMatch = filters.StringLineFilterCheckExact(filter.Key, values, filter.Not) + default: + lf, hasEmptyMatch = filters.StringLineFilter(filter.Key, values, filter.Not) + } + // if there is at least an empty exact match, there is no uniform/safe way to filter by text, + // so we should use JSON label matchers instead of text line matchers + if hasEmptyMatch { + q.jsonFilters = append(q.jsonFilters, lf.AsLabelFilters()) } else { - var lf filters.LineFilter - var hasEmptyMatch bool - if q.config.IsNumeric(filter.Key) { - lf, hasEmptyMatch = filters.NumericLineFilter(filter.Key, values, filter.Not, filter.MoreThanOrEqual) - } else if filter.Regex { - lf, hasEmptyMatch = filters.StringLineFilterCheckExact(filter.Key, values, filter.Not) - } else { - lf, hasEmptyMatch = filters.StringLineFilter(filter.Key, values, filter.Not) - } - // if there is at least an empty exact match, there is no uniform/safe way to filter by text, - // so we should use JSON label matchers instead of text line matchers - if hasEmptyMatch { - q.jsonFilters = append(q.jsonFilters, lf.AsLabelFilters()) - } else { - q.lineFilters = append(q.lineFilters, lf) - } + q.lineFilters = append(q.lineFilters, lf) } } @@ -225,6 +224,10 @@ func (q *FlowQueryBuilder) appendDNSFilter(sb *strings.Builder) { sb.WriteString("`") } +func (q *FlowQueryBuilder) appendTLSFilter(sb *strings.Builder) { + sb.WriteString(`|="TLSTypes"`) +} + func (q *FlowQueryBuilder) appendDNSLatencyFilter(sb *strings.Builder) { // ensure DnsLatencyMs field is specified and value is not zero // |~`"DnsLatencyMs`!~`DnsLatencyMs%22:0[,}]` diff --git a/pkg/loki/topology_query.go b/pkg/loki/topology_query.go index 7842725de..d4e0c5247 100644 --- a/pkg/loki/topology_query.go +++ b/pkg/loki/topology_query.go @@ -28,6 +28,15 @@ type TopologyInput struct { Groups string } +func (in *TopologyInput) GetActualDataField() string { + switch in.DataField { + case constants.MetricTypeFlows, constants.MetricTypeDNSFlows, constants.MetricTypeTLSFlows: + return "" + default: + return in.DataField + } +} + type TopologyQueryBuilder struct { *FlowQueryBuilder topology *TopologyInput @@ -71,15 +80,6 @@ func GetLabelsAndFilter(kl map[string][]string, aggregate, groups string) ([]str return fields, filter } -func getField(metricType string) string { - switch metricType { - case constants.MetricTypeFlows, constants.MetricTypeDNSFlows: - return "" - default: - return metricType - } -} - func getFactor(metricType string) string { switch metricType { case constants.MetricTypeFlowRTT: @@ -124,7 +124,6 @@ func (q *TopologyQueryBuilder) Build() string { } strLabels := strings.Join(labels, ",") - dataField := getField(q.topology.DataField) factor := getFactor(q.topology.DataField) function, quantile := GetFunctionWithQuantile(q.topology.MetricFunction) @@ -171,15 +170,20 @@ func (q *TopologyQueryBuilder) Build() string { q.appendFilter(sb, extraFilter) } - if dataField == constants.MetricTypeDNSLatency { + switch q.topology.DataField { + case constants.MetricTypeDNSLatency: q.appendDNSLatencyFilter(sb) - } else if dataField == constants.MetricTypeDNSFlows { + case constants.MetricTypeDNSFlows: q.appendDNSFilter(sb) - } else if dataField == constants.MetricTypeFlowRTT { + case constants.MetricTypeTLSFlows: + q.appendTLSFilter(sb) + case constants.MetricTypeFlowRTT: q.appendRTTFilter(sb) } q.appendJSON(sb, true) + + dataField := q.topology.GetActualDataField() if len(dataField) > 0 { sb.WriteString("|unwrap ") sb.WriteString(dataField) diff --git a/pkg/model/filters/logql.go b/pkg/model/filters/logql.go index e5e7d7d38..ac9f3df47 100644 --- a/pkg/model/filters/logql.go +++ b/pkg/model/filters/logql.go @@ -271,12 +271,22 @@ func NumericLineFilter(key string, values []string, not, moreThan bool) (LineFil typeNumber) } -func ArrayLineFilter(key string, values []string, not bool) LineFilter { +// ArrayLineFilter returns a LineFilter and true if it has an empty match +func ArrayLineFilter(key string, values []string, not bool) (LineFilter, bool) { + if len(values) == 1 && (values[0] == "" || values[0] == `""`) { + // Special case, filter for n/a; note that logQL json doesn't support arrays, so we need a different solution + if not { + // key!=n/a + return RegexMatchLineFilter(key, true, ".*"), false + } + // key=n/a + return NotContainsKeyLineFilter(key), false + } lf := LineFilter{key: key, not: not} for _, value := range values { lf.values = append(lf.values, lineMatch{valueType: typeRegexArrayContains, value: value}) } - return lf + return lf, false } // StringLineFilter returns a LineFilter and true if it has an empty match diff --git a/pkg/prometheus/inventory.go b/pkg/prometheus/inventory.go index cbc180bb9..a184f7fc9 100644 --- a/pkg/prometheus/inventory.go +++ b/pkg/prometheus/inventory.go @@ -33,6 +33,12 @@ func NewInventory(cfg *config.Prometheus) *Inventory { cpy.ValueField = constants.MetricTypeDNSFlows toAppend = append(toAppend, cpy) } + // Simulate Flows and TLSFlows value fields + if strings.Contains(cfg.Metrics[i].Name, "_tls_flows") { + cfg.Metrics[i].ValueField = constants.MetricTypeTLSFlows + } else if strings.Contains(cfg.Metrics[i].Name, "_flows") { + cfg.Metrics[i].ValueField = constants.MetricTypeFlows + } } return &Inventory{metrics: append(cfg.Metrics, toAppend...)} } diff --git a/pkg/prometheus/inventory_test.go b/pkg/prometheus/inventory_test.go index 3e5c09015..a03eee7a9 100644 --- a/pkg/prometheus/inventory_test.go +++ b/pkg/prometheus/inventory_test.go @@ -4,10 +4,25 @@ import ( "testing" "github.com/netobserv/network-observability-console-plugin/pkg/config" + "github.com/netobserv/network-observability-console-plugin/pkg/utils/constants" "github.com/stretchr/testify/assert" ) var configuredMetrics = []config.MetricInfo{ + { + Enabled: true, + Name: "netobserv_metric_tls_flows_total", + Type: "Counter", + Direction: config.AnyDirection, + Labels: []string{"SrcK8S_Namespace", "DstK8S_Namespace"}, + }, + { + Enabled: false, + Name: "netobserv_metric_tls_flows_disabled", + Type: "Counter", + Direction: config.AnyDirection, + Labels: []string{"SrcK8S_Namespace", "DstK8S_Namespace"}, + }, { Enabled: true, Name: "netobserv_metric_1", @@ -81,7 +96,7 @@ func TestInventory_Search(t *testing.T) { assert.Empty(t, search.Found) assert.Empty(t, search.Candidates) - // Search flows metrics + // Search flows metrics (and TLS metric is skipped) search = inv.Search([]string{"SrcK8S_Namespace", "DstK8S_Namespace"}, "") assert.Equal(t, []string{"netobserv_metric_3"}, search.Found) assert.Empty(t, search.Candidates) @@ -89,6 +104,11 @@ func TestInventory_Search(t *testing.T) { search = inv.Search([]string{"SrcK8S_HostName", "DstK8S_HostName"}, "") assert.Equal(t, []string{"netobserv_metric_3"}, search.Found) assert.Empty(t, search.Candidates) + + // Search TLS flows metrics + search = inv.Search([]string{"SrcK8S_Namespace", "DstK8S_Namespace"}, constants.MetricTypeTLSFlows) + assert.Equal(t, []string{"netobserv_metric_tls_flows_total"}, search.Found) + assert.Empty(t, search.Candidates) } func TestInventory_Search_RTT_Candidate(t *testing.T) { diff --git a/pkg/utils/constants/constants.go b/pkg/utils/constants/constants.go index 6b193e9d4..dffa30167 100644 --- a/pkg/utils/constants/constants.go +++ b/pkg/utils/constants/constants.go @@ -19,6 +19,7 @@ const ( MetricTypeDroppedPackets = "PktDropPackets" MetricTypeDNSLatency = "DnsLatencyMs" MetricTypeDNSFlows = "DnsFlows" + MetricTypeTLSFlows = "TlsFlows" MetricTypeFlowRTT = "TimeFlowRttNs" DefaultMetricType = MetricTypeBytes diff --git a/web/locales/en/plugin__netobserv-plugin.json b/web/locales/en/plugin__netobserv-plugin.json index 020765727..2c2c82db6 100644 --- a/web/locales/en/plugin__netobserv-plugin.json +++ b/web/locales/en/plugin__netobserv-plugin.json @@ -528,6 +528,7 @@ "Show total": "Show total", "Show total dropped": "Show total dropped", "Show overall": "Show overall", + "%": "%", "Graph type": "Graph type", "Donut": "Donut", "Bars": "Bars", @@ -649,6 +650,8 @@ "internal": "internal", "P": "P", "Pps": "Pps", + "flows": "flows", + "fps": "fps", "minimum": "minimum", "maximum": "maximum", "90th percentile": "90th percentile", @@ -697,6 +700,15 @@ "Top {{limit}} DNS response code": "Top {{limit}} DNS response code", "Total DNS response code": "Total DNS response code", "The top DNS response code extracted from DNS response headers compared to total over the selected interval": "The top DNS response code extracted from DNS response headers compared to total over the selected interval", + "TLS usage": "TLS usage", + "TLS usage (network flows per second)": "TLS usage (network flows per second)", + "TLS traffic": "TLS traffic", + "TLS usage per version": "TLS usage per version", + "TLS per version (network flows per second)": "TLS per version (network flows per second)", + "TLS usage per group": "TLS usage per group", + "TLS per group (network flows per second)": "TLS per group (network flows per second)", + "TLS usage per cipher suite": "TLS usage per cipher suite", + "TLS per cipher suite (network flows per second)": "TLS per cipher suite (network flows per second)", "rates": "rates", "with total": "with total", "Invalid custom panel id": "Invalid custom panel id" diff --git a/web/src/api/loki.ts b/web/src/api/loki.ts index 9ad6f78a3..26a8f7789 100644 --- a/web/src/api/loki.ts +++ b/web/src/api/loki.ts @@ -24,17 +24,17 @@ export interface StreamResult { values: string[][]; } -export class RecordsResult { +export interface RecordsResult { records: Record[]; stats: Stats; } -export class FlowMetricsResult { +export interface FlowMetricsResult { metrics: TopologyMetrics[]; stats: Stats; } -export class GenericMetricsResult { +export interface GenericMetricsResult { metrics: GenericMetric[]; stats: Stats; } @@ -147,6 +147,11 @@ export type NetflowMetrics = { totalDnsLatency: Result; totalDnsCount: Result; totalRtt: Result; + tlsUsagePerVersion: Result; + tlsUsagePerCipher: Result; + tlsUsagePerGroup: Result; + tlsFlowRate: Result; + totalFlowRate: Result; custom: Map>; totalCustom: Map>; }; @@ -166,6 +171,11 @@ export const defaultNetflowMetrics: NetflowMetrics = { totalDnsLatency: Result.empty(), totalDnsCount: Result.empty(), totalRtt: Result.empty(), + tlsUsagePerCipher: Result.empty(), + tlsUsagePerGroup: Result.empty(), + tlsUsagePerVersion: Result.empty(), + tlsFlowRate: Result.empty(), + totalFlowRate: Result.empty(), custom: new Map(), totalCustom: new Map() }; diff --git a/web/src/api/routes.ts b/web/src/api/routes.ts index b4d4cf235..ff0990497 100644 --- a/web/src/api/routes.ts +++ b/web/src/api/routes.ts @@ -1,7 +1,7 @@ import axios from 'axios'; import { Config, defaultConfig } from '../model/config'; import { buildExportQuery } from '../model/export-query'; -import { FlowQuery, FlowScope, isTimeMetric } from '../model/flow-query'; +import { FlowQuery, FlowScope, isTimeMetric, StructuredFlowQuery, structuredToRawQuery } from '../model/flow-query'; import { ContextSingleton } from '../utils/context'; import { TimeRange } from '../utils/datetime'; import { parseGenericMetrics, parseTopologyMetrics } from '../utils/metrics'; @@ -73,7 +73,8 @@ export const queryPrometheusMetric = (query: string): Promise => { }); }; -export const getExportFlowsURL = (params: FlowQuery, columns?: string[]): string => { +export const getExportFlowsURL = (q: StructuredFlowQuery, columns?: string[]): string => { + const params = structuredToRawQuery(q); const exportQuery = buildExportQuery(params, columns); return `${ContextSingleton.getHost()}/api/loki/export?${exportQuery}`; }; @@ -169,7 +170,11 @@ export const getFlowMetrics = (params: FlowQuery, range: number | TimeRange): Pr }); }; -export const getFlowGenericMetrics = (params: FlowQuery, range: number | TimeRange): Promise => { +export const getFlowGenericMetrics = ( + q: StructuredFlowQuery, + range: number | TimeRange +): Promise => { + const params = structuredToRawQuery(q); return getFlowMetricsGeneric(params, res => { return parseGenericMetrics( res.result as RawTopologyMetrics[], diff --git a/web/src/components/drawer/netflow-traffic-drawer.tsx b/web/src/components/drawer/netflow-traffic-drawer.tsx index b83f9d8bd..cb636f78e 100644 --- a/web/src/components/drawer/netflow-traffic-drawer.tsx +++ b/web/src/components/drawer/netflow-traffic-drawer.tsx @@ -244,6 +244,7 @@ export const NetflowTrafficDrawer = React.forwardRef = ({ showDetails, resetDefaultFilters, getStatus(ContextSingleton.getForcedNamespace()) .then(status => { if (isMounted) { - console.info('status result', status); setStatus(status); setStatusError(undefined); } diff --git a/web/src/components/messages/error.tsx b/web/src/components/messages/error.tsx index 3a5311209..b4c719fed 100644 --- a/web/src/components/messages/error.tsx +++ b/web/src/components/messages/error.tsx @@ -113,7 +113,6 @@ export const ErrorComponent: React.FC = ({ title, error }) => { getStatus(ContextSingleton.getForcedNamespace()) .then(status => { - console.info('status result', status); setStatus(status); setStatusError(undefined); }) diff --git a/web/src/components/metrics/metrics-donut.tsx b/web/src/components/metrics/metrics-donut.tsx index 06de413b9..3cdbc385f 100644 --- a/web/src/components/metrics/metrics-donut.tsx +++ b/web/src/components/metrics/metrics-donut.tsx @@ -11,12 +11,13 @@ import './metrics-content.css'; export interface MetricsDonutProps { id: string; - subTitle?: string; + internalText?: string; + internalSubtitle?: string; limit: number; metricType: MetricType; metricFunction: MetricFunction; topKMetrics: (GenericMetric | NamedMetric)[]; - totalMetric: GenericMetric | NamedMetric; + totalMetric?: GenericMetric | NamedMetric; showOthers: boolean; othersName?: string; showLast?: boolean; @@ -29,7 +30,8 @@ export interface MetricsDonutProps { export const MetricsDonut: React.FC = ({ id, - subTitle, + internalText, + internalSubtitle, metricFunction, limit, metricType, @@ -53,7 +55,10 @@ export const MetricsDonut: React.FC = ({ [metricFunction, showLast] ); - let total = getStats(totalMetric.stats); + // If total metric isn't provided, use the sum of the provided metrics + let total = totalMetric + ? getStats(totalMetric.stats) + : topKMetrics.map(m => getStats(m.stats)).reduce((prev, cur) => prev + cur); let filtered = topKMetrics; if (showOutOfScope === false) { filtered = (filtered as NamedMetric[]).filter(m => { @@ -113,7 +118,7 @@ export const MetricsDonut: React.FC = ({ name: m.name })); - const legentComponent = ( + const legendComponent = ( } data={legendData} @@ -129,16 +134,19 @@ export const MetricsDonut: React.FC = ({ return observeDimensions(containerRef, dimensions, setDimensions); }, [containerRef, dimensions, setDimensions]); + // Hide legend on small screens to prevent overlap/cropping + const showLegendResponsive = showLegend && dimensions.width >= 550; + return (
= ({ allowTooltip={showLegend} animate={animate} padding={ - showLegend + showLegendResponsive ? { bottom: 20, left: 20, - right: 400, + right: 350, top: 20 } : { @@ -164,8 +172,8 @@ export const MetricsDonut: React.FC = ({ top: 0 } } - title={`${getFormattedValue(total, metricType, metricFunction, t)}`} - subTitle={subTitle ? subTitle : t('Total')} + title={internalText || `${getFormattedValue(total, metricType, metricFunction, t)}`} + subTitle={internalSubtitle || t('Total')} />
); diff --git a/web/src/components/metrics/metrics-graph.tsx b/web/src/components/metrics/metrics-graph.tsx index bd312e6fd..4a1b4b140 100644 --- a/web/src/components/metrics/metrics-graph.tsx +++ b/web/src/components/metrics/metrics-graph.tsx @@ -69,7 +69,8 @@ export const MetricsGraph: React.FC = ({ childName: `${showBar ? 'bar-' : 'area-'}${idx}`, name: (m as NamedMetric).shortName || (m as GenericMetric).name, tooltipName: - (tooltipsTruncate ? (m as NamedMetric).shortName : (m as NamedMetric).fullName) || (m as GenericMetric).name + (tooltipsTruncate ? (m as NamedMetric).shortName || (m as GenericMetric).name : (m as NamedMetric).fullName) || + (m as GenericMetric).name })); const topKDatapoints: ChartDataPoint[][] = filteredMetrics.map(toDatapoints); diff --git a/web/src/components/modals/__tests__/export-modal.spec.tsx b/web/src/components/modals/__tests__/export-modal.spec.tsx index 3925e939d..96e243067 100644 --- a/web/src/components/modals/__tests__/export-modal.spec.tsx +++ b/web/src/components/modals/__tests__/export-modal.spec.tsx @@ -10,6 +10,8 @@ jest.mock('../../../api/routes', () => ({ getRole: jest.fn(() => Promise.resolve('admin')) })); +const emptyFilters = { match: 'all' as const, list: [] }; + describe('', () => { const props: ExportModalProps = { isModalOpen: true, @@ -17,7 +19,13 @@ describe('', () => { columns: ShuffledColumnSample, filters: [], range: 300, - flowQuery: { recordType: 'flowLog', dataSource: 'auto', limit: 100, filters: '', packetLoss: 'all' }, + flowQuery: { + recordType: 'flowLog', + dataSource: 'auto', + limit: 100, + structuredFilters: emptyFilters, + packetLoss: 'all' + }, id: 'export-modal' }; diff --git a/web/src/components/modals/columns-modal.tsx b/web/src/components/modals/columns-modal.tsx index 9b4d34358..ed44aa084 100644 --- a/web/src/components/modals/columns-modal.tsx +++ b/web/src/components/modals/columns-modal.tsx @@ -28,7 +28,7 @@ import Modal from './modal'; const COLUMNS_DRAG_ZONE = 'netobs-columns-modal'; -export const columnFilterKeys = ['source', 'destination', 'time', 'host', 'namespace', 'owner', 'ip', 'dns']; +export const columnFilterKeys = ['source', 'destination', 'time', 'host', 'namespace', 'owner', 'ip', 'dns', 'tls']; export interface ColumnsModalProps { isModalOpen: boolean; diff --git a/web/src/components/modals/export-modal.tsx b/web/src/components/modals/export-modal.tsx index fe06cc219..edcbd01bb 100644 --- a/web/src/components/modals/export-modal.tsx +++ b/web/src/components/modals/export-modal.tsx @@ -20,7 +20,7 @@ import * as React from 'react'; import { useTranslation } from 'react-i18next'; import { getExportFlowsURL } from '../../api/routes'; import { Filter } from '../../model/filters'; -import { FlowQuery } from '../../model/flow-query'; +import { StructuredFlowQuery } from '../../model/flow-query'; import { Column, getFullColumnName } from '../../utils/columns'; import { getTimeRangeOptions, TimeRange } from '../../utils/datetime'; import { formatDuration, getDateSInMiliseconds } from '../../utils/duration'; @@ -34,7 +34,7 @@ export interface ExportModalProps { isModalOpen: boolean; setModalOpen: (v: boolean) => void; range: number | TimeRange; - flowQuery: FlowQuery; + flowQuery: StructuredFlowQuery; columns: Column[]; filters: Filter[]; id?: string; diff --git a/web/src/components/slider/scope-slider.tsx b/web/src/components/slider/scope-slider.tsx index 2e60733db..2edd55e47 100644 --- a/web/src/components/slider/scope-slider.tsx +++ b/web/src/components/slider/scope-slider.tsx @@ -17,6 +17,7 @@ export const ScopeSlider: React.FC = ({ scope, setScope, scope setScope(sd.id)} diff --git a/web/src/components/tabs/netflow-overview/__tests__/netflow-overview.spec.tsx b/web/src/components/tabs/netflow-overview/__tests__/netflow-overview.spec.tsx index df28c412c..d0529706b 100644 --- a/web/src/components/tabs/netflow-overview/__tests__/netflow-overview.spec.tsx +++ b/web/src/components/tabs/netflow-overview/__tests__/netflow-overview.spec.tsx @@ -7,6 +7,7 @@ import { ScopeDefSample } from '../../../../components/__tests-data__/scopes'; import { TruncateLength } from '../../../../components/dropdowns/truncate-dropdown'; import { FlowScope, RecordType } from '../../../../model/flow-query'; import { Result } from '../../../../utils/result'; +import { FilterDefinitionSample } from '../../../__tests-data__/filters'; import { ShuffledDefaultPanels } from '../../../__tests-data__/panels'; import { NetflowOverview, NetflowOverviewProps } from '../netflow-overview'; @@ -21,6 +22,7 @@ describe('', () => { forcedSize: { width: 800, height: 800 } as DOMRect, scopes: ScopeDefSample, metricScope: 'host' as FlowScope, + filterDefinitions: FilterDefinitionSample, setMetricScope: jest.fn() }; diff --git a/web/src/components/tabs/netflow-overview/netflow-overview.tsx b/web/src/components/tabs/netflow-overview/netflow-overview.tsx index 515fac732..759064f34 100644 --- a/web/src/components/tabs/netflow-overview/netflow-overview.tsx +++ b/web/src/components/tabs/netflow-overview/netflow-overview.tsx @@ -22,7 +22,8 @@ import { import { getFlowGenericMetrics } from '../../../api/routes'; import { ScopeSlider } from '../../../components/slider/scope-slider'; import { Config } from '../../../model/config'; -import { FlowQuery, FlowScope, RecordType } from '../../../model/flow-query'; +import { FilterDefinition } from '../../../model/filters'; +import { FlowScope, RecordType, StructuredFlowQuery } from '../../../model/flow-query'; import { getStat } from '../../../model/metrics'; import { useNetflowContext } from '../../../model/netflow-context'; import { ScopeConfigDef } from '../../../model/scope'; @@ -30,6 +31,7 @@ import { TimeRange } from '../../../utils/datetime'; import { getDNSErrorDescription, getDNSRcodeDescription } from '../../../utils/dns'; import { getDSCPServiceClassName } from '../../../utils/dscp'; import { getStructuredHTTPError, StructuredError } from '../../../utils/errors'; +import { valueFormat } from '../../../utils/format'; import { localStorageOverviewKebabKey, useLocalStorage } from '../../../utils/local-storage-hook'; import { observeDOMRect, toNamedMetric } from '../../../utils/metrics-helper'; import { @@ -43,7 +45,8 @@ import { OverviewPanelId, OverviewPanelInfo, parseCustomMetricId, - rttIdMatcher + rttIdMatcher, + tlsIdMatcher } from '../../../utils/overview-panels'; import { convertRemToPixels } from '../../../utils/panel'; import { formatPort } from '../../../utils/port'; @@ -81,6 +84,7 @@ export interface NetflowOverviewProps { metrics: NetflowMetrics; loading?: boolean; isDark?: boolean; + filterDefinitions: FilterDefinition[]; resetDefaultFilters?: (c?: Config) => void; clearFilters?: () => void; truncateLength: TruncateLength; @@ -119,7 +123,7 @@ export const NetflowOverview = React.forwardRef p.id)) as RateMetrics; (Object.keys(rateMetrics) as (keyof typeof rateMetrics)[]).map(key => { const metricType = key === 'bytes' ? 'Bytes' : 'Packets'; - const fq: FlowQuery = { ...baseQuery, function: 'rate', type: metricType }; + const fq: StructuredFlowQuery = { ...baseQuery, function: 'rate', type: metricType }; promises.push( Result.fromPromise(getMetrics(fq, range)).then(res => { //set matching value and apply changes on the entire object to trigger refresh @@ -142,7 +146,7 @@ export const NetflowOverview = React.forwardRef { const metricType = key === 'bytes' ? 'Bytes' : 'Packets'; - const fq: FlowQuery = { ...baseQuery, function: 'rate', aggregateBy: 'app', type: metricType }; + const fq: StructuredFlowQuery = { ...baseQuery, function: 'rate', aggregateBy: 'app', type: metricType }; promises.push( Result.fromPromise(getMetrics(fq, range)).then(res => { //set matching value and apply changes on the entire object to trigger refresh @@ -169,7 +173,7 @@ export const NetflowOverview = React.forwardRef p.id)) as RateMetrics; (Object.keys(droppedRateMetrics) as (keyof typeof droppedRateMetrics)[]).map(key => { const metricType = key === 'bytes' ? 'PktDropBytes' : 'PktDropPackets'; - const fq: FlowQuery = { ...baseQuery, function: 'rate', type: metricType }; + const fq: StructuredFlowQuery = { ...baseQuery, function: 'rate', type: metricType }; promises.push( Result.fromPromise(getMetrics(fq, range)).then(res => { //set matching value and apply changes on the entire object to trigger refresh @@ -189,7 +193,7 @@ export const NetflowOverview = React.forwardRef p.id)) as TotalRateMetrics; (Object.keys(totalDroppedRateMetric) as (keyof typeof totalDroppedRateMetric)[]).map(key => { const metricType = key === 'bytes' ? 'PktDropBytes' : 'PktDropPackets'; - const fq: FlowQuery = { ...baseQuery, function: 'rate', aggregateBy: 'app', type: metricType }; + const fq: StructuredFlowQuery = { ...baseQuery, function: 'rate', aggregateBy: 'app', type: metricType }; promises.push( Result.fromPromise(getMetrics(fq, range)).then(res => { //set matching value and apply changes on the entire object to trigger refresh @@ -207,13 +211,13 @@ export const NetflowOverview = React.forwardRef p.id)) as FunctionMetrics; (Object.keys(dnsLatencyMetrics) as (keyof typeof dnsLatencyMetrics)[]).map(fn => { - const fq: FlowQuery = { ...baseQuery, function: fn, type: 'DnsLatencyMs' }; + const fq: StructuredFlowQuery = { ...baseQuery, function: fn, type: 'DnsLatencyMs' }; promises.push( Result.fromPromise(getMetrics(fq, range)).then(res => { const dnsLatency = res @@ -272,7 +276,7 @@ export const NetflowOverview = React.forwardRef p.id)) as TotalFunctionMetrics; (Object.keys(totalDnsLatencyMetric) as (keyof typeof totalDnsLatencyMetric)[]).map(fn => { - const fq: FlowQuery = { ...baseQuery, function: fn, aggregateBy: 'app', type: 'DnsLatencyMs' }; + const fq: StructuredFlowQuery = { ...baseQuery, function: fn, aggregateBy: 'app', type: 'DnsLatencyMs' }; promises.push( Result.fromPromise(getMetrics(fq, range)).then(res => { const totalDnsLatency = res @@ -290,14 +294,24 @@ export const NetflowOverview = React.forwardRef p.id.includes('rcode_dns_latency_flows'))) { - const fqNames: FlowQuery = { ...baseQuery, aggregateBy: 'DnsName', function: 'count', type: 'DnsFlows' }; - const fqCodes: FlowQuery = { + const fqNames: StructuredFlowQuery = { + ...baseQuery, + aggregateBy: 'DnsName', + function: 'count', + type: 'DnsFlows' + }; + const fqCodes: StructuredFlowQuery = { ...baseQuery, aggregateBy: 'DnsFlagsResponseCode', function: 'count', type: 'DnsFlows' }; - const fqTotal: FlowQuery = { ...baseQuery, aggregateBy: 'app', function: 'count', type: 'DnsFlows' }; + const fqTotal: StructuredFlowQuery = { + ...baseQuery, + aggregateBy: 'app', + function: 'count', + type: 'DnsFlows' + }; promises.push( ...[ //get dns names @@ -345,7 +359,7 @@ export const NetflowOverview = React.forwardRef p.id)) as FunctionMetrics; (Object.keys(rttMetrics) as (keyof typeof rttMetrics)[]).map(fn => { - const fq: FlowQuery = { ...baseQuery, function: fn, type: 'TimeFlowRttNs' }; + const fq: StructuredFlowQuery = { ...baseQuery, function: fn, type: 'TimeFlowRttNs' }; promises.push( Result.fromPromise(getMetrics(fq, range)).then(res => { //set matching value and apply changes on the entire object to trigger refresh @@ -364,7 +378,7 @@ export const NetflowOverview = React.forwardRef p.id)) as TotalFunctionMetrics; (Object.keys(totalRttMetric) as (keyof typeof totalRttMetric)[]).map(fn => { - const fq: FlowQuery = { ...baseQuery, function: fn, aggregateBy: 'app', type: 'TimeFlowRttNs' }; + const fq: StructuredFlowQuery = { ...baseQuery, function: fn, aggregateBy: 'app', type: 'TimeFlowRttNs' }; promises.push( Result.fromPromise(getMetrics(fq, range)).then(res => { //set matching value and apply changes on the entire object to trigger refresh @@ -384,6 +398,98 @@ export const NetflowOverview = React.forwardRef p.id.includes(tlsIdMatcher)); + if (!_.isEmpty(tlsPanels)) { + // Get TLS usage stats + const fqTotal: StructuredFlowQuery = { + ...baseQuery, + function: 'rate', + type: 'Flows', + aggregateBy: 'app' + }; + promises.push( + Result.fromPromise(getFlowGenericMetrics(fqTotal, range)).then(res => { + const totalFlowRate = res + .map(success => (success.metrics.length > 0 ? { ...success.metrics[0], name: 'Total flows' } : undefined)) + .mapError(err => getStructuredHTTPError(err, `Total flow rate`)); + currentMetrics = { ...currentMetrics, totalFlowRate }; + setMetrics(currentMetrics); + return res.map(r => r.stats).or(emptyStats); + }) + ); + const fqTLS: StructuredFlowQuery = { + ...baseQuery, + function: 'rate', + type: 'TlsFlows', + aggregateBy: 'app' + }; + promises.push( + Result.fromPromise(getFlowGenericMetrics(fqTLS, range)).then(res => { + const tlsFlowRate = res + .map(success => (success.metrics.length > 0 ? { ...success.metrics[0], name: 'TLS flows' } : undefined)) + .mapError(err => getStructuredHTTPError(err, `TLS flow rate`)); + currentMetrics = { ...currentMetrics, tlsFlowRate }; + setMetrics(currentMetrics); + return res.map(r => r.stats).or(emptyStats); + }) + ); + const fqByVersion: StructuredFlowQuery = { + ...baseQuery, + function: 'rate', + type: 'TlsFlows', + aggregateBy: 'TLSVersion' + }; + const fqByCipherSuite: StructuredFlowQuery = { + ...baseQuery, + function: 'rate', + type: 'TlsFlows', + aggregateBy: 'TLSCipherSuite' + }; + const fqByGroup: StructuredFlowQuery = { + ...baseQuery, + function: 'rate', + type: 'TlsFlows', + aggregateBy: 'TLSGroup' + }; + promises.push( + ...[ + Result.fromPromise(getFlowGenericMetrics(fqByVersion, range)).then(res => { + const tlsUsagePerVersion = res + .map(success => success.metrics) + .mapError(err => getStructuredHTTPError(err, `TLS by version`)); + currentMetrics = { ...currentMetrics, tlsUsagePerVersion }; + setMetrics(currentMetrics); + return res.map(r => r.stats).or(emptyStats); + }), + Result.fromPromise(getFlowGenericMetrics(fqByCipherSuite, range)).then(res => { + const tlsUsagePerCipher = res + .map(success => success.metrics) + .mapError(err => getStructuredHTTPError(err, `TLS by cipher suite`)); + currentMetrics = { ...currentMetrics, tlsUsagePerCipher }; + setMetrics(currentMetrics); + return res.map(r => r.stats).or(emptyStats); + }), + Result.fromPromise(getFlowGenericMetrics(fqByGroup, range)).then(res => { + const tlsUsagePerGroup = res + .map(success => success.metrics) + .mapError(err => getStructuredHTTPError(err, `TLS by group`)); + currentMetrics = { ...currentMetrics, tlsUsagePerGroup }; + setMetrics(currentMetrics); + return res.map(r => r.stats).or(emptyStats); + }) + ] + ); + } else { + setMetrics({ + ...currentMetrics, + tlsFlowRate: Result.empty(), + totalFlowRate: Result.empty(), + tlsUsagePerCipher: Result.empty(), + tlsUsagePerGroup: Result.empty(), + tlsUsagePerVersion: Result.empty() + }); + } + const customPanels = props.panels.filter(p => p.id.startsWith(customPanelMatcher)); if (!_.isEmpty(customPanels)) { //set custom metrics @@ -394,13 +500,13 @@ export const NetflowOverview = React.forwardRef { + if (props.metrics.totalFlowRate.result && props.metrics.tlsFlowRate.result) { + const tls = getStat(props.metrics.tlsFlowRate.result.stats, 'rate'); + const total = getStat(props.metrics.totalFlowRate.result.stats, 'rate'); + if (total > 0) { + return (100 * tls) / total; + } + } + return undefined; + }, [props.metrics.totalFlowRate, props.metrics.tlsFlowRate]); + const smallerTexts = props.truncateLength >= TruncateLength.M; const getPanelContent = React.useCallback( (id: OverviewPanelId, info: OverviewPanelInfo, isFocus: boolean, animate: boolean): PanelContent => { @@ -681,7 +798,7 @@ export const NetflowOverview = React.forwardRef r.error) + .find(e => e !== undefined); + if (panelError) { + return { + calculatedTitle: info.topTitle, + element: + }; + } + // No error at this point + const metrics = props.metrics.tlsFlowRate.result; + const total = props.metrics.totalFlowRate.result; + const options = getKebabOptions(id, { + graph: { options: ['bar_line', 'donut'], type: 'donut' } + }); + const isDonut = options.graph?.type === 'donut'; + return { + calculatedTitle: info.topTitle, + element: + metrics === undefined ? ( + emptyGraph(!isFocus) + ) : isDonut ? ( + + ) : ( + + ), + kebab: setKebabOptions(id, opts)} />, + bodyClassSmall: options.graph!.type === 'donut', + doubleWidth: options.graph!.type !== 'donut' + }; + } + case 'tls_per_version': + case 'tls_per_group': + case 'tls_per_cipher_suite': { + const result = + id === 'tls_per_version' + ? props.metrics.tlsUsagePerVersion + : id === 'tls_per_group' + ? props.metrics.tlsUsagePerGroup + : props.metrics.tlsUsagePerCipher; + if (result.error) { + return { + calculatedTitle: info.topTitle, + element: ( + + ) + }; + } + // No error at this point + const metric = sortMetrics(result).or([]); + const options = getKebabOptions(id, { + showTop: true, + graph: { options: ['bar_line', 'donut'], type: 'donut' } + }); + const isDonut = options.graph?.type === 'donut'; + return { + calculatedTitle: info.topTitle, + element: _.isEmpty(metric) ? ( + emptyGraph(!isFocus) + ) : isDonut ? ( + + ) : ( + + ), + kebab: setKebabOptions(id, opts)} />, + bodyClassSmall: options.graph!.type === 'donut', + doubleWidth: options.graph!.type !== 'donut' + }; + } default: { const parsedId = parseCustomMetricId(id); if (parsedId.isValid) { @@ -1048,7 +1290,7 @@ export const NetflowOverview = React.forwardRef { + return { ...current, list: _.concat(current.list, newFilters) }; +}; diff --git a/web/src/model/flow-query.ts b/web/src/model/flow-query.ts index 62c76198d..3ca32ec41 100644 --- a/web/src/model/flow-query.ts +++ b/web/src/model/flow-query.ts @@ -1,5 +1,5 @@ import { Field } from '../api/ipfix'; -import { Filter } from './filters'; +import { Filter, Filters } from './filters'; export type RecordType = 'allConnections' | 'newConnection' | 'heartbeat' | 'endConnection' | 'flowLog'; export type DataSource = 'auto' | 'loki' | 'prom'; @@ -7,7 +7,7 @@ export type Match = 'any' | 'all' | 'bidirectional'; export type PacketLoss = 'dropped' | 'hasDrops' | 'sent' | 'all'; export type MetricFunction = 'count' | 'sum' | 'avg' | 'min' | 'max' | 'p90' | 'p99' | 'rate'; export type StatFunction = MetricFunction | 'last'; -export type MetricType = 'Flows' | 'DnsFlows' | Field; +export type MetricType = 'Flows' | 'DnsFlows' | 'TlsFlows' | Field; // scope are configurable and can be any string // such as 'app', 'cluster', 'zone', 'host', 'namespace', 'owner', 'resource'... export type FlowScope = string; @@ -21,6 +21,10 @@ export type NodeType = FlowScope | 'unknown'; // 'owners'... export type Groups = string; +export type StructuredFlowQuery = Omit & { + structuredFilters: Filters; +}; + export interface FlowQuery { timeRange?: number; startTime?: string; @@ -40,13 +44,21 @@ export interface FlowQuery { step?: string; } +export const structuredToRawQuery = (q: StructuredFlowQuery): FlowQuery => { + const raw = { + ...q, + filters: filtersToString(q.structuredFilters.list, q.structuredFilters.match === 'any'), + structuredFilters: undefined + }; + delete raw.structuredFilters; + return raw; +}; + export const filtersToString = (filters: Filter[], matchAny: boolean): string => { - const matches: string[] = []; - filters.forEach(f => { - const str = f.def.encoder(f.values, f.compare, matchAny); - matches.push(str); + const encoded = filters.map(f => { + return f.def.encoder(f.values, f.compare, matchAny); }); - return encodeURIComponent(matches.join(matchAny ? '|' : '&')); + return encodeURIComponent(encoded.join(matchAny ? '|' : '&')); }; export const filterByHashId = (hashId: string): string => { diff --git a/web/src/model/netflow-context.ts b/web/src/model/netflow-context.ts index 6cb40db1c..21c3b98d5 100644 --- a/web/src/model/netflow-context.ts +++ b/web/src/model/netflow-context.ts @@ -28,6 +28,7 @@ const defaultCaps: ConfigCapabilities = { isDNSTracking: false, isFlowRTT: false, isPktDrop: false, + isTLSTracking: false, isPromOnly: true, availableScopes: [], allowedMetricTypes: [], diff --git a/web/src/utils/__tests__/back-and-forth.spec.ts b/web/src/utils/__tests__/back-and-forth.spec.ts index a79a1e902..d03a50f4e 100644 --- a/web/src/utils/__tests__/back-and-forth.spec.ts +++ b/web/src/utils/__tests__/back-and-forth.spec.ts @@ -3,7 +3,6 @@ import { getFlowMetrics, getFlowRecords } from '../../api/routes'; import { FilterDefinitionSample } from '../../components/__tests-data__/filters'; import { ScopeDefSample } from '../../components/__tests-data__/scopes'; import { Filter, FilterCompare, FilterId, Filters, FilterValue } from '../../model/filters'; -import { filtersToString } from '../../model/flow-query'; import { getFetchFunctions, mergeMetricsBNF } from '../back-and-forth'; import { ContextSingleton } from '../context'; import { findFilter } from '../filter-definitions'; @@ -24,9 +23,9 @@ const filter = (id: FilterId, values: FilterValue[], not?: boolean): Filter => { }; }; -const getEncodedFilter = (filters: Filters, matchAny: boolean) => { - getFetchFunctions(FilterDefinitionSample, filters, matchAny).getRecords({ - filters: filtersToString(filters.list, matchAny), +const getEncodedFilter = (filters: Filters) => { + getFetchFunctions(FilterDefinitionSample).getRecords({ + structuredFilters: filters, recordType: 'flowLog', dataSource: 'auto', limit: 5, @@ -36,8 +35,8 @@ const getEncodedFilter = (filters: Filters, matchAny: boolean) => { return getFlowRecordsMock.mock.calls[0][0].filters; }; -const getDecodedFilter = (filters: Filters, matchAny: boolean) => { - return decodeURIComponent(getEncodedFilter(filters, matchAny)); +const getDecodedFilter = (filters: Filters) => { + return decodeURIComponent(getEncodedFilter(filters)); }; describe('Match all, flows', () => { @@ -46,74 +45,59 @@ describe('Match all, flows', () => { }); it('should encode', () => { - const filters = getEncodedFilter( - { list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }])], match: 'all' }, - false - ); + const filters = getEncodedFilter({ list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }])], match: 'all' }); expect(filters).toEqual('SrcK8S_Name%3Dtest1%2Ctest2'); }); it('should generate AND groups', () => { - const grouped = getDecodedFilter( - { - list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('src_namespace', [{ v: 'ns' }])], - match: 'all' - }, - false - ); + const grouped = getDecodedFilter({ + list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('src_namespace', [{ v: 'ns' }])], + match: 'all' + }); expect(grouped).toEqual('SrcK8S_Name=test1,test2&SrcK8S_Namespace=ns'); }); it('should generate AND groups, back and forth', () => { - const grouped = getDecodedFilter( - { - list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('dst_port', [{ v: '443' }])], - match: 'bidirectional' - }, - false - ); + const grouped = getDecodedFilter({ + list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('dst_port', [{ v: '443' }])], + match: 'bidirectional' + }); expect(grouped).toEqual('SrcK8S_Name=test1,test2&DstPort=443|DstK8S_Name=test1,test2&SrcPort=443'); }); it('should filter for namespace to owner back and forth', () => { - const grouped = getDecodedFilter( - { - list: [filter('src_namespace', [{ v: 'ns' }]), filter('dst_owner_name', [{ v: 'test' }])], - match: 'bidirectional' - }, - false - ); + const grouped = getDecodedFilter({ + list: [filter('src_namespace', [{ v: 'ns' }]), filter('dst_owner_name', [{ v: 'test' }])], + match: 'bidirectional' + }); expect(grouped).toEqual('SrcK8S_Namespace=ns&DstK8S_OwnerName=test|DstK8S_Namespace=ns&SrcK8S_OwnerName=test'); }); it('should generate AND groups, back and forth, mixed with non-Src/Dst', () => { - const grouped = getDecodedFilter( - { - list: [ - filter('src_name', [{ v: 'test' }]), - filter('dst_port', [{ v: '443' }]), - filter('src_kind', [{ v: 'Pod' }]), - filter('protocol', [{ v: '6' }]) - ], - match: 'bidirectional' - }, - false - ); + const grouped = getDecodedFilter({ + list: [ + filter('src_name', [{ v: 'test' }]), + filter('dst_port', [{ v: '443' }]), + filter('src_kind', [{ v: 'Pod' }]), + filter('protocol', [{ v: '6' }]) + ], + match: 'bidirectional' + }); expect(grouped).toEqual( 'SrcK8S_Name=test&DstPort=443&SrcK8S_Type=Pod&Proto=6' + '|DstK8S_Name=test&SrcPort=443&DstK8S_Type=Pod&Proto=6' ); }); it('should generate simple Src K8S resource', () => { - const grouped = getDecodedFilter({ list: [filter('src_resource', [{ v: 'Pod.ns.test' }])], match: 'all' }, false); + const grouped = getDecodedFilter({ list: [filter('src_resource', [{ v: 'Pod.ns.test' }])], match: 'all' }); expect(grouped).toEqual('SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"'); }); it('should generate K8S resource, back and forth', () => { - const grouped = getDecodedFilter( - { list: [filter('src_resource', [{ v: 'Pod.ns.test' }])], match: 'bidirectional' }, - false - ); + const grouped = getDecodedFilter({ + list: [filter('src_resource', [{ v: 'Pod.ns.test' }])], + match: 'bidirectional' + }); expect(grouped).toEqual( 'SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"' + '|DstK8S_Type="Pod"&DstK8S_Namespace="ns"&DstK8S_Name="test"' @@ -121,10 +105,7 @@ describe('Match all, flows', () => { }); it('should generate Node Src/Dst K8S resource, back and forth', () => { - const grouped = getDecodedFilter( - { list: [filter('src_resource', [{ v: 'Node.test' }])], match: 'bidirectional' }, - false - ); + const grouped = getDecodedFilter({ list: [filter('src_resource', [{ v: 'Node.test' }])], match: 'bidirectional' }); expect(grouped).toEqual( 'SrcK8S_Type="Node"&SrcK8S_Namespace=""&SrcK8S_Name="test"' + '|DstK8S_Type="Node"&DstK8S_Namespace=""&DstK8S_Name="test"' @@ -132,10 +113,10 @@ describe('Match all, flows', () => { }); it('should generate Owner Src/Dst K8S resource, back and forth', () => { - const grouped = getDecodedFilter( - { list: [filter('src_resource', [{ v: 'DaemonSet.ns.test' }])], match: 'bidirectional' }, - false - ); + const grouped = getDecodedFilter({ + list: [filter('src_resource', [{ v: 'DaemonSet.ns.test' }])], + match: 'bidirectional' + }); expect(grouped).toEqual( 'SrcK8S_OwnerType="DaemonSet"&SrcK8S_Namespace="ns"&SrcK8S_OwnerName="test"' + '|DstK8S_OwnerType="DaemonSet"&DstK8S_Namespace="ns"&DstK8S_OwnerName="test"' @@ -143,13 +124,10 @@ describe('Match all, flows', () => { }); it('should generate Src/Dst K8S resource ANDed with Dst Name, back and forth', () => { - const grouped = getDecodedFilter( - { - list: [filter('src_resource', [{ v: 'Pod.ns.test' }]), filter('dst_name', [{ v: 'peer' }])], - match: 'bidirectional' - }, - false - ); + const grouped = getDecodedFilter({ + list: [filter('src_resource', [{ v: 'Pod.ns.test' }]), filter('dst_name', [{ v: 'peer' }])], + match: 'bidirectional' + }); expect(grouped).toEqual( 'SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"&DstK8S_Name=peer' + '|DstK8S_Type="Pod"&DstK8S_Namespace="ns"&DstK8S_Name="test"&SrcK8S_Name=peer' @@ -163,112 +141,41 @@ describe('Match any, flows', () => { }); it('should encode', () => { - const grouped = getEncodedFilter( - { list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }])], match: 'all' }, - true - ); + const grouped = getEncodedFilter({ list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }])], match: 'any' }); expect(grouped).toEqual('SrcK8S_Name%3Dtest1%2Ctest2'); }); it('should generate OR groups', () => { - const grouped = getDecodedFilter( - { - list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('src_namespace', [{ v: 'ns' }])], - match: 'all' - }, - true - ); + const grouped = getDecodedFilter({ + list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('src_namespace', [{ v: 'ns' }])], + match: 'any' + }); expect(grouped).toEqual('SrcK8S_Name=test1,test2|SrcK8S_Namespace=ns'); }); - it('should generate OR groups, back and forth', () => { - const grouped = getDecodedFilter( - { - list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('dst_port', [{ v: '443' }])], - match: 'bidirectional' - }, - true - ); - expect(grouped).toEqual('SrcK8S_Name=test1,test2|DstPort=443|DstK8S_Name=test1,test2|SrcPort=443'); - }); - - it('should generate flat OR groups, back and forth', () => { - const grouped = getDecodedFilter( - { - list: [ - filter('src_name', [{ v: 'test' }]), - filter('src_port', [{ v: '443' }]), - filter('src_kind', [{ v: 'Pod' }]), - filter('protocol', [{ v: '6' }]) - ], - match: 'bidirectional' - }, - true - ); - expect(grouped).toEqual( - 'SrcK8S_Name=test|SrcPort=443|SrcK8S_Type=Pod|Proto=6|DstK8S_Name=test|DstPort=443|DstK8S_Type=Pod' - ); - }); - - it('should generate simple Src K8S resource', () => { - const grouped = getDecodedFilter({ list: [filter('src_resource', [{ v: 'Pod.ns.test' }])], match: 'all' }, true); - expect(grouped).toEqual('SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"'); - }); - - it('should generate K8S resource, back and forth', () => { - const grouped = getDecodedFilter( - { list: [filter('src_resource', [{ v: 'Pod.ns.test' }])], match: 'bidirectional' }, - true - ); - expect(grouped).toEqual( - 'SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"' + - '|DstK8S_Type="Pod"&DstK8S_Namespace="ns"&DstK8S_Name="test"' - ); - }); - - it('should generate Node K8S resource, back and forth', () => { - const grouped = getDecodedFilter( - { list: [filter('src_resource', [{ v: 'Node.test' }])], match: 'bidirectional' }, - true - ); - expect(grouped).toEqual( - 'SrcK8S_Type="Node"&SrcK8S_Namespace=""&SrcK8S_Name="test"' + - '|DstK8S_Type="Node"&DstK8S_Namespace=""&DstK8S_Name="test"' - ); - }); - - it('should generate Owner K8S resource, back and forth', () => { - const grouped = getDecodedFilter( - { list: [filter('src_resource', [{ v: 'DaemonSet.ns.test' }])], match: 'bidirectional' }, - true - ); + it('should generate simple two Src K8S resource', () => { + const grouped = getDecodedFilter({ + list: [filter('src_resource', [{ v: 'Pod.ns.test1' }, { v: 'Pod.ns.test2' }])], + match: 'any' + }); expect(grouped).toEqual( - 'SrcK8S_OwnerType="DaemonSet"&SrcK8S_Namespace="ns"&SrcK8S_OwnerName="test"' + - '|DstK8S_OwnerType="DaemonSet"&DstK8S_Namespace="ns"&DstK8S_OwnerName="test"' + 'SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test1"|SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test2"' ); }); - it('should generate K8S resource, back and forth, ORed with Name', () => { - const grouped = getDecodedFilter( - { - list: [filter('src_resource', [{ v: 'Pod.ns.test' }]), filter('dst_name', [{ v: 'peer' }])], - match: 'bidirectional' - }, - true - ); - expect(grouped).toEqual( - 'SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"' + - '|DstK8S_Name=peer' + - '|DstK8S_Type="Pod"&DstK8S_Namespace="ns"&DstK8S_Name="test"' + - '|SrcK8S_Name=peer' - ); + it('should generate K8S resource, ORed with Name', () => { + const grouped = getDecodedFilter({ + list: [filter('src_resource', [{ v: 'Pod.ns.test' }]), filter('dst_name', [{ v: 'peer' }])], + match: 'any' + }); + expect(grouped).toEqual('SrcK8S_Type="Pod"&SrcK8S_Namespace="ns"&SrcK8S_Name="test"|DstK8S_Name=peer'); }); }); -const getTopoForFilter = (filters: Filters, matchAny: boolean) => { - getFetchFunctions(FilterDefinitionSample, filters, matchAny).getMetrics( +const getTopoForFilter = (filters: Filters) => { + getFetchFunctions(FilterDefinitionSample).getMetrics( { - filters: filtersToString(filters.list, matchAny), + structuredFilters: filters, recordType: 'flowLog', dataSource: 'auto', limit: 5, @@ -284,19 +191,16 @@ describe('Match all, topology', () => { }); it('should encode', () => { - getTopoForFilter({ list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }])], match: 'all' }, false); + getTopoForFilter({ list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }])], match: 'all' }); expect(getFlowMetricsMock).toHaveBeenCalledTimes(1); expect(getFlowMetricsMock.mock.calls[0][0].filters).toEqual('SrcK8S_Name%3Dtest1%2Ctest2'); }); it('should generate AND groups', () => { - getTopoForFilter( - { - list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('src_namespace', [{ v: 'ns' }])], - match: 'all' - }, - false - ); + getTopoForFilter({ + list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('src_namespace', [{ v: 'ns' }])], + match: 'all' + }); expect(getFlowMetricsMock).toHaveBeenCalledTimes(1); expect(decodeURIComponent(getFlowMetricsMock.mock.calls[0][0].filters)).toEqual( 'SrcK8S_Name=test1,test2&SrcK8S_Namespace=ns' @@ -304,13 +208,10 @@ describe('Match all, topology', () => { }); it('should generate AND groups, back and forth', () => { - getTopoForFilter( - { - list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('dst_port', [{ v: '443' }])], - match: 'bidirectional' - }, - false - ); + getTopoForFilter({ + list: [filter('src_name', [{ v: 'test1' }, { v: 'test2' }]), filter('dst_port', [{ v: '443' }])], + match: 'bidirectional' + }); expect(getFlowMetricsMock).toHaveBeenCalledTimes(3); expect(decodeURIComponent(getFlowMetricsMock.mock.calls[0][0].filters)).toEqual( 'SrcK8S_Name=test1,test2&DstPort=443' diff --git a/web/src/utils/back-and-forth.ts b/web/src/utils/back-and-forth.ts index ec63aa869..759dfa425 100644 --- a/web/src/utils/back-and-forth.ts +++ b/web/src/utils/back-and-forth.ts @@ -1,73 +1,69 @@ import { FlowMetricsResult, RecordsResult } from '../api/loki'; import { getFlowMetrics, getFlowRecords } from '../api/routes'; -import { Filter, FilterDefinition, Filters } from '../model/filters'; -import { filtersToString, FlowQuery } from '../model/flow-query'; +import { Filter, FilterDefinition } from '../model/filters'; +import { filtersToString, StructuredFlowQuery, structuredToRawQuery } from '../model/flow-query'; import { computeStepInterval, TimeRange } from './datetime'; import { setEndpointFilters, swapFilters } from './filters-helper'; import { mergeStats, substractMetrics, sumMetrics } from './metrics'; -export const getFetchFunctions = (filterDefinitions: FilterDefinition[], filters: Filters, matchAny: boolean) => { - // check back-and-forth - if (filters.list.some(f => f.def.category === 'endpoint')) { - // set endpoint filters as source filters - const srcList = setEndpointFilters(filterDefinitions, filters.list, 'src'); - // set endpoint filters as dest filters - const dstList = setEndpointFilters(filterDefinitions, filters.list, 'dst'); - - return { - getRecords: (q: FlowQuery) => { - return getFlowsBNF(q, srcList, dstList, matchAny); - }, - getMetrics: (q: FlowQuery, range: number | TimeRange) => { - return getMetricsBNF(q, range, srcList, dstList, matchAny); +export const getFetchFunctions = (filterDefinitions: FilterDefinition[]) => { + return { + getRecords: (q: StructuredFlowQuery) => { + // check back-and-forth + if (q.structuredFilters.list.some(f => f.def.category === 'endpoint')) { + // set endpoint filters as source filters + const srcList = setEndpointFilters(filterDefinitions, q.structuredFilters.list, 'src'); + // set endpoint filters as dest filters + const dstList = setEndpointFilters(filterDefinitions, q.structuredFilters.list, 'dst'); + return getFlowsBNF(q, srcList, dstList); + } else if (q.structuredFilters.match === 'bidirectional') { + const swapped = swapFilters(filterDefinitions, q.structuredFilters.list); + if (swapped.length > 0) { + return getFlowsBNF(q, q.structuredFilters.list, swapped); + } } - }; - } else if (filters.match === 'bidirectional') { - let swapped = swapFilters(filterDefinitions, filters.list); - if (matchAny) { - // In match-any mode, remove non-swappable filters as they would result in duplicates - swapped = swapped.filter(f => f.def.id.startsWith('src_') || f.def.id.startsWith('dst_')); - } - if (swapped.length > 0) { - return { - getRecords: (q: FlowQuery) => { - return getFlowsBNF(q, filters.list, swapped, matchAny); - }, - getMetrics: (q: FlowQuery, range: number | TimeRange) => { - return getMetricsBNF(q, range, filters.list, swapped, matchAny); + const rawQ = structuredToRawQuery(q); + return getFlowRecords(rawQ); + }, + getMetrics: (q: StructuredFlowQuery, range: number | TimeRange) => { + // check back-and-forth + if (q.structuredFilters.list.some(f => f.def.category === 'endpoint')) { + // set endpoint filters as source filters + const srcList = setEndpointFilters(filterDefinitions, q.structuredFilters.list, 'src'); + // set endpoint filters as dest filters + const dstList = setEndpointFilters(filterDefinitions, q.structuredFilters.list, 'dst'); + return getMetricsBNF(q, range, srcList, dstList); + } else if (q.structuredFilters.match === 'bidirectional') { + const swapped = swapFilters(filterDefinitions, q.structuredFilters.list); + if (swapped.length > 0) { + return getMetricsBNF(q, range, q.structuredFilters.list, swapped); } - }; + } + const rawQ = structuredToRawQuery(q); + return getFlowMetrics(rawQ, range); } - } - return { - getRecords: getFlowRecords, - getMetrics: getFlowMetrics }; }; const encodedPipe = encodeURIComponent('|'); -const getFlowsBNF = ( - initialQuery: FlowQuery, - orig: Filter[], - swapped: Filter[], - matchAny: boolean -): Promise => { +const getFlowsBNF = (initialQuery: StructuredFlowQuery, orig: Filter[], swapped: Filter[]): Promise => { // Combine original filters and swapped. Note that we leave any potential overlapping flows: they can be deduped with "showDuplicates: false". + const matchAny = initialQuery.structuredFilters.match === 'any'; const newFilters = filtersToString(orig, matchAny) + encodedPipe + filtersToString(swapped, matchAny); return getFlowRecords({ ...initialQuery, filters: newFilters }); }; const getMetricsBNF = ( - initialQuery: FlowQuery, + initialQuery: StructuredFlowQuery, range: number | TimeRange, orig: Filter[], - swapped: Filter[], - matchAny: boolean + swapped: Filter[] ): Promise => { // When bnf is on, this replaces the usual getMetrics with a function with same arguments that runs 3 queries and merge their results // in order to get the ORIGINAL + SWAPPED - OVERLAP // OVERLAP being ORIGINAL AND SWAPPED. // E.g: if ORIGINAL is "SrcNs=foo", SWAPPED is "DstNs=foo" and OVERLAP is "SrcNs=foo AND DstNs=foo" + const matchAny = initialQuery.structuredFilters.match === 'any'; const overlapFilters = matchAny ? undefined : [...orig, ...swapped]; const promOrig = getFlowMetrics({ ...initialQuery, filters: filtersToString(orig, matchAny) }, range); const promSwapped = getFlowMetrics({ ...initialQuery, filters: filtersToString(swapped, matchAny) }, range); diff --git a/web/src/utils/metrics.ts b/web/src/utils/metrics.ts index 9ae730c63..166004abc 100644 --- a/web/src/utils/metrics.ts +++ b/web/src/utils/metrics.ts @@ -376,6 +376,11 @@ export const getFormattedValue = (v: number, mt: MetricType, mf: MetricFunction, return valueFormat(v, 1, t('P')); } return valueFormat(v, 1, t('Pps')); + case 'Flows': + if (mf !== 'rate') { + return valueFormat(v, 1, t('flows')); + } + return valueFormat(v, 1, t('fps')); default: return valueFormat(v, 1); } diff --git a/web/src/utils/netflow-capabilities-hook.ts b/web/src/utils/netflow-capabilities-hook.ts index 894ac28aa..821e66fd9 100644 --- a/web/src/utils/netflow-capabilities-hook.ts +++ b/web/src/utils/netflow-capabilities-hook.ts @@ -4,15 +4,7 @@ import { limitValues, topValues } from '../components/dropdowns/query-options-pa import { ViewId } from '../components/netflow-traffic'; import { Config } from '../model/config'; import { Filter, FilterDefinition, Filters, getEnabledFilters } from '../model/filters'; -import { - DataSource, - filtersToString, - FlowQuery, - FlowScope, - MetricType, - PacketLoss, - RecordType -} from '../model/flow-query'; +import { DataSource, FlowScope, MetricType, PacketLoss, RecordType, StructuredFlowQuery } from '../model/flow-query'; import { parseQuickFilters, QuickFilter } from '../model/quick-filters'; import { resolveGroupTypes, ScopeConfigDef } from '../model/scope'; import { TopologyOptions } from '../model/topology'; @@ -21,7 +13,7 @@ import { Column, ColumnsId } from './columns'; import { ContextSingleton } from './context'; import { computeStepInterval, TimeRange } from './datetime'; import { checkFilterAvailable, getFilterDefinitions } from './filter-definitions'; -import { dnsIdMatcher, droppedIdMatcher, OverviewPanel, rttIdMatcher } from './overview-panels'; +import { dnsIdMatcher, droppedIdMatcher, OverviewPanel, rttIdMatcher, tlsIdMatcher } from './overview-panels'; export interface ConfigCapabilities { allowLoki: boolean; @@ -31,6 +23,7 @@ export interface ConfigCapabilities { isDNSTracking: boolean; isFlowRTT: boolean; isPktDrop: boolean; + isTLSTracking: boolean; isPromOnly: boolean; availableScopes: ScopeConfigDef[]; allowedMetricTypes: MetricType[]; @@ -41,7 +34,7 @@ export interface ConfigCapabilities { filterDefs: FilterDefinition[]; quickFilters: QuickFilter[]; defaultFilters: Filter[]; - flowQuery: FlowQuery; + flowQuery: StructuredFlowQuery; fetchFunctions: ReturnType; } @@ -103,6 +96,8 @@ export function useConfigCapabilities(params: { const isPktDrop = React.useMemo(() => config.features.includes('pktDrop'), [config.features]); + const isTLSTracking = React.useMemo(() => config.features.includes('tlsTracking'), [config.features]); + const isPromOnly = React.useMemo(() => !allowLoki || dataSource === 'prom', [allowLoki, dataSource]); // Derived collections @@ -146,9 +141,10 @@ export function useConfigCapabilities(params: { panel => (isPktDrop || !panel.id.includes(droppedIdMatcher)) && (isDNSTracking || !panel.id.includes(dnsIdMatcher)) && - (isFlowRTT || !panel.id.includes(rttIdMatcher)) + (isFlowRTT || !panel.id.includes(rttIdMatcher)) && + (isTLSTracking || !panel.id.includes(tlsIdMatcher)) ), - [isDNSTracking, isFlowRTT, isPktDrop, panels] + [isDNSTracking, isFlowRTT, isPktDrop, isTLSTracking, panels] ); const selectedPanels = React.useMemo(() => availablePanels.filter(panel => panel.isSelected), [availablePanels]); @@ -189,11 +185,10 @@ export function useConfigCapabilities(params: { return quickFilters.filter(qf => qf.default).flatMap(qf => qf.filters); }, [forcedNamespace, quickFilters]); - const flowQuery = React.useMemo((): FlowQuery => { - const enabledFilters = getEnabledFilters(forcedFilters || filters); - const query: FlowQuery = { + const flowQuery = React.useMemo((): StructuredFlowQuery => { + const query: StructuredFlowQuery = { namespace: forcedNamespace, - filters: filtersToString(enabledFilters.list, enabledFilters.match === 'any'), + structuredFilters: getEnabledFilters(forcedFilters || filters), limit: limitValues.includes(limit) ? limit : limitValues[0], recordType: recordType, dataSource: dataSource, @@ -242,10 +237,8 @@ export function useConfigCapabilities(params: { ]); const fetchFunctions = React.useMemo(() => { - const enabledFilters = getEnabledFilters(forcedFilters || filters); - const matchAny = enabledFilters.match === 'any'; - return getBackAndForthFetch(filterDefs, enabledFilters, matchAny); - }, [forcedFilters, filters, filterDefs]); + return getBackAndForthFetch(filterDefs); + }, [filterDefs]); return { allowLoki, @@ -255,6 +248,7 @@ export function useConfigCapabilities(params: { isDNSTracking, isFlowRTT, isPktDrop, + isTLSTracking, isPromOnly, availableScopes, allowedMetricTypes, diff --git a/web/src/utils/overview-panels.ts b/web/src/utils/overview-panels.ts index c693a8402..43ea54a5b 100644 --- a/web/src/utils/overview-panels.ts +++ b/web/src/utils/overview-panels.ts @@ -5,6 +5,7 @@ import { Feature, isAllowed } from './features-gate'; export const dnsIdMatcher = 'dns_latency'; export const rttIdMatcher = 'rtt'; export const droppedIdMatcher = 'dropped'; +export const tlsIdMatcher = 'tls_'; export const customPanelMatcher = 'custom'; export const getRateFunctionFromId = (id: string) => { @@ -42,6 +43,10 @@ export type OverviewPanelId = | `cause_dropped_packet_rates` | 'name_dns_latency_flows' | 'rcode_dns_latency_flows' + | 'tls_usage_global' + | 'tls_per_cipher_suite' + | 'tls_per_version' + | 'tls_per_group' | `custom_${StatFunction}_${AggregateBy}_${MetricType}` | `custom_${AggregateBy}_${MetricType}` | `custom_${MetricType}`; @@ -76,11 +81,13 @@ export const defaultPanelIds: OverviewPanelId[] = [ 'rcode_dns_latency_flows', 'bottom_min_rtt', // should remove from defaults? 'top_avg_rtt', - 'top_p90_rtt' + 'top_p90_rtt', + 'tls_usage_global', + 'tls_per_version' ]; export const getDefaultOverviewPanels = (customIds?: string[]): OverviewPanel[] => { - let ids: OverviewPanelId[] = []; + let ids: OverviewPanelId[] = ['tls_usage_global', 'tls_per_version', 'tls_per_group', 'tls_per_cipher_suite']; /* list of panels and default selection behavior * isSelected can safely be used on feature related panels @@ -163,6 +170,7 @@ export const parseCustomMetricId = (id: string) => { switch (type) { case 'Flows': case 'DnsFlows': + case 'TlsFlows': fn = 'count'; break; case 'Bytes': @@ -325,6 +333,31 @@ export const getOverviewPanelInfo = ( 'The top DNS response code extracted from DNS response headers compared to total over the selected interval' ) }; + case 'tls_usage_global': + return { + title: t('TLS usage'), + topTitle: t('TLS usage (network flows per second)'), + subtitle: t('TLS traffic'), + chartType: t('donut or lines') + }; + case 'tls_per_version': + return { + title: t('TLS usage per version'), + topTitle: t('TLS per version (network flows per second)'), + chartType: t('donut or lines') + }; + case 'tls_per_group': + return { + title: t('TLS usage per group'), + topTitle: t('TLS per group (network flows per second)'), + chartType: t('donut or lines') + }; + case 'tls_per_cipher_suite': + return { + title: t('TLS usage per cipher suite'), + topTitle: t('TLS per cipher suite (network flows per second)'), + chartType: t('donut or lines') + }; default: const parsedId = parseCustomMetricId(id); if (parsedId.isValid) {