NETOBSERV-2608: TLS, metrics and charts [pf5]

config/sample-config.yaml

@@ -697,11 +697,11 @@ frontend:
       filter: tls_cipher_suite
       width: 15
       feature: tlsTracking
-    - id: TLSCurve
+    - id: TLSGroup
       group: Protocol Info
-      name: TLS Curve
-      field: TLSCurve
-      filter: tls_curve
+      name: TLS Group
+      field: TLSGroup
+      filter: tls_group
       width: 10
       feature: tlsTracking
     - id: TLSTypes
@@ -1411,11 +1411,11 @@ frontend:
       placeholder: 'E.g: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256'
       hint: Specify a TLS cipher suite.
       feature: tlsTracking
-    - id: tls_curve
-      name: TLS curve
+    - id: tls_group
+      name: TLS group
       component: text
       placeholder: 'E.g: X25519'
-      hint: Specify a TLS curve name.
+      hint: Specify a TLS group name.
       feature: tlsTracking
     - id: tls_types
       name: TLS packet type
@@ -1837,9 +1837,9 @@ frontend:
     - name: TLSCipherSuite
       type: string
       description: TLS cipher suite
-    - name: TLSCurve
+    - name: TLSGroup
       type: string
-      description: TLS curve name
+      description: TLS group name
     - name: Dscp
       type: number
       description: Differentiated Services Code Point (DSCP) value

pkg/loki/flow_query.go

@@ -126,25 +126,24 @@ func (q *FlowQueryBuilder) addLineFilters(filter filters.Match, values []string)
 		return
 	}
 
-	if q.config.IsArray(filter.Key) {
-		q.lineFilters = append(q.lineFilters, filters.ArrayLineFilter(filter.Key, values, filter.Not))
+	var lf filters.LineFilter
+	var hasEmptyMatch bool
+	switch {
+	case q.config.IsArray(filter.Key):
+		lf, hasEmptyMatch = filters.ArrayLineFilter(filter.Key, values, filter.Not)
+	case q.config.IsNumeric(filter.Key):
+		lf, hasEmptyMatch = filters.NumericLineFilter(filter.Key, values, filter.Not, filter.MoreThanOrEqual)
+	case filter.Regex:
+		lf, hasEmptyMatch = filters.StringLineFilterCheckExact(filter.Key, values, filter.Not)
+	default:
+		lf, hasEmptyMatch = filters.StringLineFilter(filter.Key, values, filter.Not)
+	}
+	// if there is at least an empty exact match, there is no uniform/safe way to filter by text,
+	// so we should use JSON label matchers instead of text line matchers
+	if hasEmptyMatch {
+		q.jsonFilters = append(q.jsonFilters, lf.AsLabelFilters())
 	} else {
-		var lf filters.LineFilter
-		var hasEmptyMatch bool
-		if q.config.IsNumeric(filter.Key) {
-			lf, hasEmptyMatch = filters.NumericLineFilter(filter.Key, values, filter.Not, filter.MoreThanOrEqual)
-		} else if filter.Regex {
-			lf, hasEmptyMatch = filters.StringLineFilterCheckExact(filter.Key, values, filter.Not)
-		} else {
-			lf, hasEmptyMatch = filters.StringLineFilter(filter.Key, values, filter.Not)
-		}
-		// if there is at least an empty exact match, there is no uniform/safe way to filter by text,
-		// so we should use JSON label matchers instead of text line matchers
-		if hasEmptyMatch {
-			q.jsonFilters = append(q.jsonFilters, lf.AsLabelFilters())
-		} else {
-			q.lineFilters = append(q.lineFilters, lf)
-		}
+		q.lineFilters = append(q.lineFilters, lf)
 	}
 }
 
@@ -225,6 +224,10 @@ func (q *FlowQueryBuilder) appendDNSFilter(sb *strings.Builder) {
 	sb.WriteString("`")
 }
 
+func (q *FlowQueryBuilder) appendTLSFilter(sb *strings.Builder) {
+	sb.WriteString(`|="TLSTypes"`)
+}
+
 func (q *FlowQueryBuilder) appendDNSLatencyFilter(sb *strings.Builder) {
 	// ensure DnsLatencyMs field is specified and value is not zero
 	// |~`"DnsLatencyMs`!~`DnsLatencyMs%22:0[,}]`

pkg/loki/topology_query.go

@@ -28,6 +28,15 @@ type TopologyInput struct {
 	Groups         string
 }
 
+func (in *TopologyInput) GetActualDataField() string {
+	switch in.DataField {
+	case constants.MetricTypeFlows, constants.MetricTypeDNSFlows, constants.MetricTypeTLSFlows:
+		return ""
+	default:
+		return in.DataField
+	}
+}
+
 type TopologyQueryBuilder struct {
 	*FlowQueryBuilder
 	topology           *TopologyInput
@@ -71,15 +80,6 @@ func GetLabelsAndFilter(kl map[string][]string, aggregate, groups string) ([]str
 	return fields, filter
 }
 
-func getField(metricType string) string {
-	switch metricType {
-	case constants.MetricTypeFlows, constants.MetricTypeDNSFlows:
-		return ""
-	default:
-		return metricType
-	}
-}
-
 func getFactor(metricType string) string {
 	switch metricType {
 	case constants.MetricTypeFlowRTT:
@@ -124,7 +124,6 @@ func (q *TopologyQueryBuilder) Build() string {
 	}
 	strLabels := strings.Join(labels, ",")
 
-	dataField := getField(q.topology.DataField)
 	factor := getFactor(q.topology.DataField)
 	function, quantile := GetFunctionWithQuantile(q.topology.MetricFunction)
 
@@ -171,15 +170,20 @@ func (q *TopologyQueryBuilder) Build() string {
 		q.appendFilter(sb, extraFilter)
 	}
 
-	if dataField == constants.MetricTypeDNSLatency {
+	switch q.topology.DataField {
+	case constants.MetricTypeDNSLatency:
 		q.appendDNSLatencyFilter(sb)
-	} else if dataField == constants.MetricTypeDNSFlows {
+	case constants.MetricTypeDNSFlows:
 		q.appendDNSFilter(sb)
-	} else if dataField == constants.MetricTypeFlowRTT {
+	case constants.MetricTypeTLSFlows:
+		q.appendTLSFilter(sb)
+	case constants.MetricTypeFlowRTT:
 		q.appendRTTFilter(sb)
 	}
 
 	q.appendJSON(sb, true)
+
+	dataField := q.topology.GetActualDataField()
 	if len(dataField) > 0 {
 		sb.WriteString("|unwrap ")
 		sb.WriteString(dataField)

pkg/model/filters/logql.go

@@ -271,12 +271,22 @@ func NumericLineFilter(key string, values []string, not, moreThan bool) (LineFil
 		typeNumber)
 }
 
-func ArrayLineFilter(key string, values []string, not bool) LineFilter {
+// ArrayLineFilter returns a LineFilter and true if it has an empty match
+func ArrayLineFilter(key string, values []string, not bool) (LineFilter, bool) {
+	if len(values) == 1 && (values[0] == "" || values[0] == `""`) {
+		// Special case, filter for n/a; note that logQL json doesn't support arrays, so we need a different solution
+		if not {
+			// key!=n/a
+			return RegexMatchLineFilter(key, true, ".*"), false
+		}
+		// key=n/a
+		return NotContainsKeyLineFilter(key), false
+	}
 	lf := LineFilter{key: key, not: not}
 	for _, value := range values {
 		lf.values = append(lf.values, lineMatch{valueType: typeRegexArrayContains, value: value})
 	}
-	return lf
+	return lf, false
 }
 
 // StringLineFilter returns a LineFilter and true if it has an empty match

pkg/prometheus/inventory.go

@@ -33,6 +33,12 @@ func NewInventory(cfg *config.Prometheus) *Inventory {
 			cpy.ValueField = constants.MetricTypeDNSFlows
 			toAppend = append(toAppend, cpy)
 		}
+		// Simulate Flows and TLSFlows value fields
+		if strings.Contains(cfg.Metrics[i].Name, "_tls_flows") {
+			cfg.Metrics[i].ValueField = constants.MetricTypeTLSFlows
+		} else if strings.Contains(cfg.Metrics[i].Name, "_flows") {
+			cfg.Metrics[i].ValueField = constants.MetricTypeFlows
+		}
 	}
 	return &Inventory{metrics: append(cfg.Metrics, toAppend...)}
 }

pkg/prometheus/inventory_test.go

@@ -4,10 +4,25 @@ import (
 	"testing"
 
 	"github.com/netobserv/network-observability-console-plugin/pkg/config"
+	"github.com/netobserv/network-observability-console-plugin/pkg/utils/constants"
 	"github.com/stretchr/testify/assert"
 )
 
 var configuredMetrics = []config.MetricInfo{
+	{
+		Enabled:   true,
+		Name:      "netobserv_metric_tls_flows_total",
+		Type:      "Counter",
+		Direction: config.AnyDirection,
+		Labels:    []string{"SrcK8S_Namespace", "DstK8S_Namespace"},
+	},
+	{
+		Enabled:   false,
+		Name:      "netobserv_metric_tls_flows_disabled",
+		Type:      "Counter",
+		Direction: config.AnyDirection,
+		Labels:    []string{"SrcK8S_Namespace", "DstK8S_Namespace"},
+	},
 	{
 		Enabled:    true,
 		Name:       "netobserv_metric_1",
@@ -81,14 +96,19 @@ func TestInventory_Search(t *testing.T) {
 	assert.Empty(t, search.Found)
 	assert.Empty(t, search.Candidates)
 
-	// Search flows metrics
+	// Search flows metrics (and TLS metric is skipped)
 	search = inv.Search([]string{"SrcK8S_Namespace", "DstK8S_Namespace"}, "")
 	assert.Equal(t, []string{"netobserv_metric_3"}, search.Found)
 	assert.Empty(t, search.Candidates)
 
 	search = inv.Search([]string{"SrcK8S_HostName", "DstK8S_HostName"}, "")
 	assert.Equal(t, []string{"netobserv_metric_3"}, search.Found)
 	assert.Empty(t, search.Candidates)
+
+	// Search TLS flows metrics
+	search = inv.Search([]string{"SrcK8S_Namespace", "DstK8S_Namespace"}, constants.MetricTypeTLSFlows)
+	assert.Equal(t, []string{"netobserv_metric_tls_flows_total"}, search.Found)
+	assert.Empty(t, search.Candidates)
 }
 
 func TestInventory_Search_RTT_Candidate(t *testing.T) {

pkg/utils/constants/constants.go

@@ -19,6 +19,7 @@ const (
 	MetricTypeDroppedPackets = "PktDropPackets"
 	MetricTypeDNSLatency     = "DnsLatencyMs"
 	MetricTypeDNSFlows       = "DnsFlows"
+	MetricTypeTLSFlows       = "TlsFlows"
 	MetricTypeFlowRTT        = "TimeFlowRttNs"
 	DefaultMetricType        = MetricTypeBytes
 

web/locales/en/plugin__netobserv-plugin.json

@@ -528,6 +528,7 @@
   "Show total": "Show total",
   "Show total dropped": "Show total dropped",
   "Show overall": "Show overall",
+  "%": "%",
   "Graph type": "Graph type",
   "Donut": "Donut",
   "Bars": "Bars",
@@ -649,6 +650,8 @@
   "internal": "internal",
   "P": "P",
   "Pps": "Pps",
+  "flows": "flows",
+  "fps": "fps",
   "minimum": "minimum",
   "maximum": "maximum",
   "90th percentile": "90th percentile",
@@ -697,6 +700,15 @@
   "Top {{limit}} DNS response code": "Top {{limit}} DNS response code",
   "Total DNS response code": "Total DNS response code",
   "The top DNS response code extracted from DNS response headers compared to total over the selected interval": "The top DNS response code extracted from DNS response headers compared to total over the selected interval",
+  "TLS usage": "TLS usage",
+  "TLS usage (network flows per second)": "TLS usage (network flows per second)",
+  "TLS traffic": "TLS traffic",
+  "TLS usage per version": "TLS usage per version",
+  "TLS per version (network flows per second)": "TLS per version (network flows per second)",
+  "TLS usage per group": "TLS usage per group",
+  "TLS per group (network flows per second)": "TLS per group (network flows per second)",
+  "TLS usage per cipher suite": "TLS usage per cipher suite",
+  "TLS per cipher suite (network flows per second)": "TLS per cipher suite (network flows per second)",
   "rates": "rates",
   "with total": "with total",
   "Invalid custom panel id": "Invalid custom panel id"

web/src/api/loki.ts

@@ -24,17 +24,17 @@ export interface StreamResult {
   values: string[][];
 }
 
-export class RecordsResult {
+export interface RecordsResult {
   records: Record[];
   stats: Stats;
 }
 
-export class FlowMetricsResult {
+export interface FlowMetricsResult {
   metrics: TopologyMetrics[];
   stats: Stats;
 }
 
-export class GenericMetricsResult {
+export interface GenericMetricsResult {
   metrics: GenericMetric[];
   stats: Stats;
 }
@@ -147,6 +147,11 @@ export type NetflowMetrics = {
   totalDnsLatency: Result<TotalFunctionMetrics, StructuredError | string>;
   totalDnsCount: Result<GenericMetric, StructuredError | string>;
   totalRtt: Result<TotalFunctionMetrics, StructuredError | string>;
+  tlsUsagePerVersion: Result<GenericMetric[], StructuredError | string>;
+  tlsUsagePerCipher: Result<GenericMetric[], StructuredError | string>;
+  tlsUsagePerGroup: Result<GenericMetric[], StructuredError | string>;
+  tlsFlowRate: Result<GenericMetric, StructuredError | string>;
+  totalFlowRate: Result<GenericMetric, StructuredError | string>;
   custom: Map<string, Result<TopologyMetrics[] | GenericMetric[], StructuredError | string>>;
   totalCustom: Map<string, Result<TopologyMetrics | GenericMetric, StructuredError | string>>;
 };
@@ -166,6 +171,11 @@ export const defaultNetflowMetrics: NetflowMetrics = {
   totalDnsLatency: Result.empty(),
   totalDnsCount: Result.empty(),
   totalRtt: Result.empty(),
+  tlsUsagePerCipher: Result.empty(),
+  tlsUsagePerGroup: Result.empty(),
+  tlsUsagePerVersion: Result.empty(),
+  tlsFlowRate: Result.empty(),
+  totalFlowRate: Result.empty(),
   custom: new Map(),
   totalCustom: new Map()
 };

web/src/api/routes.ts

@@ -1,7 +1,7 @@
 import axios from 'axios';
 import { Config, defaultConfig } from '../model/config';
 import { buildExportQuery } from '../model/export-query';
-import { FlowQuery, FlowScope, isTimeMetric } from '../model/flow-query';
+import { FlowQuery, FlowScope, isTimeMetric, StructuredFlowQuery, structuredToRawQuery } from '../model/flow-query';
 import { ContextSingleton } from '../utils/context';
 import { TimeRange } from '../utils/datetime';
 import { parseGenericMetrics, parseTopologyMetrics } from '../utils/metrics';
@@ -73,7 +73,8 @@ export const queryPrometheusMetric = (query: string): Promise<unknown> => {
   });
 };
 
-export const getExportFlowsURL = (params: FlowQuery, columns?: string[]): string => {
+export const getExportFlowsURL = (q: StructuredFlowQuery, columns?: string[]): string => {
+  const params = structuredToRawQuery(q);
   const exportQuery = buildExportQuery(params, columns);
   return `${ContextSingleton.getHost()}/api/loki/export?${exportQuery}`;
 };
@@ -169,7 +170,11 @@ export const getFlowMetrics = (params: FlowQuery, range: number | TimeRange): Pr
   });
 };
 
-export const getFlowGenericMetrics = (params: FlowQuery, range: number | TimeRange): Promise<GenericMetricsResult> => {
+export const getFlowGenericMetrics = (
+  q: StructuredFlowQuery,
+  range: number | TimeRange
+): Promise<GenericMetricsResult> => {
+  const params = structuredToRawQuery(q);
   return getFlowMetricsGeneric(params, res => {
     return parseGenericMetrics(
       res.result as RawTopologyMetrics[],

web/src/components/drawer/netflow-traffic-drawer.tsx

@@ -244,6 +244,7 @@ export const NetflowTrafficDrawer = React.forwardRef<NetflowTrafficDrawerHandle,
                   metrics={props.metrics}
                   loading={props.loading}
                   isDark={isDarkTheme}
+                  filterDefinitions={caps.filterDefs}
                   resetDefaultFilters={getResetDefaultFiltersProp()}
                   clearFilters={getClearFiltersProp()}
                   truncateLength={props.overviewTruncateLength}

web/src/components/messages/empty.tsx

@@ -45,7 +45,6 @@ export const Empty: React.FC<EmptyProps> = ({ showDetails, resetDefaultFilters,
     getStatus(ContextSingleton.getForcedNamespace())
       .then(status => {
         if (isMounted) {
-          console.info('status result', status);
           setStatus(status);
           setStatusError(undefined);
         }

web/src/components/messages/error.tsx

@@ -113,7 +113,6 @@ export const ErrorComponent: React.FC<ErrorProps> = ({ title, error }) => {
 
     getStatus(ContextSingleton.getForcedNamespace())
       .then(status => {
-        console.info('status result', status);
         setStatus(status);
         setStatusError(undefined);
       })