diff --git a/common/roles/partition-prometheus-rules/README.md b/common/roles/partition-prometheus-rules/README.md index 8fe0b932d..fd855a33e 100644 --- a/common/roles/partition-prometheus-rules/README.md +++ b/common/roles/partition-prometheus-rules/README.md @@ -14,3 +14,4 @@ Contains default variables for partition prometheus alert manager rules. This wa | partition_prometheus_rules_node_exporter | | node-exporter alert rules | | partition_prometheus_rules_pixiecore | | pixiecore alert rules | | partition_prometheus_rules_sonic_exporter | | sonic-exporter alert rules | +| partition_prometheus_rules_goflow2 | | goflow2 sFlow collector rules | diff --git a/common/roles/partition-prometheus-rules/defaults/main.yaml b/common/roles/partition-prometheus-rules/defaults/main.yaml index c4e846654..0b7ef019e 100644 --- a/common/roles/partition-prometheus-rules/defaults/main.yaml +++ b/common/roles/partition-prometheus-rules/defaults/main.yaml @@ -343,3 +343,38 @@ partition_prometheus_rules_sonic_exporter: | annotations: description: "Sonic switch {{$labels.instance}} in partition {{$externalLabels.partition}} could not be scraped for more than 10 minutes." {% endraw %} + +partition_prometheus_rules_sonic_systemd: | + {% raw %} + - name: sonic-systemd.rules + rules: + - record: sonic:systemd_service_uptime_seconds + expr: time() - node_systemd_unit_start_time_seconds{name=~"(swss|bgp).service"} + - alert: SonicServiceNotActive + expr: node_systemd_unit_state{name=~"(swss|bgp).service", state="active"} == 0 + for: 5m + labels: + severity: critical + annotations: + description: "SONiC service {{ $labels.name }} on {{ $labels.instance }} is not active." + - alert: SonicServiceRestarted + expr: changes(node_systemd_unit_start_time_seconds{name=~"(swss|bgp).service"}[8h]) > 0 + labels: + severity: critical + annotations: + description: "SONiC service {{ $labels.name }} on {{ $labels.instance }} has restarted in the last 8 hours." + {% endraw %} + +partition_prometheus_rules_goflow2: | + {% raw %} + - name: goflow2-collector.rules + rules: + - alert: GoflowCollectorDown + expr: up{job="goflow2"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "goflow2 sFlow collector down ({{ $labels.instance }})" + description: "goflow2 instance {{ $labels.instance }} has been unreachable for more than 2 minutes. sFlow ingestion is halted." + {% endraw %} diff --git a/control-plane/roles/monitoring/tasks/main.yaml b/control-plane/roles/monitoring/tasks/main.yaml index c6e6e0280..25d1f30ec 100644 --- a/control-plane/roles/monitoring/tasks/main.yaml +++ b/control-plane/roles/monitoring/tasks/main.yaml @@ -106,6 +106,8 @@ - rethinkdb.yaml - alertmanager.yaml - sonic-exporter.yaml + - goflow2-north-south.yaml + - goflow2-east-west.yaml when: monitoring_grafana_enabled - name: Deploy Grafana dashboards for Gardener diff --git a/control-plane/roles/monitoring/templates/grafana-dashboards/goflow2-east-west.yaml b/control-plane/roles/monitoring/templates/grafana-dashboards/goflow2-east-west.yaml new file mode 100644 index 000000000..d4102e7b2 --- /dev/null +++ b/control-plane/roles/monitoring/templates/grafana-dashboards/goflow2-east-west.yaml @@ -0,0 +1,644 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-goflow2-east-west + labels: + grafana_dashboard: "1" +data: + goflow2-east-west.json: | + {% raw %} + { + "annotations": { + "list": [] + }, + "editable": true, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "options": { + "displayLabels": [ + "percent", + "name" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "percent" + ] + }, + "pieType": "donut", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "sum by (vrf) (count_over_time({app=\"goflow2\", scope=\"internal\", partition=~\"$partition\", vrf!=\"\"} [$__range]))", + "legendFormat": "{{vrf}}", + "refId": "A" + } + ], + "title": "Traffic by Tenant (VRF)", + "type": "piechart" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 2, + "options": { + "displayLabels": [ + "percent", + "name" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "percent" + ] + }, + "pieType": "donut", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "sum by (vni) (count_over_time({app=\"goflow2\", scope=\"internal\", partition=~\"$partition\"} | vni != \"\" [$__range]))", + "legendFormat": "{{vni}}", + "refId": "A" + } + ], + "title": "Traffic by VNI", + "type": "piechart" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 3, + "maxDataPoints": 60, + "fieldConfig": { + "defaults": { + "custom": { + "lineInterpolation": "linear", + "fillOpacity": 80, + "showPoints": "never", + "spanNulls": true, + "stacking": { "group": "A", "mode": "normal" } + }, + "unit": "bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "calcs": [ + "mean", + "max" + ], + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "topk(10, sum by (vrf) (rate({app=\"goflow2\", scope=\"internal\", partition=~\"$partition\", vrf!=\"\"} | unwrap bytes [$__interval])) * 256 * 8)", + "legendFormat": "{{vrf}}", + "refId": "A" + } + ], + "title": "Bandwidth by Tenant (VRF)", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 4, + "fieldConfig": { + "defaults": { + "custom": { + "filterable": true + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "Flow Count", + "desc": true + } + ] + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "topk(10, sum by (src_addr) (count_over_time({app=\"goflow2\", scope=\"internal\", partition=~\"$partition\", vrf=~\"$vrf\"} | src_addr != \"\" [$__range])))", + "queryType": "instant", + "legendFormat": "", + "refId": "A" + } + ], + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "columns" + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "renameByName": { + "src_addr": "Source IP", + "Value #A": "Flow Count", + "Value": "Flow Count" + } + } + } + ], + "title": "Top 10 Source IPs (tenant)", + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 8 + }, + "id": 5, + "fieldConfig": { + "defaults": { + "custom": { + "filterable": true + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "Flow Count", + "desc": true + } + ] + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "topk(10, sum by (dst_addr) (count_over_time({app=\"goflow2\", scope=\"internal\", partition=~\"$partition\", vrf=~\"$vrf\"} | dst_addr != \"\" [$__range])))", + "queryType": "instant", + "legendFormat": "", + "refId": "A" + } + ], + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "columns" + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "renameByName": { + "dst_addr": "Destination IP", + "Value #A": "Flow Count", + "Value": "Flow Count" + } + } + } + ], + "title": "Top 10 Destination IPs (tenant)", + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 8 + }, + "id": 6, + "fieldConfig": { + "defaults": { + "custom": { + "filterable": true + } + }, + "overrides": [] + }, + "options": { + "showHeader": true, + "sortBy": [ + { + "displayName": "Flow Count", + "desc": true + } + ] + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "topk(10, sum by (src_addr, dst_addr) (count_over_time({app=\"goflow2\", scope=\"internal\", partition=~\"$partition\", vrf=~\"$vrf\"} | src_addr != \"\" | dst_addr != \"\" [$__range])))", + "queryType": "instant", + "legendFormat": "", + "refId": "A" + } + ], + "transformations": [ + { + "id": "labelsToFields", + "options": { + "mode": "columns" + } + }, + { + "id": "merge", + "options": {} + }, + { + "id": "organize", + "options": { + "renameByName": { + "src_addr": "Source IP", + "dst_addr": "Destination IP", + "Value #A": "Flow Count", + "Value": "Flow Count" + } + } + } + ], + "title": "Top 10 Conversations (tenant)", + "type": "table" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 7, + "options": { + "displayLabels": [ + "percent", + "name" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "percent" + ] + }, + "pieType": "donut", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "sum by (app_proto) (count_over_time({app=\"goflow2\", scope=\"internal\", partition=~\"$partition\", vrf=~\"$vrf\"} | app_proto!~\"UNCLASSIFIED|UNKNOWN|\" [$__range]))", + "legendFormat": "{{app_proto}}", + "refId": "A" + } + ], + "title": "Application Protocol Breakdown (tenant)", + "type": "piechart" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 8, + "options": { + "displayLabels": [ + "percent", + "name" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "percent" + ] + }, + "pieType": "donut", + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "sum by (sampler_address) (count_over_time({app=\"goflow2\", scope=\"internal\", partition=~\"$partition\", vrf=~\"$vrf\", sampler_address!=\"\"} [$__range]))", + "legendFormat": "{{sampler_address}}", + "refId": "A" + } + ], + "title": "Traffic by Sampler (Device)", + "type": "piechart" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 9, + "maxDataPoints": 60, + "fieldConfig": { + "defaults": { + "custom": { + "lineInterpolation": "linear", + "fillOpacity": 80, + "showPoints": "never", + "spanNulls": true, + "stacking": { "group": "A", "mode": "normal" } + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "calcs": [ + "mean", + "max" + ], + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "sum by (app_proto) (count_over_time({app=\"goflow2\", scope=\"internal\", partition=~\"$partition\", vrf=~\"$vrf\"} | app_proto!=\"\" | app_proto!~\"UNCLASSIFIED|UNKNOWN\" [$__interval]))", + "legendFormat": "{{app_proto}}", + "refId": "A" + } + ], + "title": "Protocol Traffic over Time (tenant)", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 10, + "maxDataPoints": 60, + "fieldConfig": { + "defaults": { + "custom": { + "lineInterpolation": "linear", + "fillOpacity": 80, + "showPoints": "never", + "spanNulls": true, + "stacking": { "group": "A", "mode": "normal" } + }, + "unit": "bps" + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "calcs": [ + "mean", + "max" + ], + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "expr": "sum by (app_proto) (rate({app=\"goflow2\", scope=\"internal\", partition=~\"$partition\", vrf=~\"$vrf\"} | app_proto!=\"\" | app_proto!~\"UNCLASSIFIED|UNKNOWN\" | unwrap bytes [$__interval])) * 256 * 8", + "legendFormat": "{{app_proto}}", + "refId": "A" + } + ], + "title": "Estimated Bandwidth over Time (tenant)", + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 39, + "tags": [ + "sflow", + "goflow2", + "vector", + "east-west", + "tenant" + ], + "templating": { + "list": [ + { + "hide": 0, + "includeAll": false, + "label": "Loki", + "multi": false, + "name": "DS_LOKI", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "definition": "label_values({app=\"goflow2\", scope=\"internal\"}, partition)", + "hide": 0, + "includeAll": true, + "allValue": ".*", + "label": "Partition", + "multi": true, + "name": "partition", + "options": [], + "query": "label_values({app=\"goflow2\", scope=\"internal\"}, partition)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "datasource": { + "type": "loki", + "uid": "${DS_LOKI}" + }, + "definition": "label_values({app=\"goflow2\", scope=\"internal\", partition=~\"$partition\"}, vrf)", + "hide": 0, + "includeAll": true, + "allValue": ".*", + "label": "Tenant (VRF)", + "multi": true, + "name": "vrf", + "options": [], + "query": "label_values({app=\"goflow2\", scope=\"internal\", partition=~\"$partition\"}, vrf)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "sFlow \u2014 East/West (Tenant)", + "uid": "goflow2-east-west", + "version": 1, + "weekStart": "" + } + {% endraw %} diff --git a/control-plane/roles/monitoring/templates/grafana-dashboards/goflow2-north-south.yaml b/control-plane/roles/monitoring/templates/grafana-dashboards/goflow2-north-south.yaml new file mode 100644 index 000000000..cb5608441 --- /dev/null +++ b/control-plane/roles/monitoring/templates/grafana-dashboards/goflow2-north-south.yaml @@ -0,0 +1,656 @@ +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboard-goflow2-north-south + labels: + grafana_dashboard: "1" +data: + goflow2-north-south.json: | + {% raw %} + { + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, + "id": 1, + "panels": [], + "title": "Collector Health", + "type": "row" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { "group": "A", "mode": "none" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + } + }, + "overrides": [] + }, + "gridPos": { "h": 7, "w": 12, "x": 0, "y": 1 }, + "id": 2, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "rate(goflow2_flow_process_sf_total{partition=~\"$partition\"}[5m])", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "sFlow Packets Received per Second", + "type": "timeseries" + }, + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "gridPos": { "h": 7, "w": 12, "x": 12, "y": 1 }, + "id": 3, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true }, + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "expr": "rate(goflow2_flow_process_sf_samples_total{type=\"unknown\", partition=~\"$partition\"}[5m])", + "legendFormat": "{{agent}}", + "refId": "A" + } + ], + "title": "Unknown/Unprocessable Samples per Second", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }, + "id": 4, + "panels": [], + "title": "Flow Analysis (Loki — enriched via Vector)", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 50 }, + "id": 23, + "panels": [], + "title": "IP & Protocol Top Talkers", + "type": "row" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { "h": 12, "w": 8, "x": 0, "y": 51 }, + "id": 5, + "fieldConfig": { + "defaults": { "custom": { "align": "auto", "displayMode": "auto" } }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Flow Count" }, + "properties": [{ "id": "custom.width", "value": 120 }] + } + ] + }, + "options": { "showHeader": true, "sortBy": [{ "displayName": "Flow Count", "desc": true }] }, + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } }, + { "id": "merge", "options": {} }, + { + "id": "organize", + "options": { + "excludeByName": { "Time": true }, + "renameByName": { "src_addr": "Source IP", "Value": "Flow Count", "Value #A": "Flow Count" } + } + }, + { "id": "sortBy", "options": { "fields": [{ "displayName": "Flow Count", "desc": true }] } } + ], + "targets": [ + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "topk(10, sum by (src_addr) (count_over_time({app=\"goflow2\", partition=~\"$partition\", scope=\"inbound\"} | app_proto != \"\" | src_addr != \"\" [$__range])))", + "legendFormat": "{{src_addr}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top 10 Source IPs", + "type": "table" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { "h": 12, "w": 8, "x": 8, "y": 51 }, + "id": 15, + "fieldConfig": { + "defaults": { "custom": { "align": "auto", "displayMode": "auto" } }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Flow Count" }, + "properties": [{ "id": "custom.width", "value": 120 }] + } + ] + }, + "options": { "showHeader": true, "sortBy": [{ "displayName": "Flow Count", "desc": true }] }, + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } }, + { "id": "merge", "options": {} }, + { + "id": "organize", + "options": { + "excludeByName": { "Time": true }, + "renameByName": { "dst_addr": "Destination IP", "Value": "Flow Count", "Value #A": "Flow Count" } + } + }, + { "id": "sortBy", "options": { "fields": [{ "displayName": "Flow Count", "desc": true }] } } + ], + "targets": [ + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "topk(10, sum by (dst_addr) (count_over_time({app=\"goflow2\", partition=~\"$partition\", scope=\"inbound\"} | app_proto != \"\" | dst_addr != \"\" [$__range])))", + "legendFormat": "{{dst_addr}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top 10 Destination IPs", + "type": "table" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { "h": 12, "w": 8, "x": 16, "y": 51 }, + "id": 16, + "fieldConfig": { + "defaults": { "custom": { "align": "auto", "displayMode": "auto" } }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Flow Count" }, + "properties": [{ "id": "custom.width", "value": 120 }] + } + ] + }, + "options": { "showHeader": true, "sortBy": [{ "displayName": "Flow Count", "desc": true }] }, + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } }, + { "id": "merge", "options": {} }, + { + "id": "organize", + "options": { + "excludeByName": { "Time": true }, + "renameByName": { "src_addr": "Source IP", "dst_addr": "Destination IP", "Value": "Flow Count", "Value #A": "Flow Count" } + } + }, + { "id": "sortBy", "options": { "fields": [{ "displayName": "Flow Count", "desc": true }] } } + ], + "targets": [ + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "topk(10, sum by (src_addr, dst_addr) (count_over_time({app=\"goflow2\", partition=~\"$partition\", scope=\"inbound\"} | app_proto != \"\" | src_addr != \"\" | dst_addr != \"\" [$__range])))", + "legendFormat": "{{src_addr}} → {{dst_addr}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top 10 Conversations", + "type": "table" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 63 }, + "id": 7, + "options": { + "displayLabels": ["percent", "name"], + "legend": { "displayMode": "table", "placement": "right", "calcs": ["percent"] }, + "pieType": "donut", + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "sum by (app_proto) (count_over_time({app=\"goflow2\", partition=~\"$partition\", scope=\"inbound\"} | app_proto!~\"UNCLASSIFIED|UNKNOWN|\" [$__range]))", + "legendFormat": "{{app_proto}}", + "refId": "A" + } + ], + "title": "Application Protocol Breakdown", + "type": "piechart" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { "h": 8, "w": 8, "x": 8, "y": 63 }, + "id": 17, + "options": { + "displayLabels": ["percent", "name"], + "legend": { "displayMode": "table", "placement": "right", "calcs": ["percent"] }, + "pieType": "donut", + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "sum by (country_code) (count_over_time({app=\"goflow2-geo\", partition=~\"$partition\"} | json country_code [$__range]))", + "legendFormat": "{{country_code}}", + "refId": "A" + } + ], + "title": "Traffic by Country", + "type": "piechart" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { "h": 8, "w": 8, "x": 16, "y": 63 }, + "id": 9, + "options": { + "displayLabels": ["percent", "name"], + "legend": { "displayMode": "table", "placement": "right", "calcs": ["percent"] }, + "pieType": "donut", + "tooltip": { "mode": "single", "sort": "none" } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "sum by (sampler_address) (count_over_time({app=\"goflow2\", partition=~\"$partition\", sampler_address!=\"\", scope=\"inbound\"} [$__range]))", + "legendFormat": "{{sampler_address}}", + "refId": "A" + } + ], + "title": "Traffic by Sampler (Device)", + "type": "piechart" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "flows", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 71 }, + "id": 12, + "maxDataPoints": 60, + "options": { + "legend": { "calcs": ["sum", "max"], "displayMode": "table", "placement": "right", "sortBy": "Max", "sortDesc": true, "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "sum by (app_proto) (count_over_time({app=\"goflow2\", partition=~\"$partition\", scope=\"inbound\"} | app_proto!=\"\" | app_proto!~\"UNCLASSIFIED|UNKNOWN\" [$__interval]))", + "legendFormat": "{{app_proto}}", + "refId": "A" + } + ], + "title": "Protocol Traffic over Time", + "type": "timeseries" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { "h": 8, "w": 24, "x": 0, "y": 79 }, + "id": 13, + "maxDataPoints": 60, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "sortBy": "Max", "sortDesc": true, "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "sum by (app_proto) (rate({app=\"goflow2\", partition=~\"$partition\", scope=\"inbound\"} | app_proto!=\"\" | app_proto!~\"UNCLASSIFIED|UNKNOWN\" | unwrap bytes [$__interval])) * 256 * 8", + "legendFormat": "{{app_proto}}", + "refId": "A" + } + ], + "title": "Estimated Bandwidth by Protocol", + "type": "timeseries" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { "h": 28, "w": 24, "x": 0, "y": 9 }, + "id": 14, + "fieldConfig": { + "defaults": { + "color": { "mode": "continuous-RdYlGr" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 10 }, + { "color": "orange", "value": 50 }, + { "color": "red", "value": 200 } + ] + } + }, + "overrides": [] + }, + "options": { + "view": { "id": "coords", "lat": 48.1374, "lon": 11.5755, "zoom": 3 }, + "controls": { "showZoom": true, "showAttribution": true }, + "layers": [ + { + "type": "heatmap", + "name": "Source density", + "config": { + "weight": { "field": "Value #A", "fixed": 1, "min": 0, "max": 1 }, + "radius": 25, + "blur": 15, + "opacity": 0.9 + }, + "location": { + "mode": "coords", + "latitude": "latitude", + "longitude": "longitude" + }, + "tooltip": false + } + ] + }, + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } }, + { "id": "merge", "options": {} }, + { "id": "convertFieldType", "options": { "conversions": [ + { "targetField": "latitude", "destinationType": "number" }, + { "targetField": "longitude", "destinationType": "number" } + ]}} + ], + "targets": [ + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "sum by (latitude, longitude) (count_over_time({app=\"goflow2-geo\", partition=~\"$partition\"} | json latitude, longitude [$__range]))", + "legendFormat": "{{country_code}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Source IP Geolocation", + "description": "GeoIP by DB-IP, CC BY 4.0.", + "type": "geomap" + }, + { + "collapsed": false, + "gridPos": { "h": 1, "w": 24, "x": 0, "y": 37 }, + "id": 19, + "panels": [], + "title": "ASN Top Talkers (DB-IP)", + "type": "row" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "gridPos": { "h": 12, "w": 8, "x": 0, "y": 38 }, + "id": 20, + "fieldConfig": { + "defaults": { "custom": { "align": "auto", "displayMode": "auto" } }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Flow Count" }, + "properties": [{ "id": "custom.width", "value": 120 }] + } + ] + }, + "options": { "showHeader": true, "sortBy": [{ "displayName": "Flow Count", "desc": true }] }, + "transformations": [ + { "id": "labelsToFields", "options": { "mode": "columns" } }, + { "id": "merge", "options": {} }, + { + "id": "organize", + "options": { + "excludeByName": { "Time": true }, + "renameByName": { + "src_asn": "ASN", + "src_asn_org": "Organization", + "Value": "Flow Count", + "Value #A": "Flow Count" + } + } + }, + { "id": "sortBy", "options": { "fields": [{ "displayName": "Flow Count", "desc": true }] } } + ], + "targets": [ + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "topk(10, sum by (src_asn, src_asn_org) (count_over_time({app=\"goflow2\", partition=~\"$partition\", scope=\"inbound\"} | src_asn_org != \"PRIVATE\" | src_asn_org != \"UNKNOWN\" | src_asn_org != \"\" [$__range])))", + "legendFormat": "{{src_asn_org}}", + "queryType": "instant", + "refId": "A" + } + ], + "title": "Top 10 Source ASNs", + "description": "ASN data by DB-IP, CC BY 4.0.", + "type": "table" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "flows", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] } + }, + "overrides": [] + }, + "gridPos": { "h": 12, "w": 8, "x": 8, "y": 38 }, + "id": 21, + "maxDataPoints": 60, + "options": { + "legend": { "calcs": ["sum", "max"], "displayMode": "table", "placement": "right", "sortBy": "Max", "sortDesc": true, "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "topk(10, sum by (src_asn_org) (count_over_time({app=\"goflow2\", partition=~\"$partition\", scope=\"inbound\"} | src_asn_org != \"PRIVATE\" | src_asn_org != \"UNKNOWN\" | src_asn_org != \"\" [$__interval])))", + "legendFormat": "{{src_asn_org}}", + "refId": "A" + } + ], + "title": "Top ASN Traffic over Time", + "type": "timeseries" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + }, + "mappings": [], + "thresholds": { "mode": "absolute", "steps": [{ "color": "green", "value": null }] }, + "unit": "bps" + }, + "overrides": [] + }, + "gridPos": { "h": 12, "w": 8, "x": 16, "y": 38 }, + "id": 22, + "maxDataPoints": 60, + "options": { + "legend": { "calcs": ["mean", "max"], "displayMode": "table", "placement": "right", "sortBy": "Max", "sortDesc": true, "showLegend": true }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "targets": [ + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "expr": "topk(10, sum by (src_asn_org) (rate({app=\"goflow2\", partition=~\"$partition\", scope=\"inbound\"} | src_asn_org != \"PRIVATE\" | src_asn_org != \"UNKNOWN\" | src_asn_org != \"\" | unwrap bytes [$__interval])) * 256 * 8)", + "legendFormat": "{{src_asn_org}}", + "refId": "A" + } + ], + "title": "Estimated Bandwidth by Source ASN", + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 38, + "style": "dark", + "tags": ["sflow", "goflow2", "vector", "north-south"], + "templating": { + "list": [ + { + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "hide": 0, + "includeAll": false, + "label": "Loki", + "multi": false, + "name": "DS_LOKI", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "datasource": { "type": "loki", "uid": "${DS_LOKI}" }, + "definition": "label_values({app=\"goflow2\"}, partition)", + "hide": 0, + "includeAll": true, + "allValue": ".*", + "label": "Partition", + "multi": true, + "name": "partition", + "options": [], + "query": "label_values({app=\"goflow2\"}, partition)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "timepicker": {}, + "timezone": "", + "title": "sFlow — North/South (Edge)", + "uid": "goflow2-north-south", + "version": 10 + } + {% endraw %} diff --git a/control-plane/roles/monitoring/templates/thanos-values.yaml b/control-plane/roles/monitoring/templates/thanos-values.yaml index 0b5bc8a65..ef2c66421 100644 --- a/control-plane/roles/monitoring/templates/thanos-values.yaml +++ b/control-plane/roles/monitoring/templates/thanos-values.yaml @@ -57,6 +57,8 @@ ruler: {{ lookup('ansible.builtin.vars', 'partition_prometheus_rules_node_exporter') | indent(width=8, first=false) }} {{ lookup('ansible.builtin.vars', 'partition_prometheus_rules_pixiecore') | indent(width=8, first=false) }} {{ lookup('ansible.builtin.vars', 'partition_prometheus_rules_sonic_exporter') | indent(width=8, first=false) }} + {{ lookup('ansible.builtin.vars', 'partition_prometheus_rules_sonic_systemd') | indent(width=8, first=false) }} + {{ lookup('ansible.builtin.vars', 'partition_prometheus_rules_goflow2') | indent(width=8, first=false) }} `{{ '}}' }} {% endif %} diff --git a/partition/roles/monitoring/node-exporter/defaults/main.yaml b/partition/roles/monitoring/node-exporter/defaults/main.yaml index ae8d854b4..d72410d50 100644 --- a/partition/roles/monitoring/node-exporter/defaults/main.yaml +++ b/partition/roles/monitoring/node-exporter/defaults/main.yaml @@ -4,3 +4,4 @@ monitoring_node_exporter_image_tag: latest monitoring_node_exporter_port: 9100 monitoring_node_exporter_dir: /node_exporter monitoring_node_exporter_docker_log_driver: json-file +monitoring_node_exporter_systemd_unit: "(bgp|swss)\\.service" diff --git a/partition/roles/monitoring/node-exporter/tasks/main.yaml b/partition/roles/monitoring/node-exporter/tasks/main.yaml index 658a5bbc6..acfd81578 100644 --- a/partition/roles/monitoring/node-exporter/tasks/main.yaml +++ b/partition/roles/monitoring/node-exporter/tasks/main.yaml @@ -22,8 +22,12 @@ target_port: "{{ monitoring_node_exporter_port }}" systemd_docker_volumes: - "/:/host:ro,rslave" + - "/var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket" systemd_docker_command: - --path.rootfs=/host - --path.procfs=/host/proc - --path.sysfs=/host/sys + - --collector.systemd + - --collector.systemd.enable-start-time-metrics + - "--collector.systemd.unit-include={{ monitoring_node_exporter_systemd_unit }}" systemd_docker_log_driver: "{{ monitoring_node_exporter_docker_log_driver }}" diff --git a/partition/roles/monitoring/prometheus/defaults/main.yaml b/partition/roles/monitoring/prometheus/defaults/main.yaml index ff4dae6e7..55f3f97a8 100644 --- a/partition/roles/monitoring/prometheus/defaults/main.yaml +++ b/partition/roles/monitoring/prometheus/defaults/main.yaml @@ -31,6 +31,7 @@ prometheus_ping_targets: [] prometheus_ipmi_exporter_targets: [] prometheus_sonic_exporter_targets: [] prometheus_blackbox_exporter_targets: [] +prometheus_goflow2_targets: [] prometheus_lightbox_exporter_targets: [] prometheus_lightos_smart_targets: [] prometheus_hosts_content: diff --git a/partition/roles/monitoring/prometheus/tasks/main.yaml b/partition/roles/monitoring/prometheus/tasks/main.yaml index c86a0fe31..44e0f3228 100644 --- a/partition/roles/monitoring/prometheus/tasks/main.yaml +++ b/partition/roles/monitoring/prometheus/tasks/main.yaml @@ -178,6 +178,20 @@ mode: "0644" notify: Restart prometheus +- name: Generate prometheus_goflow2_sd + ansible.builtin.set_fact: + prometheus_goflow2_sd: + - labels: + job: goflow2 + targets: "{{ prometheus_goflow2_targets }}" + +- name: Create goflow2.yaml + ansible.builtin.copy: + content: "{{ prometheus_goflow2_sd | to_nice_yaml(indent=2) }}" + dest: "{{ prometheus_config_host_dir }}/file_sd/goflow2.yaml" + mode: "0644" + notify: Restart prometheus + - name: Ensure rules directory ansible.builtin.file: path: "{{ prometheus_config_host_dir }}/rules" @@ -206,6 +220,10 @@ content_var: partition_prometheus_rules_pixiecore - file: sonic-exporter.yaml content_var: partition_prometheus_rules_sonic_exporter + - file: sonic-systemd.yaml + content_var: partition_prometheus_rules_sonic_systemd + - file: goflow2.yaml + content_var: partition_prometheus_rules_goflow2 notify: Restart prometheus - name: Deploy prometheus service diff --git a/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2 b/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2 index d0f0b1517..7147525f3 100644 --- a/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2 +++ b/partition/roles/monitoring/prometheus/templates/prometheus.yaml.j2 @@ -106,6 +106,15 @@ scrape_configs: regex: (.*):\d+ target_label: instance +- job_name: goflow2 + file_sd_configs: + - files: + - '/etc/prometheus/file_sd/goflow2.yaml' + relabel_configs: + - source_labels: [__address__] + regex: (.*):\d+ + target_label: instance + - job_name: blackbox-exporter metrics_path: /metrics static_configs: diff --git a/partition/roles/promtail/handlers/main.yaml b/partition/roles/promtail/handlers/main.yaml index 4de0eff23..775265553 100644 --- a/partition/roles/promtail/handlers/main.yaml +++ b/partition/roles/promtail/handlers/main.yaml @@ -4,3 +4,7 @@ name: promtail enabled: true state: restarted + +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true diff --git a/partition/roles/sflow-collector/defaults/main.yaml b/partition/roles/sflow-collector/defaults/main.yaml new file mode 100644 index 000000000..0c70075f8 --- /dev/null +++ b/partition/roles/sflow-collector/defaults/main.yaml @@ -0,0 +1,34 @@ +--- +sflow_collector_profile: combined + +sflow_collector_port: 6343 +sflow_collector_metrics_port: 9099 +sflow_collector_internal_port: 9001 + +sflow_collector_image_name: netsampler/goflow2 +sflow_collector_image_tag: latest + +sflow_collector_vector_image_name: timberio/vector +sflow_collector_vector_image_tag: 0.55.0-distroless-libc + +sflow_collector_vector_geo_sample_rate: 10 + +sflow_collector_config_host_dir: /etc/sflow-collector +sflow_collector_data_host_dir: /var/lib/sflow-collector + +sflow_collector_loki_endpoint: +sflow_collector_loki_basic_auth_user: +sflow_collector_loki_basic_auth_password: +sflow_collector_loki_tls_verify: true + +sflow_collector_geoip_download_image: alpine:3 + +sflow_collector_docker_log_driver: json-file + +sflow_collector_spool_tmpfs_size: 256m + +sflow_collector_logrotate_size: 32M +sflow_collector_logrotate_keep: 1 +sflow_collector_logrotate_interval: 2min + +sflow_collector_vector_buffer_max_events: 100000 diff --git a/partition/roles/sflow-collector/files/sflow-mmdb-refresh.service b/partition/roles/sflow-collector/files/sflow-mmdb-refresh.service new file mode 100644 index 000000000..5b6a67121 --- /dev/null +++ b/partition/roles/sflow-collector/files/sflow-mmdb-refresh.service @@ -0,0 +1,25 @@ +[Unit] +Description=Refresh GeoIP and ASN mmdb files for the sFlow Vector pipeline +After=docker.service network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/sflow-mmdb-refresh.sh geoip +ExecStart=/usr/local/bin/sflow-mmdb-refresh.sh asn + +# Hardening as root is required to interact with the vector service, +# These heavily limit what root can do +NoNewPrivileges=true +ProtectSystem=strict +ReadWritePaths=/var/lib/sflow-collector /tmp +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectKernelLogs=true +ProtectControlGroups=true +ProtectClock=true +ProtectHostname=true +LockPersonality=true +RestrictSUIDSGID=true +RestrictRealtime=true +RestrictNamespaces=true diff --git a/partition/roles/sflow-collector/files/sflow-mmdb-refresh.sh b/partition/roles/sflow-collector/files/sflow-mmdb-refresh.sh new file mode 100644 index 000000000..bc86ef0b9 --- /dev/null +++ b/partition/roles/sflow-collector/files/sflow-mmdb-refresh.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Downloads a DP-IP Database and replaces the existing one. +# DB-IP updates monthly, so the current month is attempted first +# If the DB for the current month is not available yet, the scripts +# falls back to last month. + +set -euo pipefail + +KIND="${1:-}" +DATA_DIR="/var/lib/sflow-collector" +IMAGE="alpine:3" + +case "$KIND" in + geoip) + SLUG="dbip-city-lite" + OUT_DIR="${DATA_DIR}/geoip" + OUT_FILE="geoip.mmdb" + ;; + asn) + SLUG="dbip-asn-lite" + OUT_DIR="${DATA_DIR}/asn" + OUT_FILE="asn.mmdb" + ;; + *) + echo "usage: $0 {geoip|asn}" >&2 + exit 2 + ;; +esac + +mkdir -p "$OUT_DIR" + +YEAR=$(date +%Y) +MONTH=$(date +%m) +PREV_MONTH=$((10#$MONTH - 1)) +PREV_YEAR="$YEAR" +if [ "$PREV_MONTH" -eq 0 ]; then PREV_MONTH=12; PREV_YEAR=$((YEAR - 1)); fi +PREV_MONTH=$(printf '%02d' "$PREV_MONTH") + +TMP="$(mktemp -d)" +trap 'rm -rf "$TMP"' EXIT + +for URL in \ + "https://download.db-ip.com/free/${SLUG}-${YEAR}-${MONTH}.mmdb.gz" \ + "https://download.db-ip.com/free/${SLUG}-${PREV_YEAR}-${PREV_MONTH}.mmdb.gz"; do + echo "Trying $URL ..." + if docker run --rm --network host -v "$TMP:/out" "$IMAGE" \ + sh -c "wget -q -O /out/db.gz '$URL' && gunzip /out/db.gz"; then + mv -f "$TMP/db" "$OUT_DIR/$OUT_FILE.new" + chmod 644 "$OUT_DIR/$OUT_FILE.new" + mv -f "$OUT_DIR/$OUT_FILE.new" "$OUT_DIR/$OUT_FILE" + echo "Updated $OUT_DIR/$OUT_FILE" + # Reload Vector so the new mmdb is mapped. Restart is acceptable monthly. + if systemctl is-active --quiet vector; then + systemctl restart vector + fi + exit 0 + fi +done + +echo "ERROR: failed to refresh $KIND mmdb from both current and previous month" >&2 +exit 1 diff --git a/partition/roles/sflow-collector/files/sflow-mmdb-refresh.timer b/partition/roles/sflow-collector/files/sflow-mmdb-refresh.timer new file mode 100644 index 000000000..d7a30fa1e --- /dev/null +++ b/partition/roles/sflow-collector/files/sflow-mmdb-refresh.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Monthly refresh of DB-IP GeoIP and ASN mmdb files + +[Timer] +OnCalendar=monthly +RandomizedDelaySec=3600 +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/partition/roles/sflow-collector/files/vrf-port-export.py b/partition/roles/sflow-collector/files/vrf-port-export.py new file mode 100644 index 000000000..0c28b7cd7 --- /dev/null +++ b/partition/roles/sflow-collector/files/vrf-port-export.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +import csv +import os +import subprocess +import time +import redis + +CSV_FILE = "/var/lib/sflow-collector/ifindex.csv" +CSV_FILE_TMP = CSV_FILE + ".tmp" +CSV_FIELDS = ["sampler_address", "ifindex", "interface", "vrf", "vni"] +INTERVAL = 60 +CONFIG_DB = 4 + + +def get_ifindex(interface): + try: + return int(open(f"/sys/class/net/{interface}/ifindex").read().strip()) + except (OSError, ValueError): + return None + + +# The sflow packets contain the sampler address used to send the flow sample to the collector. +# This IP can be used to match the Interface to VRF mapping to a flow sample. +def get_sampler_address(db): + for key in db.keys("SFLOW_COLLECTOR|*"): + collector_ip = db.hget(key, "collector_ip") + if not collector_ip: + continue + try: + tokens = subprocess.run( + ["ip", "route", "get", collector_ip], + capture_output=True, text=True, timeout=5, + ).stdout.split() + if "src" in tokens: + return tokens[tokens.index("src") + 1] + except Exception: + pass + return None + + +def get_vni(db, vrf): + return db.hget(f"VRF|{vrf}", "vni") + + +def get_vrf_mappings(db, sampler_address): + mappings = [] + + for key in db.keys("INTERFACE|*"): + try: + _, interface = key.split("|") + except ValueError: + continue + vrf = db.hget(key, "vrf_name") + ifindex = get_ifindex(interface) + if vrf and ifindex is not None: + vni = get_vni(db, vrf) + mappings.append({"interface": interface, "vrf": vrf, "vni": vni, "ifindex": ifindex, "sampler_address": sampler_address}) + + for key in db.keys("VLAN_MEMBER|*"): + try: + _, vlan, interface = key.split("|") + except ValueError: + continue + vrf = db.hget(f"VLAN_INTERFACE|{vlan}", "vrf_name") + ifindex = get_ifindex(interface) + if vrf and ifindex is not None: + vni = get_vni(db, vrf) + mappings.append({"interface": interface, "vlan": vlan, "vrf": vrf, "vni": vni, "ifindex": ifindex, "sampler_address": sampler_address}) + + return mappings + + +def write_csv(mappings): + # Vector's ifindex_vrf enrichment table is keyed on (sampler_address, ifindex); + # Duplicates may appear on hybrid ports. Not expected in production. + deduped = {} + for entry in mappings: + key = (entry.get("sampler_address") or "", entry.get("ifindex")) + deduped[key] = entry + try: + with open(CSV_FILE_TMP, "w", newline="") as csv_fp: + writer = csv.DictWriter(csv_fp, fieldnames=CSV_FIELDS, extrasaction="ignore") + writer.writeheader() + for entry in deduped.values(): + writer.writerow({k: ("" if entry.get(k) is None else entry.get(k)) for k in CSV_FIELDS}) + os.replace(CSV_FILE_TMP, CSV_FILE) + except OSError as exc: + print(f"vrf-port-export: skipping CSV write: {exc}", flush=True) + + +def main(): + db = redis.Redis(host="127.0.0.1", port=6379, db=CONFIG_DB, decode_responses=True) + + while True: + sampler_address = get_sampler_address(db) + write_csv(get_vrf_mappings(db, sampler_address)) + time.sleep(INTERVAL) + + +if __name__ == "__main__": + main() diff --git a/partition/roles/sflow-collector/files/vrf-port-export.service b/partition/roles/sflow-collector/files/vrf-port-export.service new file mode 100644 index 000000000..7866480af --- /dev/null +++ b/partition/roles/sflow-collector/files/vrf-port-export.service @@ -0,0 +1,12 @@ +[Unit] +Description=VRF/VNI to interface mapping exporter for the sFlow collector (ifindex enrichment) +After=network.target + +[Service] +Type=simple +ExecStart=/usr/bin/python3 /usr/local/bin/vrf-port-export.py +Restart=on-failure +RestartSec=10 + +[Install] +WantedBy=multi-user.target diff --git a/partition/roles/sflow-collector/files/well_known_ports.csv b/partition/roles/sflow-collector/files/well_known_ports.csv new file mode 100644 index 000000000..698de05e1 --- /dev/null +++ b/partition/roles/sflow-collector/files/well_known_ports.csv @@ -0,0 +1,243 @@ +port,protocol +1,TCPMUX +5,RJE +7,ECHO +9,DISCARD +11,SYSTAT +13,DAYTIME +17,QOTD +18,MSP +19,CHARGEN +20,FTP-DATA +21,FTP +22,SSH +23,TELNET +25,SMTP +26,RSFTP +37,TIME +42,WINS +43,WHOIS +49,TACACS +53,DNS +57,MTP +67,DHCP +68,DHCP +69,TFTP +70,GOPHER +79,FINGER +80,HTTP +88,KERBEROS +101,HOSTNAME +102,TSAP +107,RTELNET +109,POP2 +110,POP3 +111,SUNRPC +113,IDENT +115,SFTP +117,UUCP +118,SQL +119,NNTP +123,NTP +135,EPMAP +137,NETBIOS-NS +138,NETBIOS-DGM +139,NETBIOS-SSN +143,IMAP +161,SNMP +162,SNMPTRAP +170,PRINT-SRV +179,BGP +194,IRC +209,QMTP +213,IPX +220,IMAP3 +264,BGMP +311,APPLE-ADMIN +318,TSP +366,ODMR +389,LDAP +401,UPS +427,SLP +443,HTTPS +444,SNPP +445,SMB +464,KERBEROS-CHG +465,SMTPS +500,ISAKMP +502,MODBUS +512,REXEC +513,RLOGIN +514,SYSLOG +515,LPD +517,TALK +518,NTALK +520,RIP +521,RIPNG +524,NCP +530,RPC +531,AIM +532,NETNEWS +533,NETWALL +540,UUCP +543,KLOGIN +544,KSHELL +546,DHCPV6 +547,DHCPV6 +548,AFP +554,RTSP +563,NNTPS +587,SMTP-SUBMISSION +591,FILEMAKER +601,SYSLOG-MGMT +604,TUNNEL +623,IPMI +631,IPP +636,LDAPS +639,MSDP +646,LDP +647,DHCP-FAILOVER +666,DOOM +674,ACAP +694,HA-HEARTBEAT +698,OLSR +700,EPP +706,SILC +711,TDP +749,KERBEROS-ADM +750,KERBEROS-IV +830,NETCONF-SSH +831,NETCONF-BEEP +853,DNS-TLS +860,ISCSI +873,RSYNC +901,SWAT +902,VMWARE +981,CHECKPOINT-FW +989,FTPS-DATA +990,FTPS +992,TELNETS +993,IMAPS +995,POP3S +1025,MSRPC +1080,SOCKS +1194,OPENVPN +1241,NESSUS +1311,DELL-OPENMANAGE +1433,MSSQL +1434,MSSQL-MONITOR +1494,CITRIX-ICA +1512,WINS +1524,INGRES +1589,CISCO-VQP +1701,L2TP +1719,H323 +1720,H323 +1723,PPTP +1725,STEAM +1755,MSMEDIA +1812,RADIUS +1813,RADIUS-ACCT +1883,MQTT +1900,SSDP +1985,HSRP +2000,CISCO-SCCP +2049,NFS +2082,CPANEL +2083,CPANEL-SSL +2181,ZOOKEEPER +2222,DIRECTADMIN +2375,DOCKER +2376,DOCKER-TLS +2377,DOCKER-SWARM +2379,ETCD +2380,ETCD-PEER +2401,CVS +2483,ORACLE +2484,ORACLE-SSL +3128,HTTP-PROXY +3260,ISCSI +3268,LDAP-GC +3269,LDAPS-GC +3306,MYSQL +3389,RDP +3478,STUN +3479,STUN +3689,DAAP +3690,SVN +4369,EPMD +4500,IPSEC-NAT +4505,SALTSTACK +4506,SALTSTACK +4789,VXLAN +5000,UPNP +5004,RTP +5044,LOGSTASH +5060,SIP +5061,SIPS +5222,XMPP +5223,XMPP-SSL +5353,MDNS +5355,LLMNR +5432,POSTGRESQL +5601,KIBANA +5671,AMQPS +5672,AMQP +5683,COAP +5684,COAPS +5800,VNC-HTTP +5900,VNC +5985,WINRM +5986,WINRM-TLS +6000,X11 +6343,SFLOW +6379,REDIS +6380,REDIS-TLS +6443,KUBERNETES-API +6514,SYSLOG-TLS +7946,DOCKER-SWARM-GOSSIP +8080,HTTP-ALT +8088,SPLUNK-HEC +8125,STATSD +8140,PUPPET +8161,ACTIVEMQ +8200,VAULT +8300,CONSUL-RPC +8301,CONSUL-GOSSIP +8302,CONSUL-GOSSIP +8443,HTTPS-ALT +8472,VXLAN +8500,CONSUL-HTTP +8600,CONSUL-DNS +8883,MQTT-TLS +8888,HTTP-ALT +9000,SONARQUBE +9042,CASSANDRA +9090,PROMETHEUS +9091,PROMETHEUS-PUSHGW +9092,KAFKA +9093,KAFKA-TLS +9100,NODE-EXPORTER +9200,ELASTICSEARCH +9300,ELASTICSEARCH +9411,ZIPKIN +9600,LOGSTASH-MONITOR +10250,KUBELET +10255,KUBELET-RO +10256,KUBE-PROXY +10257,KUBE-CONTROLLER +10259,KUBE-SCHEDULER +11211,MEMCACHED +13720,NETBACKUP +13721,NETBACKUP +15672,RABBITMQ-MGMT +15692,RABBITMQ-METRICS +16443,MICROK8S-API +25565,MINECRAFT +27017,MONGODB +27018,MONGODB +27019,MONGODB +28015,RETHINKDB +29015,RETHINKDB +32400,PLEX +33434,TRACEROUTE diff --git a/partition/roles/sflow-collector/handlers/main.yaml b/partition/roles/sflow-collector/handlers/main.yaml new file mode 100644 index 000000000..d613fdb4b --- /dev/null +++ b/partition/roles/sflow-collector/handlers/main.yaml @@ -0,0 +1,19 @@ +--- +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true + +- name: Restart goflow2 + ansible.builtin.systemd: + name: goflow2 + state: restarted + +- name: Restart vector + ansible.builtin.systemd: + name: vector + state: restarted + +- name: Restart vrf-port-export + ansible.builtin.systemd: + name: vrf-port-export + state: restarted diff --git a/partition/roles/sflow-collector/meta/main.yml b/partition/roles/sflow-collector/meta/main.yml new file mode 100644 index 000000000..48b2dd583 --- /dev/null +++ b/partition/roles/sflow-collector/meta/main.yml @@ -0,0 +1,17 @@ +--- +galaxy_info: + role_name: sflow-collector + author: metal-stack + description: Deploys an sFlow collection and enrichment pipeline. + license: MIT + min_ansible_version: "2.10" + galaxy_tags: [] + platforms: + - name: Debian + versions: + - all + +dependencies: + - role: ansible-common + - role: metal-roles/common/roles/defaults + - role: metal-roles/partition/roles/defaults diff --git a/partition/roles/sflow-collector/tasks/main.yaml b/partition/roles/sflow-collector/tasks/main.yaml new file mode 100644 index 000000000..1a4472544 --- /dev/null +++ b/partition/roles/sflow-collector/tasks/main.yaml @@ -0,0 +1,238 @@ +--- +- name: Gather release versions + setup_yaml: + +- name: Check mandatory variables for this role are set + ansible.builtin.assert: + fail_msg: "not all mandatory variables given, check role documentation" + quiet: true + that: + - sflow_collector_loki_endpoint is defined and sflow_collector_loki_endpoint | length > 0 + - sflow_collector_image_tag is defined + - sflow_collector_vector_image_tag is defined + - sflow_collector_profile in ['fabric', 'edge', 'combined'] + +- name: Create sflow-collector directories + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + loop: + - "{{ sflow_collector_config_host_dir }}" + - "{{ sflow_collector_data_host_dir }}" + - "{{ sflow_collector_data_host_dir }}/geoip" + - "{{ sflow_collector_data_host_dir }}/asn" + - "{{ sflow_collector_data_host_dir }}/vector" + +- name: Create goflow2 spool directory owned by the in-container flow user + ansible.builtin.file: + path: "{{ sflow_collector_data_host_dir }}/spool" + state: directory + owner: "100" + group: "65533" + mode: "0755" + +- name: Mount tmpfs on goflow2 spool + ansible.posix.mount: + path: "{{ sflow_collector_data_host_dir }}/spool" + src: tmpfs + fstype: tmpfs + opts: "size={{ sflow_collector_spool_tmpfs_size }},uid=100,gid=65533,mode=0755" + state: mounted + +- name: Deploy well-known-ports enrichment CSV + ansible.builtin.copy: + src: well_known_ports.csv + dest: "{{ sflow_collector_data_host_dir }}/well_known_ports.csv" + owner: root + group: root + mode: "0644" + notify: Restart vector + +- name: Writing initial empty ifindex.csv + ansible.builtin.copy: + content: "sampler_address,ifindex,interface,vrf,vni\n" + dest: "{{ sflow_collector_data_host_dir }}/ifindex.csv" + force: false + owner: root + group: root + mode: "0644" + when: sflow_collector_profile in ['fabric', 'combined'] + +- name: Deploy vrf-port-export script + ansible.builtin.copy: + src: vrf-port-export.py + dest: /usr/local/bin/vrf-port-export.py + owner: root + group: root + mode: "0755" + notify: Restart vrf-port-export + when: sflow_collector_profile in ['fabric', 'combined'] + +- name: Deploy vrf-port-export systemd service + ansible.builtin.copy: + src: vrf-port-export.service + dest: /etc/systemd/system/vrf-port-export.service + owner: root + group: root + mode: "0644" + notify: + - Reload systemd + - Restart vrf-port-export + when: sflow_collector_profile in ['fabric', 'combined'] + +- name: Ensure vrf-port-export service is started + ansible.builtin.systemd: + name: vrf-port-export + enabled: true + state: started + daemon_reload: true + when: sflow_collector_profile in ['fabric', 'combined'] + +- name: Template Vector configuration + ansible.builtin.template: + src: vector.yaml.j2 + dest: "{{ sflow_collector_config_host_dir }}/vector.yaml" + owner: root + group: root + mode: "0644" + notify: Restart vector + +- name: Deploy mmdb refresh script + ansible.builtin.copy: + src: sflow-mmdb-refresh.sh + dest: /usr/local/bin/sflow-mmdb-refresh.sh + owner: root + group: root + mode: "0755" + when: sflow_collector_profile in ['edge', 'combined'] + +- name: Deploy mmdb refresh systemd service + timer + ansible.builtin.copy: + src: "{{ item }}" + dest: "/etc/systemd/system/{{ item }}" + owner: root + group: root + mode: "0644" + loop: + - sflow-mmdb-refresh.service + - sflow-mmdb-refresh.timer + when: sflow_collector_profile in ['edge', 'combined'] + +- name: Enable mmdb refresh timer + ansible.builtin.systemd: + name: sflow-mmdb-refresh.timer + enabled: true + state: started + daemon_reload: true + when: sflow_collector_profile in ['edge', 'combined'] + +- name: Fetch GeoIP and ASN databases on first deploy + ansible.builtin.command: /usr/local/bin/sflow-mmdb-refresh.sh "{{ item }}" + args: + creates: "{{ sflow_collector_data_host_dir }}/{{ item }}/{{ item }}.mmdb" + loop: + - geoip + - asn + when: sflow_collector_profile in ['edge', 'combined'] + +- name: Remove legacy logrotate.d config + ansible.builtin.file: + path: /etc/logrotate.d/sflow-collector + state: absent + +- name: Deploy logrotate config for goflow2 spool + ansible.builtin.copy: + content: | + {{ sflow_collector_data_host_dir }}/spool/flows.jsonl { + size {{ sflow_collector_logrotate_size }} + rotate {{ sflow_collector_logrotate_keep }} + missingok + notifempty + nocompress + create 0644 + postrotate + /usr/bin/systemctl reload goflow2.service + endscript + } + dest: "{{ sflow_collector_config_host_dir }}/flows.logrotate" + owner: root + group: root + mode: "0644" + +- name: Deploy sflow spool logrotate systemd service + timer + ansible.builtin.template: + src: "{{ item }}.j2" + dest: "/etc/systemd/system/{{ item }}" + owner: root + group: root + mode: "0644" + loop: + - sflow-logrotate.service + - sflow-logrotate.timer + +- name: Enable sflow spool logrotate timer + ansible.builtin.systemd: + name: sflow-logrotate.timer + enabled: true + state: started + daemon_reload: true + +- name: Deploy goflow2 service + ansible.builtin.include_role: + name: ansible-common/roles/systemd-docker-service + vars: + systemd_service_name: goflow2 + systemd_docker_image_name: "{{ sflow_collector_image_name }}" + systemd_docker_image_tag: "{{ sflow_collector_image_tag }}" + systemd_service_after: docker.service + systemd_service_requires: docker.service + systemd_docker_network: host + systemd_docker_volumes: + - "{{ sflow_collector_data_host_dir }}/spool:/spool" + systemd_docker_command: + - "-listen=sflow://:{{ sflow_collector_port }}" + - "-addr=:{{ sflow_collector_metrics_port }}" + - "-format=json" + - "-transport.file=/spool/flows.jsonl" + systemd_docker_log_driver: "{{ sflow_collector_docker_log_driver }}" + +- name: Deploy vector service + ansible.builtin.include_role: + name: ansible-common/roles/systemd-docker-service + vars: + systemd_service_name: vector + systemd_docker_image_name: "{{ sflow_collector_vector_image_name }}" + systemd_docker_image_tag: "{{ sflow_collector_vector_image_tag }}" + systemd_service_after: docker.service goflow2.service + systemd_service_requires: docker.service + systemd_docker_network: host + systemd_service_environment: + VECTOR_LOG: info + systemd_docker_volumes: >- + {{ + [ + sflow_collector_config_host_dir + '/vector.yaml:/etc/vector/vector.yaml:ro', + sflow_collector_data_host_dir + '/well_known_ports.csv:/etc/vector/enrichment/well_known_ports.csv:ro', + sflow_collector_data_host_dir + '/spool:/spool:ro', + sflow_collector_data_host_dir + '/vector:/var/lib/vector' + ] + + ([ + sflow_collector_data_host_dir + '/geoip:/etc/vector/enrichment/geoip:ro', + sflow_collector_data_host_dir + '/asn:/etc/vector/enrichment/asn:ro' + ] if sflow_collector_profile in ['edge', 'combined'] else []) + + ([ + sflow_collector_data_host_dir + '/ifindex.csv:/etc/vector/enrichment/ifindex.csv:ro' + ] if sflow_collector_profile in ['fabric', 'combined'] else []) + }} + systemd_docker_command: + - --config-dir + - /etc/vector/ + systemd_docker_log_driver: "{{ sflow_collector_docker_log_driver }}" + +- name: Wait for goflow2 metrics endpoint + ansible.builtin.wait_for: + port: "{{ sflow_collector_metrics_port }}" + timeout: 60 diff --git a/partition/roles/sflow-collector/templates/sflow-logrotate.service.j2 b/partition/roles/sflow-collector/templates/sflow-logrotate.service.j2 new file mode 100644 index 000000000..b051c590c --- /dev/null +++ b/partition/roles/sflow-collector/templates/sflow-logrotate.service.j2 @@ -0,0 +1,21 @@ +[Unit] +Description=Rotate the goflow2 sFlow spool +After=goflow2.service + +[Service] +Type=oneshot +ExecStart=/usr/sbin/logrotate --state {{ sflow_collector_data_host_dir }}/logrotate.status {{ sflow_collector_config_host_dir }}/flows.logrotate + +NoNewPrivileges=true +ProtectSystem=strict +ReadWritePaths={{ sflow_collector_data_host_dir }} {{ sflow_collector_data_host_dir }}/spool +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectKernelLogs=true +ProtectControlGroups=true +ProtectClock=true +ProtectHostname=true +LockPersonality=true +RestrictSUIDSGID=true +RestrictRealtime=true +RestrictNamespaces=true diff --git a/partition/roles/sflow-collector/templates/sflow-logrotate.timer.j2 b/partition/roles/sflow-collector/templates/sflow-logrotate.timer.j2 new file mode 100644 index 000000000..22641e93b --- /dev/null +++ b/partition/roles/sflow-collector/templates/sflow-logrotate.timer.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Periodic rotation of the goflow2 sFlow spool + +[Timer] +OnBootSec={{ sflow_collector_logrotate_interval }} +OnUnitActiveSec={{ sflow_collector_logrotate_interval }} +AccuracySec=15s + +[Install] +WantedBy=timers.target diff --git a/partition/roles/sflow-collector/templates/vector.yaml.j2 b/partition/roles/sflow-collector/templates/vector.yaml.j2 new file mode 100644 index 000000000..23196ce4a --- /dev/null +++ b/partition/roles/sflow-collector/templates/vector.yaml.j2 @@ -0,0 +1,331 @@ +{% set sflow_edge = sflow_collector_profile in ['edge', 'combined'] %} +{% set sflow_fabric = sflow_collector_profile in ['fabric', 'combined'] %} +data_dir: /var/lib/vector + +enrichment_tables: +{% if sflow_edge %} + geoip: + type: mmdb + path: /etc/vector/enrichment/geoip/geoip.mmdb + + asn: + type: mmdb + path: /etc/vector/enrichment/asn/asn.mmdb + +{% endif %} + well_known_ports: + type: file + file: + path: /etc/vector/enrichment/well_known_ports.csv + encoding: + type: csv + schema: + port: integer + protocol: string + +{% if sflow_fabric %} + ifindex_vrf: + type: file + file: + path: /etc/vector/enrichment/ifindex.csv + encoding: + type: csv + schema: + sampler_address: string + ifindex: integer + interface: string + vrf: string + vni: string +{% endif %} + +sources: + goflow2_raw: + type: file + include: + - /spool/flows*.jsonl + read_from: beginning + +transforms: + parse_goflow2: + type: remap + inputs: + - goflow2_raw + source: | + parsed, err = parse_json(.message) + if err != null { + log("failed to parse goflow2 line: " + err, level: "warn") + abort + } + . = parsed + + .dst_port = to_int(.dst_port) ?? 0 + .src_port = to_int(.src_port) ?? 0 + .in_if = to_int(.in_if) ?? 0 + + # goflow2 already emits proto as a string ("TCP", "UDP", "ICMPv6", etc.) + .proto_name = string!(.proto) + + .app_proto = "UNCLASSIFIED" + .vrf = "default" + .vni = "" + .interface = "" + .src_asn = 0 + .src_asn_org = "" + .sampler_address = string(.sampler_address) ?? "UNKNOWN" + .scope = "" + + if (string(.src_addr) ?? "") == "" { + .scope = "l2" + dmac = downcase(string(.dst_mac) ?? "") + if dmac == "01:80:c2:00:00:0e" || dmac == "01:80:c2:00:00:03" { + .app_proto = "LLDP" + } else if dmac == "01:80:c2:00:00:00" { + .app_proto = "STP" + } else if dmac == "01:80:c2:00:00:01" { + .app_proto = "PAUSE" + } else if dmac == "01:80:c2:00:00:02" { + .app_proto = "LACP" + } else if dmac == "01:80:c2:00:00:14" || dmac == "01:80:c2:00:00:15" { + .app_proto = "ISIS" + } else if dmac == "01:00:0c:cc:cc:cc" { + .app_proto = "CDP" + } else if dmac == "01:00:0c:cc:cc:cd" { + .app_proto = "PVST" + } else if dmac == "01:1b:19:00:00:00" { + .app_proto = "PTP" + } else if dmac == "ff:ff:ff:ff:ff:ff" { + .app_proto = "BROADCAST" + } else if starts_with(dmac, "01:00:5e") || starts_with(dmac, "33:33") { + .app_proto = "MULTICAST" + } else { + .app_proto = "L2-OTHER" + } + } + + enrich_proto: + type: remap + inputs: + - parse_goflow2 + source: | + # Flow monitoring typically samples uplink ingress, meaning that the dst port is most likely of interest + # If neither port matches, the lower one is chosen + if .scope != "l2" { + row, err = get_enrichment_table_record("well_known_ports", { "port": .dst_port }) + if err == null { + .app_proto = row.protocol + } else { + row, err = get_enrichment_table_record("well_known_ports", { "port": .src_port }) + if err == null { + .app_proto = row.protocol + } else { + dst = to_int(.dst_port) ?? 0 + src = to_int(.src_port) ?? 0 + if dst > 0 && (src == 0 || dst <= src) { + .app_proto = to_string(dst) + } else if src > 0 { + .app_proto = to_string(src) + } + } + } + } + +{% if sflow_fabric %} + enrich_vrf: + type: remap + inputs: + - enrich_proto + source: | + row, err = get_enrichment_table_record("ifindex_vrf", { "sampler_address": .sampler_address, "ifindex": .in_if }) + if err == null { + .vrf = row.vrf + .vni = row.vni + .interface = row.interface + } + +{% endif %} + enrich_asn: + type: remap + inputs: + - {{ 'enrich_vrf' if sflow_fabric else 'enrich_proto' }} + source: | + if .scope != "l2" { + src_private = (ip_cidr_contains("10.0.0.0/8", .src_addr) ?? false) || + (ip_cidr_contains("172.16.0.0/12", .src_addr) ?? false) || + (ip_cidr_contains("192.168.0.0/16", .src_addr) ?? false) || + (ip_cidr_contains("127.0.0.0/8", .src_addr) ?? false) || + (ip_cidr_contains("169.254.0.0/16", .src_addr) ?? false) || + (ip_cidr_contains("::1/128", .src_addr) ?? false) || + (ip_cidr_contains("fc00::/7", .src_addr) ?? false) || + (ip_cidr_contains("fe80::/10", .src_addr) ?? false) +{% if sflow_edge %} + if src_private { + .src_asn = 0 + .src_asn_org = "PRIVATE" + } else { + row, err = get_enrichment_table_record("asn", {"ip": .src_addr}) + if err == null { + .src_asn = to_int(row.autonomous_system_number) ?? 0 + .src_asn_org = string(row.autonomous_system_organization) ?? "UNKNOWN" + } else { + .src_asn = 0 + .src_asn_org = "UNKNOWN" + } + } + +{% endif %} + dst_private = (ip_cidr_contains("10.0.0.0/8", .dst_addr) ?? false) || + (ip_cidr_contains("172.16.0.0/12", .dst_addr) ?? false) || + (ip_cidr_contains("192.168.0.0/16", .dst_addr) ?? false) || + (ip_cidr_contains("127.0.0.0/8", .dst_addr) ?? false) || + (ip_cidr_contains("169.254.0.0/16", .dst_addr) ?? false) || + (ip_cidr_contains("::1/128", .dst_addr) ?? false) || + (ip_cidr_contains("fc00::/7", .dst_addr) ?? false) || + (ip_cidr_contains("fe80::/10", .dst_addr) ?? false) + + if !src_private { + .scope = "inbound" + } else if dst_private { + .scope = "internal" + } else { + .scope = "outbound" + } + } + +{% if sflow_edge %} + geo_sample: + type: sample + inputs: + - {{ 'enrich_vrf' if sflow_fabric else 'enrich_proto' }} + rate: {{ sflow_collector_vector_geo_sample_rate }} + + enrich_geo: + type: remap + inputs: + - geo_sample + source: | + # Drop private, loopback, and link-local addresses. Useful in the lab, in production this can be avoided + # by using a dedicated WAN facing sample stream. + if .scope == "l2" { + abort + } + if (ip_cidr_contains("10.0.0.0/8", .src_addr) ?? false) || + (ip_cidr_contains("172.16.0.0/12", .src_addr) ?? false) || + (ip_cidr_contains("192.168.0.0/16", .src_addr) ?? false) || + (ip_cidr_contains("127.0.0.0/8", .src_addr) ?? false) || + (ip_cidr_contains("169.254.0.0/16", .src_addr) ?? false) || + (ip_cidr_contains("::1/128", .src_addr) ?? false) || + (ip_cidr_contains("fc00::/7", .src_addr) ?? false) || + (ip_cidr_contains("fe80::/10", .src_addr) ?? false) { + abort + } + + geo, err = get_enrichment_table_record("geoip", {"ip": .src_addr}) + if err == null { + .country_code = string(geo.country.iso_code) ?? "UNKNOWN" + lat = to_float(geo.location.latitude) ?? null + lon = to_float(geo.location.longitude) ?? null + if lat != null && lon != null { + .latitude = lat + .longitude = lon + } + } else { + .country_code = "UNKNOWN" + } +{% endif %} + +sinks: + loki_goflow2: + type: loki + inputs: + - enrich_asn + endpoint: {{ sflow_collector_loki_endpoint }} +{% if sflow_collector_loki_basic_auth_user %} + auth: + strategy: basic + user: {{ sflow_collector_loki_basic_auth_user }} + password: {{ sflow_collector_loki_basic_auth_password }} +{% endif %} +{% if sflow_collector_loki_endpoint.startswith('https://') %} + tls: + verify_certificate: {{ sflow_collector_loki_tls_verify | to_json }} +{% endif %} + encoding: + codec: json + only_fields: + - src_addr + - dst_addr + - src_port + - dst_port + - proto + - bytes + - packets + labels: + app: goflow2 + partition: "{{ metal_partition_id }}" +{% raw %} + proto_name: "{{ .proto_name }}" + vrf: "{{ .vrf }}" + sampler_address: "{{ .sampler_address }}" + scope: "{{ .scope }}" + structured_metadata: + app_proto: "{{ .app_proto }}" + src_addr: "{{ .src_addr }}" + dst_addr: "{{ .dst_addr }}" + src_port: "{{ .src_port }}" + dst_port: "{{ .dst_port }}" + src_asn: "{{ .src_asn }}" + src_asn_org: "{{ .src_asn_org }}" + interface: "{{ .interface }}" + vni: "{{ .vni }}" + bytes: "{{ .bytes }}" + packets: "{{ .packets }}" + src_mac: "{{ .src_mac }}" + dst_mac: "{{ .dst_mac }}" +{% endraw %} + remove_label_fields: true + healthcheck: + enabled: false + batch: + max_bytes: 1048576 + timeout_secs: 5 + request: + retry_attempts: 5 + buffer: + type: memory + max_events: {{ sflow_collector_vector_buffer_max_events }} + when_full: drop_newest + +{% if sflow_edge %} + loki_geo: + type: loki + inputs: + - enrich_geo + endpoint: {{ sflow_collector_loki_endpoint }} +{% if sflow_collector_loki_basic_auth_user %} + auth: + strategy: basic + user: {{ sflow_collector_loki_basic_auth_user }} + password: {{ sflow_collector_loki_basic_auth_password }} +{% endif %} +{% if sflow_collector_loki_endpoint.startswith('https://') %} + tls: + verify_certificate: {{ sflow_collector_loki_tls_verify | to_json }} +{% endif %} + encoding: + codec: json + labels: + app: goflow2-geo + partition: "{{ metal_partition_id }}" + remove_label_fields: false + healthcheck: + enabled: false + batch: + max_bytes: 524288 + timeout_secs: 10 + request: + retry_attempts: 5 + buffer: + type: memory + max_events: {{ sflow_collector_vector_buffer_max_events }} + when_full: drop_newest +{% endif %} diff --git a/partition/roles/sonic-config/README.md b/partition/roles/sonic-config/README.md index 4a62155aa..2d3c9d09d 100644 --- a/partition/roles/sonic-config/README.md +++ b/partition/roles/sonic-config/README.md @@ -626,4 +626,39 @@ sonic_config_vtep: # The local VLAN interface. vlan: Vlan1001 + +# sFlow configuration. An empty dict disables sFlow. +sonic_config_sflow: + # Whether sFlow is enabled globally on the switch. + enabled: true + + # Polling interval in seconds (0 disables polling). + polling_interval: 20 + + # List of sFlow collectors that receive the samples. + collectors: + # Name of the collector entry. + - name: collector0 + + # IP address of the collector. + ip: 10.1.2.3 + + # UDP port of the collector. SONiC default is 6343. + port: 6343 + + # VRF used to reach the collector (e.g. `default` or `mgmt`). + vrf: default + + # Per-interface sFlow sessions. Only interfaces that should differ from the + # global defaults need to be listed here. + sessions: + # Interface to sample on. + - interface: Ethernet0 + + # Enable or disable sampling on this interface. + # Omit to inherit the global state. + enabled: true + + # Packet sampling rate (1 sample per N packets). + sample_rate: 4096 ``` diff --git a/partition/roles/sonic-config/defaults/main.yaml b/partition/roles/sonic-config/defaults/main.yaml index 5390ec3b7..22addb04f 100644 --- a/partition/roles/sonic-config/defaults/main.yaml +++ b/partition/roles/sonic-config/defaults/main.yaml @@ -29,3 +29,5 @@ sonic_config_timezone: Europe/Berlin sonic_config_vlan_subinterfaces: [] sonic_config_vlans: [] sonic_config_vtep: {} + +sonic_config_sflow: {} diff --git a/partition/roles/sonic-config/templates/sonic-config.yaml.j2 b/partition/roles/sonic-config/templates/sonic-config.yaml.j2 index a302a864b..ced54ec45 100644 --- a/partition/roles/sonic-config/templates/sonic-config.yaml.j2 +++ b/partition/roles/sonic-config/templates/sonic-config.yaml.j2 @@ -19,3 +19,4 @@ ssh_sourceranges: {{ sonic_config_ssh_sourceranges | to_json }} vlan_subinterfaces: {{ sonic_config_vlan_subinterfaces | to_json }} vlans: {{ sonic_config_vlans | to_json }} vtep: {{ sonic_config_vtep | to_json }} +sflow: {{ sonic_config_sflow | to_json }}