diff --git a/app/vlinsert/syslog/syslog.go b/app/vlinsert/syslog/syslog.go index cf62cf17e6..99d325b08d 100644 --- a/app/vlinsert/syslog/syslog.go +++ b/app/vlinsert/syslog/syslog.go @@ -34,6 +34,9 @@ import ( var ( syslogTimezone = flag.String("syslog.timezone", "Local", "Timezone to use when parsing timestamps in RFC3164 syslog messages. Timezone must be a valid IANA Time Zone. "+ "For example: America/New_York, Europe/Berlin, Etc/GMT+3 . See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") + syslogMsgField = flagutil.NewArrayString("syslog.msgField", "Fields to use as the _msg field. "+ + "Defaults to 'message' for plain syslog and 'cef.name' (name of the event) for CEF-formatted logs. "+ + "See https://docs.victoriametrics.com/victorialogs/keyconcepts/#message-field") listenAddrTCP = flagutil.NewArrayString("syslog.listenAddr.tcp", "Comma-separated list of TCP addresses to listen to for Syslog messages. "+ "See https://docs.victoriametrics.com/victorialogs/data-ingestion/syslog/") @@ -605,14 +608,22 @@ func processLine(line []byte, currentYear int, timezone *time.Location, useLocal p.AddField("hostname", remoteIP) } } - logstorage.RenameField(p.Fields, msgFields, "_msg") + logstorage.RenameField(p.Fields, getMsgFields(), "_msg") lmp.AddRow(ts, p.Fields, -1) return nil } var timeFields = []string{"timestamp"} -var msgFields = []string{"message"} + +var defaultMsgFields = []string{"message", "cef.name"} + +func getMsgFields() []string { + if len(*syslogMsgField) > 0 { + return *syslogMsgField + } + return defaultMsgFields +} var ( errorsTotal = metrics.NewCounter(`vl_errors_total{type="syslog"}`) diff --git a/app/vlinsert/syslog/syslog_test.go b/app/vlinsert/syslog/syslog_test.go index 6f0f4525a3..1569ad4193 100644 --- a/app/vlinsert/syslog/syslog_test.go +++ b/app/vlinsert/syslog/syslog_test.go @@ -102,7 +102,7 @@ Sep 19 08:26:10 host CEF:0|Security|threatmanager|1.0|100|worm successfully stop currentYear := 2023 timestampsExpected := []int64{1685794113000000000, 1695111970000000000, 1685880513000000000, 1685814132345000000} resultExpected := `{"format":"rfc3164","hostname":"abcd","app_name":"systemd","_msg":"Starting Update the local ESM caches...","remote_ip":"1.2.3.4"} -{"format":"rfc3164","hostname":"host","app_name":"CEF","cef.version":"0","cef.device_vendor":"Security","cef.device_product":"threatmanager","cef.device_version":"1.0","cef.device_event_class_id":"100","cef.name":"worm successfully stopped","cef.severity":"10","cef.extension.src":"10.0.0.1","cef.extension.dst":"2.1.2.2","cef.extension.spt":"1232","remote_ip":"1.2.3.4"} +{"format":"rfc3164","hostname":"host","app_name":"CEF","cef.version":"0","cef.device_vendor":"Security","cef.device_product":"threatmanager","cef.device_version":"1.0","cef.device_event_class_id":"100","_msg":"worm successfully stopped","cef.severity":"10","cef.extension.src":"10.0.0.1","cef.extension.dst":"2.1.2.2","cef.extension.spt":"1232","remote_ip":"1.2.3.4"} {"priority":"165","facility_keyword":"local4","level":"notice","facility":"20","severity":"5","format":"rfc3164","hostname":"abcd","app_name":"systemd","proc_id":"345","_msg":"abc defg","remote_ip":"1.2.3.4"} {"priority":"123","facility_keyword":"solaris-cron","level":"error","facility":"15","severity":"3","format":"rfc5424","hostname":"mymachine.example.com","app_name":"appname","proc_id":"12345","msg_id":"ID47","exampleSDID@32473.iut":"3","exampleSDID@32473.eventSource":"Application 123 = ] 56","exampleSDID@32473.eventID":"11211","_msg":"This is a test message with structured data.","remote_ip":"1.2.3.4"}` f(data, currentYear, timestampsExpected, resultExpected) diff --git a/docs/victorialogs/CHANGELOG.md b/docs/victorialogs/CHANGELOG.md index 299303cd2c..c5749394d6 100644 --- a/docs/victorialogs/CHANGELOG.md +++ b/docs/victorialogs/CHANGELOG.md @@ -25,6 +25,7 @@ according to the following docs: * FEATURE: [querying API](https://docs.victoriametrics.com/victorialogs/querying/): allow using [`limit`](https://docs.victoriametrics.com/victorialogs/logsql/#limit-pipe) and [`offset`](https://docs.victoriametrics.com/victorialogs/logsql/#offset-pipe) pipes after the [`stats` pipe](https://docs.victoriametrics.com/victorialogs/logsql/#stats-pipe) in queries to [`/select/logsql/stats_query`](https://docs.victoriametrics.com/victorialogs/querying/#querying-log-stats). This enables the usage for these pipes in [alerting and recording rules for VictoriaLogs](https://docs.victoriametrics.com/victorialogs/vmalert/). See [#1296](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1296). * FEATURE: [alerts](https://github.com/VictoriaMetrics/VictoriaLogs/blob/master/deployment/docker/rules): add new alerting rules `PersistentQueueRunsOutOfSpaceIn12Hours` and `PersistentQueueRunsOutOfSpaceIn4Hours` for `vlagent` persistent queue capacity. These alerts help users to take proactive actions before `vlagent` starts dropping logs due to insufficient persistent queue space. See [#10193](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/10193) * FEATURE: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): remove the `Date format` setting and always display timestamps with nanosecond precision. See [#1161](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1161). +* FEATURE: [data ingestion](https://docs.victoriametrics.com/victorialogs/data-ingestion/): add an ability to override the default list of `_msg` fields for syslog ingestion protocol. The default list of `_msg` fields has been updated to `message` (for plain syslog) and `cef.name` (name of the event for CEF-formatted logs). It is useful for CEF-formatted logs which may contain an arbitrary number of additional log fields. See [#1362](https://github.com/VictoriaMetrics/VictoriaLogs/issues/1362). * BUGFIX: [vlagent](https://docs.victoriametrics.com/victorialogs/vlagent/): hide sensitive values passed via `-remoteWrite.proxyURL` in `/metrics`, `/flags`, and startup logs. Previously these values could be exposed in plain text. See [#1320](https://github.com/VictoriaMetrics/VictoriaLogs/pull/1320). * BUGFIX: [web UI](https://docs.victoriametrics.com/victorialogs/querying/#web-ui): sanitize markdown URLs in logs rendered with `markdown parsing` enabled, allowing only `http`, `https`, `mailto`, and `tel` schemes for active links and images. See [#1313](https://github.com/VictoriaMetrics/VictoriaLogs/pull/1313).