Skip to content

add nano seconds support in LogFileReader and protocol converter #2522

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
alexandnpu wants to merge 2 commits into
base: main
Choose a base branch
from
+47 −7
Open

add nano seconds support in LogFileReader and protocol converter #2522

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b31ef73
add nano seconds support in LogFileReader and protocol converter
alexandnpu Jan 28, 2026
f9b3bfa
clang format
alexandnpu Jan 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 17 additions & 1 deletion core/file_server/reader/LogFileReader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2526,7 +2526,23 @@ PipelineEventGroup LogFileReader::GenerateEventGroup(LogFileReaderPtr reader, Lo
if (AppConfig::GetInstance()->EnableLogTimeAutoAdjust()) {
logtime += GetTimeDelta();
}
event->SetTimestamp(logtime);
// Check if nanosecond timestamp is enabled in global config
const GlobalConfig& globalConfig = reader->mReaderConfig.second->GetGlobalConfig();
if (globalConfig.mEnableTimestampNanosecond) {
// Use nanosecond precision timestamp
uint64_t currentTimeNs = GetCurrentTimeInNanoSeconds();
if (AppConfig::GetInstance()->EnableLogTimeAutoAdjust()) {
// When auto-adjusting, use the adjusted second timestamp with current nanosecond part
event->SetTimestamp(logtime, static_cast<uint32_t>(currentTimeNs % kNanoPerSeconds));
} else {
// When not auto-adjusting, use the full nanosecond timestamp
event->SetTimestamp(static_cast<time_t>(currentTimeNs / kNanoPerSeconds),
static_cast<uint32_t>(currentTimeNs % kNanoPerSeconds));
}
} else {
// Use regular second timestamp
event->SetTimestamp(logtime);
}
event->SetContentNoCopy(DEFAULT_CONTENT_KEY, logBuffer->rawBuffer);
event->SetPosition(logBuffer->readOffset, logBuffer->readLength);

Expand Down
18 changes: 15 additions & 3 deletions pkg/protocol/converter/converter_single_log_flatten.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,10 +48,22 @@ func (c *Converter) ConvertToSingleProtocolLogsFlatten(logGroup *protocol.LogGro
}
}

if newKey, ok := c.ProtocolKeyRenameMap[protocolKeyTime]; ok {
customSingleLog[newKey] = log.Time
// Set timestamp with nanosecond precision if enabled
timestamp := log.Time
if c.GlobalConfig != nil && c.GlobalConfig.EnableTimestampNanosecond {
// Combine seconds and nanoseconds
timestampNs := uint64(timestamp) * 1000000000 + uint64(log.GetTimeNs())
if newKey, ok := c.ProtocolKeyRenameMap[protocolKeyTime]; ok {
customSingleLog[newKey] = timestampNs
} else {
customSingleLog[protocolKeyTime] = timestampNs
}
} else {
customSingleLog[protocolKeyTime] = log.Time
if newKey, ok := c.ProtocolKeyRenameMap[protocolKeyTime]; ok {
customSingleLog[newKey] = timestamp
} else {
customSingleLog[protocolKeyTime] = timestamp
}
}

convertedLogs[i] = customSingleLog
Expand Down
18 changes: 15 additions & 3 deletions pkg/protocol/converter/custom_single_log.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,22 @@ func (c *Converter) ConvertToSingleProtocolLogs(logGroup *protocol.LogGroup, tar
desiredValues[i] = desiredValue

customSingleLog := make(map[string]interface{}, numProtocolKeys)
if newKey, ok := c.ProtocolKeyRenameMap[protocolKeyTime]; ok {
customSingleLog[newKey] = log.Time
// Set timestamp with nanosecond precision if enabled
timestamp := log.Time
if c.GlobalConfig != nil && c.GlobalConfig.EnableTimestampNanosecond {
// Combine seconds and nanoseconds
timestampNs := uint64(timestamp) * 1000000000 + uint64(log.GetTimeNs())
if newKey, ok := c.ProtocolKeyRenameMap[protocolKeyTime]; ok {
customSingleLog[newKey] = timestampNs
} else {
customSingleLog[protocolKeyTime] = timestampNs
}
} else {
customSingleLog[protocolKeyTime] = log.Time
if newKey, ok := c.ProtocolKeyRenameMap[protocolKeyTime]; ok {
customSingleLog[newKey] = timestamp
} else {
customSingleLog[protocolKeyTime] = timestamp
}
}
if newKey, ok := c.ProtocolKeyRenameMap[protocolKeyContent]; ok {
customSingleLog[newKey] = contents
Expand Down
Loading