diff --git a/pkgs/sdk/server/src/Integrations/FileDataSourceBuilder.cs b/pkgs/sdk/server/src/Integrations/FileDataSourceBuilder.cs index 893ecd2f..2b12d83d 100644 --- a/pkgs/sdk/server/src/Integrations/FileDataSourceBuilder.cs +++ b/pkgs/sdk/server/src/Integrations/FileDataSourceBuilder.cs @@ -111,7 +111,8 @@ public FileDataSourceBuilder FileReader(FileDataTypes.IFileReader fileReader) /// Whenever possible, you should update a file's entire contents in one atomic operation; in Unix-like OSes, /// that can be done by creating a temporary file, writing to it, and then renaming it to replace the original /// file. In Windows, that is not always possible, so FileDataSource might detect an update before the file has - /// been fully written; in that case it will retry until it succeeds. + /// been fully written; in that case it will retry several times until the file can be parsed, giving up after + /// repeated failures until another change to the file is detected. /// /// /// Note that auto-updating may not work if any of the files you specified has an invalid directory path. diff --git a/pkgs/sdk/server/src/Internal/DataSources/FileDataSource.cs b/pkgs/sdk/server/src/Internal/DataSources/FileDataSource.cs index 91cb989b..0a4a58d9 100644 --- a/pkgs/sdk/server/src/Internal/DataSources/FileDataSource.cs +++ b/pkgs/sdk/server/src/Internal/DataSources/FileDataSource.cs @@ -22,12 +22,38 @@ internal sealed class FileDataSource : IDataSource private readonly FlagFileDataMerger _dataMerger; private readonly FileDataTypes.IFileReader _fileReader; private readonly bool _skipMissingPaths; + private readonly bool _autoUpdate; private readonly Logger _logger; private volatile bool _started; private volatile bool _loadedValidData; + private volatile bool _disposed; private volatile int _lastVersion; private object _updateLock = new object(); + private const int MaxLoadAttempts = 5; + private static readonly TimeSpan RetryDelay = TimeSpan.FromMilliseconds(600); + // Consecutive load failures (parse or read) per path within the current failure episode. + // An externally triggered load (Start or a file-change notification) starts a new episode + // and clears this, so the retry budget is per-episode, not per-lifetime. Only touched + // inside _updateLock. + private readonly Dictionary _loadFailureCounts = new Dictionary(); + // Whether a delayed retry is already scheduled; at most one retry chain exists at a time, + // since each retry re-reads every path anyway. Only touched inside _updateLock. + private bool _retryPending; + + /// + /// Constructs a file data source that loads flag and segment data from local files. + /// + /// receives the data set produced by each successful load + /// reads file contents; injectable for testing + /// the file paths to load, in order + /// true to watch the files and reload on changes; also enables the + /// bounded retry of loads that fail while a file is being written + /// optional parser for non-JSON content (for example YAML); + /// null to parse JSON only + /// true to skip missing files instead of failing the load + /// how to handle a key that appears in more than one file + /// the destination for log output public FileDataSource(IDataSourceUpdates dataSourceUpdates, FileDataTypes.IFileReader fileReader, List paths, bool autoUpdate, Func alternateParser, bool skipMissingPaths, FileDataTypes.DuplicateKeysHandling duplicateKeysHandling, @@ -40,6 +66,7 @@ public FileDataSource(IDataSourceUpdates dataSourceUpdates, FileDataTypes.IFileR _dataMerger = new FlagFileDataMerger(duplicateKeysHandling); _fileReader = fileReader; _skipMissingPaths = skipMissingPaths; + _autoUpdate = autoUpdate; _lastVersion = 0; if (autoUpdate) { @@ -62,7 +89,7 @@ public FileDataSource(IDataSourceUpdates dataSourceUpdates, FileDataTypes.IFileR public Task Start() { _started = true; - LoadAll(); + LoadAll(isRetry: false); // We always complete the start task regardless of whether we successfully loaded data or not; // if the data files were bad, they're unlikely to become good within the short interval that @@ -83,14 +110,38 @@ private void Dispose(bool disposing) { if (disposing) { + _disposed = true; _reloader?.Dispose(); } } - private void LoadAll() + private void LoadAll(bool isRetry) { lock (_updateLock) { + if (_disposed) + { + return; + } + if (!isRetry) + { + // An externally triggered load starts a new failure episode: failures + // observed from here on get a fresh retry budget, and any state left over + // from a previous episode is discarded. + _loadFailureCounts.Clear(); + } + else + { + _retryPending = false; + if (_loadFailureCounts.Count == 0) + { + // The failure state was cleared in the meantime (an externally triggered + // load succeeded, or the chain gave up) — a reload would be redundant and + // would re-Init identical data at bumped versions, firing spurious change + // events. + return; + } + } var version = Interlocked.Increment(ref _lastVersion); var flags = new Dictionary(); var segments = new Dictionary(); @@ -100,7 +151,21 @@ private void LoadAll() { var content = _fileReader.ReadAllText(path); _logger.Debug("file data: {0}", content); - var data = _parser.Parse(content, version); + FullDataSet data; + try + { + data = _parser.Parse(content, version); + } + catch (Exception e) + { + // A file-change notification can fire while the file is mid-write, so a parse + // failure may just mean we read an empty or partially written file. This applies + // to any configured parser (JSON or alternate), so we treat every failure of + // Parse — as opposed to reading the file — as potentially transient. + HandleParseFailure(path, e); + return; + } + _loadFailureCounts.Remove(path); _dataMerger.AddToData(data, flags, segments); } catch (FileNotFoundException) when (_skipMissingPaths) @@ -109,7 +174,20 @@ private void LoadAll() } catch (Exception e) { - LogHelpers.LogException(_logger, "Failed to load " + path, e); + if (isRetry) + { + // A transient read error (for example, a writer replacing the file) + // must not end a retry episode early: paths that were promised + // retries would keep stale data with budget remaining, and another + // file-change notification is not guaranteed. Charge the failure to + // the same per-path budget and continue the chain; it logs a Warn + // while retrying and an Error only on give-up. + HandleRetryLoadFailure(path, e); + } + else + { + LogHelpers.LogException(_logger, "Failed to load " + path, e); + } return; } } @@ -124,12 +202,106 @@ private void LoadAll() } } + // Called under _updateLock when parsing a path's content fails. Since an externally + // triggered load clears _loadFailureCounts before reading, any existing count for the + // path belongs to the current episode. + private void HandleParseFailure(string path, Exception e) + { + if (!_autoUpdate) + { + // With auto-update off, files are documented to be loaded only once, so we don't + // retry in the background — Start()'s result stays final. + LogHelpers.LogException(_logger, "Failed to parse " + path, e); + return; + } + + _loadFailureCounts.TryGetValue(path, out var previousAttempts); + var attempts = previousAttempts + 1; + _loadFailureCounts[path] = attempts; + + if (attempts < MaxLoadAttempts) + { + _logger.Warn("{0}: Failed to parse file ({1}); will retry in {2} ms in case it was incompletely written", + path, LogValues.ExceptionSummary(e), RetryDelay.TotalMilliseconds); + _logger.Debug("{0}", LogValues.ExceptionTrace(e)); + ScheduleRetry(); + } + else + { + EndEpisode(path); + LogHelpers.LogException(_logger, + string.Format("{0}: Failed to parse file after {1} attempts", path, MaxLoadAttempts), e); + } + } + + // Called under _updateLock when a retry attempt fails before parsing (for example, a + // transient read error). Charges the failure to the path's per-episode budget and + // continues or ends the retry chain. + private void HandleRetryLoadFailure(string path, Exception e) + { + _loadFailureCounts.TryGetValue(path, out var previousAttempts); + var attempts = previousAttempts + 1; + _loadFailureCounts[path] = attempts; + + if (attempts < MaxLoadAttempts) + { + _logger.Warn("{0}: Failed to read file on a retry ({1}); will retry again in {2} ms", + path, LogValues.ExceptionSummary(e), RetryDelay.TotalMilliseconds); + _logger.Debug("{0}", LogValues.ExceptionTrace(e)); + ScheduleRetry(); + } + else + { + EndEpisode(path); + LogHelpers.LogException(_logger, + string.Format("{0}: Failed to load file after {1} attempts; will not retry until the next detected file change", + path, MaxLoadAttempts), e); + } + } + + // Called under _updateLock. Ends the current failure episode for every path: the chain + // stopped at failedPath on every attempt, so any other paths with recorded failures were + // never re-attempted and their promised retries cannot happen. + private void EndEpisode(string failedPath) + { + _loadFailureCounts.Remove(failedPath); + foreach (var abandoned in _loadFailureCounts.Keys) + { + _logger.Error("{0}: Will not be retried because {1} repeatedly failed to load; both will be re-read on the next detected file change", + abandoned, failedPath); + } + _loadFailureCounts.Clear(); + } + + // Called under _updateLock. + private void ScheduleRetry() + { + if (_retryPending) + { + return; // the already-scheduled retry will re-read every path + } + _retryPending = true; + Task.Run(async () => + { + await Task.Delay(RetryDelay).ConfigureAwait(false); + try + { + LoadAll(isRetry: true); + } + catch (Exception e) + { + // Nothing observes this task, so any escaping exception would otherwise vanish. + LogHelpers.LogException(_logger, "Unexpected error while retrying file data load", e); + } + }); + } + private void TriggerReload() { if (_started) { _logger.Info("detected file modification, reloading"); - LoadAll(); + LoadAll(isRetry: false); } } } diff --git a/pkgs/sdk/server/test/Internal/DataSources/FileDataSourceTest.cs b/pkgs/sdk/server/test/Internal/DataSources/FileDataSourceTest.cs index 5322a38c..1b5dc29b 100644 --- a/pkgs/sdk/server/test/Internal/DataSources/FileDataSourceTest.cs +++ b/pkgs/sdk/server/test/Internal/DataSources/FileDataSourceTest.cs @@ -1,7 +1,10 @@ using System; +using System.Collections.Concurrent; +using System.IO; using System.Linq; using System.Threading; using Castle.Core.Internal; +using LaunchDarkly.Logging; using LaunchDarkly.Sdk.Server.Integrations; using LaunchDarkly.Sdk.Server.Interfaces; using LaunchDarkly.Sdk.Server.Internal.Model; @@ -151,50 +154,7 @@ public void ModifiedFileIsReloadedIfAutoUpdateIsOn() file.SetContentFromPath(TestUtils.TestFilePath("segment-only.json")); - AssertHelpers.ExpectPredicate(_updateSink.Inits, actual => - { - var segments = actual.Data.First(item => item.Key == DataModel.Segments); - var features = actual.Data.First(item => item.Key == DataModel.Features); - if (!features.Value.Items.IsNullOrEmpty()) - { - return false; - } - - var segmentItems = segments.Value.Items.ToList(); - - if (segmentItems.Count != 1) - { - return false; - } - - var segmentDescriptor = segmentItems[0]; - if (segmentDescriptor.Key != "seg1") - { - return false; - } - - if (segmentDescriptor.Value.Version == 1) - { - return false; - } - - if (!(segmentDescriptor.Value.Item is Segment segment)) - { - return false; - } - - if (segment.Deleted) - { - return false; - } - - if (segment.Included.Count != 1) - { - return false; - } - - return segment.Included[0] == "user1"; - }, + AssertHelpers.ExpectPredicate(_updateSink.Inits, IsSegmentOnlyDataAfterReload, "Did not receive expected update from the file data source.", TimeSpan.FromSeconds(30)); } @@ -270,16 +230,9 @@ public void ModifiedFileIsReloadedEvenIfOneFileIsMissingIfSkipMissingPathsIsSet( file1.SetContentFromPath(TestUtils.TestFilePath("segment-only.json")); - // Use ExpectJsonValue to handle potential race conditions where the file watcher - // may trigger multiple reload events. This keeps checking events until one matches - // the expected JSON or the timeout expires. - var newData = AssertHelpers.ExpectJsonValue( - _updateSink.Inits, - DataSetAsJson(ExpectedDataSetForSegmentOnlyFile(2)), - DataSetAsJson, + AssertHelpers.ExpectPredicate(_updateSink.Inits, IsSegmentOnlyDataAfterReload, + "Did not receive expected update from the file data source.", TimeSpan.FromSeconds(30)); - - AssertJsonEqual(DataSetAsJson(ExpectedDataSetForSegmentOnlyFile(2)), DataSetAsJson(newData)); } } } @@ -298,19 +251,301 @@ public void IfFlagsAreBadAtStartTimeAutoUpdateCanStillLoadGoodDataLater() file.SetContentFromPath(TestUtils.TestFilePath("segment-only.json")); - // Use ExpectJsonValue to handle potential race conditions where the file watcher - // may trigger multiple reload events. This keeps checking events until one matches - // the expected JSON or the timeout expires. - // Note that the expected version is 2 because we increment the version on each - // *attempt* to load the files, not on each successful load. - var newData = AssertHelpers.ExpectJsonValue( - _updateSink.Inits, - DataSetAsJson(ExpectedDataSetForSegmentOnlyFile(2)), - DataSetAsJson, + AssertHelpers.ExpectPredicate(_updateSink.Inits, IsSegmentOnlyDataAfterReload, + "Did not receive expected update from the file data source.", TimeSpan.FromSeconds(30)); + } + } + } + + private const string ValidFlagJson = @"{""flagValues"":{""flag1"":""a""}}"; + private const string TruncatedFlagJson = @"{""flagValues"": {"; // invalid as JSON and as YAML + + // Simulates reading a file that is mid-write: returns truncated content until Bad is + // cleared, and counts reads so tests can observe retry attempts deterministically + // without depending on real file-watcher timing. + private class ScriptedFileReader : FileDataTypes.IFileReader + { + private int _reads; + public volatile bool Bad = true; + public volatile bool Throw = false; + public int Reads => Volatile.Read(ref _reads); + + public string ReadAllText(string path) + { + Interlocked.Increment(ref _reads); + if (Throw) + { + throw new IOException("simulated transient read error"); + } + return Bad ? TruncatedFlagJson : ValidFlagJson; + } + } + + private static void WaitUntil(Func condition, string description) + { + var deadline = DateTime.UtcNow.AddSeconds(15); + while (!condition() && DateTime.UtcNow < deadline) + { + Thread.Sleep(20); + } + Assert.True(condition(), "timed out waiting for " + description); + } + + private static void WaitForReads(ScriptedFileReader reader, int count) => + WaitUntil(() => reader.Reads >= count, count + " file reads"); + + [Fact] + public void ParseFailureFromPartialReadIsRetriedUntilContentIsComplete() + { + var reader = new ScriptedFileReader(); + using (var file = TempFile.Create()) + { + factory.FilePaths(file.Path).AutoUpdate(true).FileReader(reader); + using (var fp = MakeDataSource()) + { + fp.Start(); + WaitForReads(reader, 2); + reader.Bad = false; // as if the write completed, with no further notification + _updateSink.Inits.ExpectValue(TimeSpan.FromSeconds(5)); + Assert.True(fp.Initialized); + } + } + } + + [Fact] + public void ParseRetryStopsAfterMaxAttemptsAndDoesNotInit() + { + var reader = new ScriptedFileReader(); + using (var file = TempFile.Create()) + { + factory.FilePaths(file.Path).AutoUpdate(true).FileReader(reader); + using (var fp = MakeDataSource()) + { + fp.Start(); + WaitForReads(reader, 5); // initial attempt + 4 retries + Thread.Sleep(1500); // longer than two retry delays + Assert.Equal(5, reader.Reads); // budget exhausted, no further attempts + _updateSink.Inits.ExpectNoValue(); + Assert.False(fp.Initialized); + } + } + } + + [Fact] + public void ParseRetryBudgetResetsForANewFailureEpisode() + { + var reader = new ScriptedFileReader(); + using (var file = TempFile.Create()) + { + factory.FilePaths(file.Path).AutoUpdate(true).FileReader(reader); + using (var fp = MakeDataSource()) + { + fp.Start(); + WaitForReads(reader, 5); // episode 1: all attempts fail + Thread.Sleep(1500); + Assert.Equal(5, reader.Reads); // episode 1 exhausted, nothing pending + + // A new file-change notification starts a new episode with a fresh retry + // budget, even though its first read still sees partial content. + file.SetContent("trigger-new-episode"); + WaitForReads(reader, 6); + + reader.Bad = false; // write completed; no further notification arrives + _updateSink.Inits.ExpectValue(TimeSpan.FromSeconds(5)); + } + } + } + + [Fact] + public void ParseRetryAppliesWhenAlternateParserIsConfigured() + { + var yaml = new DeserializerBuilder().Build(); + var reader = new ScriptedFileReader(); + using (var file = TempFile.Create()) + { + factory.FilePaths(file.Path).AutoUpdate(true).FileReader(reader) + .Parser(s => yaml.Deserialize(s)); + using (var fp = MakeDataSource()) + { + fp.Start(); + reader.Bad = false; + _updateSink.Inits.ExpectValue(TimeSpan.FromSeconds(5)); + } + } + } + + [Fact] + public void ParseFailureIsNotRetriedIfAutoUpdateIsOff() + { + var reader = new ScriptedFileReader(); + using (var file = TempFile.Create()) + { + factory.FilePaths(file.Path).AutoUpdate(false).FileReader(reader); + using (var fp = MakeDataSource()) + { + var task = fp.Start(); + Assert.True(task.IsCompleted); + Assert.False(fp.Initialized); + reader.Bad = false; + _updateSink.Inits.ExpectNoValue(TimeSpan.FromSeconds(2)); + Assert.False(fp.Initialized); + Assert.Equal(1, reader.Reads); + } + } + } + + [Fact] + public void TransientReadErrorDuringRetryDoesNotEndTheEpisode() + { + var reader = new ScriptedFileReader(); + using (var file = TempFile.Create()) + { + factory.FilePaths(file.Path).AutoUpdate(true).FileReader(reader); + using (var fp = MakeDataSource()) + { + fp.Start(); // parse fails, schedules a retry + // The retry's read fails transiently (e.g. a writer is replacing the file); + // the content itself is complete from here on. + reader.Bad = false; + reader.Throw = true; + WaitForReads(reader, 2); + reader.Throw = false; + + // The episode still has budget, so the chain must continue and load the data + // instead of dying on the non-parse failure. + _updateSink.Inits.ExpectValue(TimeSpan.FromSeconds(5)); + } + } + } + + [Fact] + public void PendingParseRetryIsCanceledByDispose() + { + var reader = new ScriptedFileReader(); + using (var file = TempFile.Create()) + { + factory.FilePaths(file.Path).AutoUpdate(true).FileReader(reader); + using (var fp = MakeDataSource()) + { + fp.Start(); // schedules a retry + fp.Dispose(); + // No new load may start after Dispose; any retry scheduled before it must + // observe the disposal and do nothing. (Reads are captured after Dispose so + // the test stays valid even if a retry fired before Dispose ran.) + var readsAtDispose = reader.Reads; + Thread.Sleep(1500); + Assert.Equal(readsAtDispose, reader.Reads); + } + } + } + + // The multi-path tests below use paths in a directory that does not exist, so the file + // watcher fails to construct (logged and swallowed, _reloader == null). That makes the + // load sequence fully deterministic: the only externally triggered loads are the Start() + // calls, which exercise the same code path as a file-change notification. + private const string MultiPathA = "/nonexistent-ld-filedatasource-test-dir/a.json"; + private const string MultiPathB = "/nonexistent-ld-filedatasource-test-dir/b.json"; - AssertJsonEqual(DataSetAsJson(ExpectedDataSetForSegmentOnlyFile(2)), DataSetAsJson(newData)); + private class PerPathScriptedReader : FileDataTypes.IFileReader + { + private readonly ConcurrentDictionary _reads = new ConcurrentDictionary(); + private readonly ConcurrentDictionary _bad = new ConcurrentDictionary(); + + public void SetBad(string path, bool bad) { _bad[path] = bad; } + public int Reads(string path) => _reads.TryGetValue(path, out var n) ? n : 0; + + public string ReadAllText(string path) + { + _reads.AddOrUpdate(path, 1, (_, n) => n + 1); + if (_bad.TryGetValue(path, out var bad) && bad) + { + return TruncatedFlagJson; } + // distinct flag keys per path, so a successful merge of both files can't throw + // on duplicate keys (the builder default is DuplicateKeysHandling.Throw) + return path == MultiPathA + ? @"{""flagValues"":{""flagA"":""a""}}" + : @"{""flagValues"":{""flagB"":""b""}}"; + } + } + + [Fact] + public void PendingParseRetryIsSkippedIfAnExternalReloadAlreadySucceeded() + { + var reader = new ScriptedFileReader(); + using (var file = TempFile.Create()) + { + factory.FilePaths(file.Path).AutoUpdate(true).FileReader(reader); + using (var fp = MakeDataSource()) + { + fp.Start(); // fails, schedules a retry + + // An externally triggered load (same code path as a file-change notification) + // succeeds before the pending retry fires. + reader.Bad = false; + fp.Start(); + _updateSink.Inits.ExpectValue(TimeSpan.FromSeconds(1)); + var readsAfterSuccess = reader.Reads; + + Thread.Sleep(1500); // past the retry delay + // The pending retry saw that the load already succeeded and did not reload, + // so no redundant Init (which would fire spurious change events) occurred. + Assert.Equal(readsAfterSuccess, reader.Reads); + _updateSink.Inits.ExpectNoValue(); + } + } + } + + [Fact] + public void ParseFailureInANewEpisodeGetsAFullRetryBudgetAfterAnotherPathGaveUp() + { + var reader = new PerPathScriptedReader(); + reader.SetBad(MultiPathB, true); + factory.FilePaths(MultiPathA, MultiPathB).AutoUpdate(true).FileReader(reader); + using (var fp = MakeDataSource()) + { + fp.Start(); // A parses, B fails; B accumulates failures across retries + WaitUntil(() => reader.Reads(MultiPathB) >= 4, "4 reads of path B"); + reader.SetBad(MultiPathA, true); // now every attempt stops at A + WaitUntil(() => reader.Reads(MultiPathA) >= 9, "A to exhaust its retry budget"); + Thread.Sleep(1500); // the retry chain is dead + Assert.Equal(9, reader.Reads(MultiPathA)); + + // A new externally triggered load fails at A; A recovers before the retry fires. + fp.Start(); + reader.SetBad(MultiPathA, false); + WaitUntil(() => reader.Reads(MultiPathB) >= 5, "the retry to reach path B"); + reader.SetBad(MultiPathB, false); + + // B's failure was its first in the new episode, so it gets a fresh retry budget + // instead of inheriting the dead episode's count and giving up immediately. + _updateSink.Inits.ExpectValue(TimeSpan.FromSeconds(5)); + } + } + + [Fact] + public void PathAbandonedWhenAnotherPathGivesUpGetsATerminalLogMessage() + { + var reader = new PerPathScriptedReader(); + reader.SetBad(MultiPathB, true); + factory.FilePaths(MultiPathA, MultiPathB).AutoUpdate(true).FileReader(reader); + using (var fp = MakeDataSource()) + { + fp.Start(); + WaitUntil(() => reader.Reads(MultiPathB) >= 4, "4 reads of path B"); + reader.SetBad(MultiPathA, true); + WaitUntil(() => reader.Reads(MultiPathA) >= 9, "A to exhaust its retry budget"); + Thread.Sleep(1500); // the retry chain is dead + + // B was last warned "will retry in 600 ms", but A's give-up ended the chain. + // That promise must be either fulfilled (B re-read) or terminated with an + // error log naming B. + var bRetried = reader.Reads(MultiPathB) >= 5; + var bTerminallyLogged = LogCapture.GetMessages().Any(m => + m.Level == LogLevel.Error && m.Text.Contains(MultiPathB)); + Assert.True(bRetried || bTerminallyLogged, + "path B was promised a retry but was never re-read and got no terminal error log"); } } @@ -365,5 +600,38 @@ private static FullDataSet ExpectedDataSetForSegmentOnlyFile(int new SegmentBuilder("seg1").Version(version).Included("user1").Build() ) .Build(); + + // Predicate that matches the structure of segment-only.json reloaded after the initial load. + // We deliberately don't pin the exact version: with the file watcher firing on truncate-then-write + // and the JsonException retry, the version of the successful load is non-deterministic, so we + // only require that it isn't the initial version 1. + private static bool IsSegmentOnlyDataAfterReload(FullDataSet actual) + { + var features = actual.Data.First(item => item.Key == DataModel.Features); + if (!features.Value.Items.IsNullOrEmpty()) + { + return false; + } + + var segments = actual.Data.First(item => item.Key == DataModel.Segments); + var segmentItems = segments.Value.Items.ToList(); + if (segmentItems.Count != 1) + { + return false; + } + + var segmentDescriptor = segmentItems[0]; + if (segmentDescriptor.Key != "seg1" || segmentDescriptor.Value.Version == 1) + { + return false; + } + + if (!(segmentDescriptor.Value.Item is Segment segment) || segment.Deleted) + { + return false; + } + + return segment.Included.Count == 1 && segment.Included[0] == "user1"; + } } }