diff --git a/src/DataStax.AstraDB.DataApi/Collections/Collection.cs b/src/DataStax.AstraDB.DataApi/Collections/Collection.cs index 64465929..3339d934 100644 --- a/src/DataStax.AstraDB.DataApi/Collections/Collection.cs +++ b/src/DataStax.AstraDB.DataApi/Collections/Collection.cs @@ -16,6 +16,7 @@ using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Core.Commands; +using DataStax.AstraDB.DataApi.Core.Enumeration; using DataStax.AstraDB.DataApi.Core.Query; using DataStax.AstraDB.DataApi.Core.Results; using DataStax.AstraDB.DataApi.SerDes; @@ -63,7 +64,7 @@ internal Collection(string collectionName, Database database, CommandOptions com /// /// The type of the documents in the collection. /// The type of the id field for documents in the collection. -public class Collection : IQueryRunner> where T : class +public class Collection where T : class { private readonly string _collectionName; private readonly Database _database; @@ -308,7 +309,7 @@ public Task DropAsync() /// public T FindOne() { - return FindOne(null, new DocumentFindOptions(), null); + return FindOne(null, new CollectionFindOneOptions(), null); } /// @@ -317,7 +318,7 @@ public T FindOne() /// public T FindOne(CommandOptions commandOptions) { - return FindOne(null, new DocumentFindOptions(), commandOptions); + return FindOne(null, new CollectionFindOneOptions(), commandOptions); } /// @@ -326,23 +327,23 @@ public T FindOne(CommandOptions commandOptions) /// public T FindOne(CollectionFilter filter) { - return FindOne(filter, new DocumentFindOptions(), null); + return FindOne(filter, new CollectionFindOneOptions(), null); } /// - /// Synchronous version of + /// Synchronous version of /// - /// - public T FindOne(DocumentFindOptions findOptions) + /// + public T FindOne(CollectionFindOneOptions findOptions) { return FindOne(null, findOptions, null); } /// - /// Synchronous version of + /// Synchronous version of /// - /// - public T FindOne(DocumentFindOptions findOptions, CommandOptions commandOptions) + /// + public T FindOne(CollectionFindOneOptions findOptions, CommandOptions commandOptions) { return FindOne(null, findOptions, commandOptions); } @@ -353,23 +354,23 @@ public T FindOne(DocumentFindOptions findOptions, CommandOptions commandOptio /// public T FindOne(CollectionFilter filter, CommandOptions commandOptions) { - return FindOne(filter, new DocumentFindOptions(), commandOptions); + return FindOne(filter, new CollectionFindOneOptions(), commandOptions); } /// - /// Synchronous version of + /// Synchronous version of /// - /// - public T FindOne(CollectionFilter filter, DocumentFindOptions findOptions) + /// + public T FindOne(CollectionFilter filter, CollectionFindOneOptions findOptions) { return FindOne(filter, findOptions, null); } /// - /// Synchronous version of + /// Synchronous version of /// - /// - public T FindOne(CollectionFilter filter, DocumentFindOptions findOptions, CommandOptions commandOptions) + /// + public T FindOne(CollectionFilter filter, CollectionFindOneOptions findOptions, CommandOptions commandOptions) { return FindOneAsync(filter, findOptions, commandOptions, true).ResultSync(); } @@ -380,7 +381,7 @@ public T FindOne(CollectionFilter filter, DocumentFindOptions findOptions, /// public TResult FindOne() { - return FindOne(null, new DocumentFindOptions(), null); + return FindOne(null, new CollectionFindOneOptions(), null); } /// @@ -389,32 +390,32 @@ public TResult FindOne() /// public TResult FindOne(CommandOptions commandOptions) { - return FindOne(null, new DocumentFindOptions(), commandOptions); + return FindOne(null, new CollectionFindOneOptions(), commandOptions); } /// - /// Synchronous version of + /// Synchronous version of /// - /// + /// public TResult FindOne(CollectionFilter filter) { - return FindOne(filter, new DocumentFindOptions(), null); + return FindOne(filter, new CollectionFindOneOptions(), null); } /// - /// Synchronous version of + /// Synchronous version of /// - /// - public TResult FindOne(DocumentFindOptions findOptions) + /// + public TResult FindOne(CollectionFindOneOptions findOptions) { return FindOne(null, findOptions, null); } /// - /// Synchronous version of + /// Synchronous version of /// - /// - public TResult FindOne(DocumentFindOptions findOptions, CommandOptions commandOptions) + /// + public TResult FindOne(CollectionFindOneOptions findOptions, CommandOptions commandOptions) { return FindOne(null, findOptions, commandOptions); } @@ -425,23 +426,23 @@ public TResult FindOne(DocumentFindOptions findOptions, CommandOptio /// public TResult FindOne(CollectionFilter filter, CommandOptions commandOptions) { - return FindOne(filter, new DocumentFindOptions(), commandOptions); + return FindOne(filter, new CollectionFindOneOptions(), commandOptions); } /// - /// Synchronous version of + /// Synchronous version of /// - /// - public TResult FindOne(CollectionFilter filter, DocumentFindOptions findOptions) + /// + public TResult FindOne(CollectionFilter filter, CollectionFindOneOptions findOptions) { return FindOne(filter, findOptions, null); } /// - /// Synchronous version of + /// Synchronous version of /// - /// - public TResult FindOne(CollectionFilter filter, DocumentFindOptions findOptions, CommandOptions commandOptions) + /// + public TResult FindOne(CollectionFilter filter, CollectionFindOneOptions findOptions, CommandOptions commandOptions) { return FindOneAsync(filter, findOptions, commandOptions, true).ResultSync(); } @@ -452,32 +453,32 @@ public TResult FindOne(CollectionFilter filter, DocumentFindOptions< /// public Task FindOneAsync() { - return FindOneAsync(null, new DocumentFindOptions(), null); + return FindOneAsync(null, new CollectionFindOneOptions(), null); } /// /// public Task FindOneAsync(CommandOptions commandOptions) { - return FindOneAsync(null, new DocumentFindOptions(), commandOptions); + return FindOneAsync(null, new CollectionFindOneOptions(), commandOptions); } /// - /// Returns a single document from the collection based on the provided . - /// This will return the first document found, most often used in conjunction with . - /// See for more details on sorting, projecting and the other options for finding a document. + /// Returns a single document from the collection based on the provided . + /// This will return the first document found, most often used in conjunction with . + /// See for more details on sorting, projecting and the other options for finding a document. /// /// /// - public Task FindOneAsync(DocumentFindOptions findOptions) + public Task FindOneAsync(CollectionFindOneOptions findOptions) { return FindOneAsync(null, findOptions, null); } - /// + /// /// /// - public Task FindOneAsync(DocumentFindOptions findOptions, CommandOptions commandOptions) + public Task FindOneAsync(CollectionFindOneOptions findOptions, CommandOptions commandOptions) { return FindOneAsync(null, findOptions, commandOptions); } @@ -495,7 +496,7 @@ public Task FindOneAsync(DocumentFindOptions findOptions, CommandOptions c /// public Task FindOneAsync(CollectionFilter filter) { - return FindOneAsync(filter, new DocumentFindOptions(), null); + return FindOneAsync(filter, new CollectionFindOneOptions(), null); } /// @@ -503,22 +504,22 @@ public Task FindOneAsync(CollectionFilter filter) /// public Task FindOneAsync(CollectionFilter filter, CommandOptions commandOptions) { - return FindOneAsync(filter, new DocumentFindOptions(), commandOptions); + return FindOneAsync(filter, new CollectionFindOneOptions(), commandOptions); } /// /// /// - public Task FindOneAsync(CollectionFilter filter, DocumentFindOptions findOptions) + public Task FindOneAsync(CollectionFilter filter, CollectionFindOneOptions findOptions) { return FindOneAsync(filter, findOptions, null); } - /// + /// /// /// /// - public Task FindOneAsync(CollectionFilter filter, DocumentFindOptions findOptions, CommandOptions commandOptions) + public Task FindOneAsync(CollectionFilter filter, CollectionFindOneOptions findOptions, CommandOptions commandOptions) { return FindOneAsync(filter, findOptions, commandOptions, false); } @@ -534,14 +535,14 @@ public Task FindOneAsync(CollectionFilter filter, DocumentFindOptions f /// public Task FindOneAsync() { - return FindOneAsync(null, new DocumentFindOptions(), null); + return FindOneAsync(null, new CollectionFindOneOptions(), null); } /// /// public Task FindOneAsync(CommandOptions commandOptions) { - return FindOneAsync(null, new DocumentFindOptions(), commandOptions); + return FindOneAsync(null, new CollectionFindOneOptions(), commandOptions); } /// @@ -550,22 +551,22 @@ public Task FindOneAsync(CommandOptions commandOptions) /// /// var exclusiveProjection = Builders<FullObject>.Projection /// .Exclude("PropertyTwo"); - /// var findOptions = new FindOptions<FullObject>() + /// var findOptions = new CollectionFindOneOptions<FullObject>() /// { /// Projection = exclusiveProjection /// }; /// var result = await collection.FindOneAsync<ObjectWithoutPropertyTwo>(findOptions); /// /// - public Task FindOneAsync(DocumentFindOptions findOptions) + public Task FindOneAsync(CollectionFindOneOptions findOptions) { return FindOneAsync(null, findOptions, null); } - /// + /// /// /// - public Task FindOneAsync(DocumentFindOptions findOptions, CommandOptions commandOptions) + public Task FindOneAsync(CollectionFindOneOptions findOptions, CommandOptions commandOptions) { return FindOneAsync(null, findOptions, commandOptions); } @@ -575,39 +576,30 @@ public Task FindOneAsync(DocumentFindOptions findOptions, C /// public Task FindOneAsync(CollectionFilter filter, CommandOptions commandOptions) { - return FindOneAsync(filter, new DocumentFindOptions(), commandOptions); + return FindOneAsync(filter, new CollectionFindOneOptions(), commandOptions); } - /// + /// /// /// - public Task FindOneAsync(CollectionFilter filter, DocumentFindOptions findOptions) + public Task FindOneAsync(CollectionFilter filter, CollectionFindOneOptions findOptions) { return FindOneAsync(filter, findOptions, null); } - /// + /// /// /// /// - public Task FindOneAsync(CollectionFilter filter, DocumentFindOptions findOptions, CommandOptions commandOptions) + public Task FindOneAsync(CollectionFilter filter, CollectionFindOneOptions findOptions, CommandOptions commandOptions) { return FindOneAsync(filter, findOptions, commandOptions, false); } - private async Task FindOneAsync(CollectionFilter filter, DocumentFindOptions findOptions, CommandOptions commandOptions, bool runSynchronously) + private async Task FindOneAsync(CollectionFilter filter, CollectionFindOneOptions findOptions, CommandOptions commandOptions, bool runSynchronously) { - findOptions = findOptions != null ? findOptions.Clone() : new DocumentFindOptions(); - if (filter != null) - { - if (findOptions.Filter == null) - { - findOptions.Filter = filter; - } else - { - throw new ArgumentException("Cannot pass a filter both within FindOptions and as stand-alone argument"); - } - } + findOptions = findOptions != null ? findOptions.Clone() : new CollectionFindOneOptions(); + findOptions = findOptions.WithFilterParam(filter); var command = CreateCommand("findOne").WithPayload(findOptions).AddCommandOptions(commandOptions); var response = await command.RunAsyncReturnDocumentData, TResult, FindStatusResult>(runSynchronously).ConfigureAwait(false); return response.Data.Document; @@ -616,18 +608,17 @@ private async Task FindOneAsync(CollectionFilter filter, Do /// /// Find documents in the collection. /// - /// The Find() methods return a object that can be used to further structure the query + /// The Find() methods return a object that can be used to further structure the query /// by adding Sort, Projection, Skip, Limit, etc. to affect the final results. /// - /// The object can be directly enumerated both synchronously and asynchronously. - /// Secondly, the results can be paged through more manually by using the method. + /// The object can be directly enumerated both synchronously and asynchronously. /// /// /// /// Synchronous Enumeration: /// - /// var FindEnumerator = collection.Find(); - /// foreach (var document in FindEnumerator) + /// var cursor = collection.Find(); + /// foreach (var document in cursor) /// { /// // Process document /// } @@ -649,7 +640,7 @@ private async Task FindOneAsync(CollectionFilter filter, Do /// however BulkOperationCancellationToken settings are ignored due to the nature of Enumeration. /// If you need to enforce a timeout for the entire operation, you can pass a to GetAsyncEnumerator. /// - public FindEnumerator> Find() + public CollectionFindCursor Find() { return Find(null, null); } @@ -664,24 +655,25 @@ public FindEnumerator> Find() /// var results = collection.Find(filter).Sort(sort); /// /// - public FindEnumerator> Find(CollectionFilter filter) + public CollectionFindCursor Find(CollectionFilter filter) { return Find(filter, null); } /// - /// - public FindEnumerator> Find(CommandOptions commandOptions) + /// + public CollectionFindCursor Find(CollectionFindManyOptions findOptions) { - return Find(null, commandOptions); + return Find(null, findOptions); } /// /// - /// - public FindEnumerator> Find(CollectionFilter filter, CommandOptions commandOptions) + /// + public CollectionFindCursor Find(CollectionFilter filter, CollectionFindManyOptions findOptions) { - return Find(filter, commandOptions); + findOptions ??= new CollectionFindManyOptions(); + return new(findOptions.WithFilterParam(filter), null, RunFindManyAsync); } /// @@ -689,7 +681,7 @@ public FindEnumerator> Find(CollectionFilter f /// The Find alternatives that accept a TResult type parameter allow for deserializing the document as a different type /// (most commonly used when using projection to return a subset of fields) /// - public FindEnumerator> Find() where TResult : class + public CollectionFindCursor Find() where TResult : class { return Find(null, null); } @@ -699,39 +691,43 @@ public FindEnumerator> Find() wher /// The Find alternatives that accept a TResult type parameter allow for deserializing the document as a different type /// (most commonly used when using projection to return a subset of fields) /// - public FindEnumerator> Find(CollectionFilter filter) where TResult : class + public CollectionFindCursor Find(CollectionFilter filter) where TResult : class { return Find(filter, null); } - /// + /// /// /// The Find alternatives that accept a TResult type parameter allow for deserializing the document as a different type /// (most commonly used when using projection to return a subset of fields) /// - public FindEnumerator> Find(CommandOptions commandOptions) where TResult : class + public CollectionFindCursor Find(CollectionFindManyOptions findOptions) where TResult : class { - return Find(null, commandOptions); + return Find(null, findOptions); } /// /// - /// - public FindEnumerator> Find(CollectionFilter filter, CommandOptions commandOptions) where TResult : class + /// + public CollectionFindCursor Find(CollectionFilter filter, CollectionFindManyOptions findOptions) where TResult : class { - var findOptions = new DocumentFindManyOptions() - { - Filter = filter - }; - return new FindEnumerator>(this, findOptions, commandOptions); + findOptions ??= new CollectionFindManyOptions(); + return new(findOptions.WithFilterParam(filter), null, RunFindManyAsync); } - internal async Task, FindStatusResult>> RunFindManyAsync(Filter filter, IFindManyOptions> findOptions, CommandOptions commandOptions, bool runSynchronously) + internal async Task> RunFindManyAsync(CollectionFindCursor cursor, string nextPageState, bool runSynchronously) where TResult : class { - findOptions.Filter = filter; - var command = CreateCommand("find").WithPayload(findOptions).AddCommandOptions(commandOptions); + var options = cursor.FindOptions.Clone(); + options.PageState = nextPageState; + + var command = CreateCommand("find").WithPayload(options).AddCommandOptions(cursor.CommandOptions); var response = await command.RunAsyncReturnDocumentData, TResult, FindStatusResult>(runSynchronously).ConfigureAwait(false); - return response; + + return new FindPage( + response.Data.NextPageState, + response.Data.Items, + response.Status?.SortVector + ); } /// @@ -2119,9 +2115,4 @@ internal Command CreateCommand(string name) var optionsTree = GetOptionsTree().ToArray(); return new Command(name, _database.Client, optionsTree, new DatabaseCommandUrlBuilder(_database, _collectionName)); } - - Task, FindStatusResult>> IQueryRunner>.RunFindManyAsync(Filter filter, IFindManyOptions> findOptions, CommandOptions commandOptions, bool runSynchronously) - { - return RunFindManyAsync(filter, findOptions, commandOptions, runSynchronously); - } } diff --git a/src/DataStax.AstraDB.DataApi/Core/Cursor.cs b/src/DataStax.AstraDB.DataApi/Core/Cursor.cs deleted file mode 100644 index a937c1cc..00000000 --- a/src/DataStax.AstraDB.DataApi/Core/Cursor.cs +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright DataStax, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using DataStax.AstraDB.DataApi.Core.Results; -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; - -namespace DataStax.AstraDB.DataApi.Core; - -/// -/// A cursor for iterating over the results of a query in a streaming manner. -/// -/// When multiple results are returned by the underlying API, they are returned in batches. -/// You can use the method to iterate over the batches -/// and to access the current batch of results. -/// -/// The and methods create a new cursor to ensure -/// iteration starts from the first batch, allowing multiple enumerations of the same query. -/// Use the cursor directly if you need access to the SortVector or want to manually control iterating over the batches. -/// -/// The type of the documents in the collection. -public class Cursor : IDisposable, IParentCursor -{ - private readonly SemaphoreSlim _semaphore = new SemaphoreSlim(1, 1); - private ApiFindResult _currentBatch; - private readonly Func, FindStatusResult>>> _fetchNextBatch; - private readonly IParentCursor _parentCursor; - - - internal Cursor( - Func, FindStatusResult>>> fetchNextBatch, - IParentCursor parentCursor = null) - { - _fetchNextBatch = fetchNextBatch ?? throw new ArgumentNullException(nameof(fetchNextBatch)); - _parentCursor = parentCursor; - } - - /// - /// Gets a value indicating whether the cursor has been started or not. - /// - public bool IsStarted { get; internal set; } = false; - - /// - /// The current batch of results. - /// - public IEnumerable Current - { - get - { - if (_currentBatch == null) - { - throw new InvalidOperationException("Cursor has not been started. Call MoveNextAsync first."); - } - return _currentBatch.Items; - } - } - - /// - /// An array containing the sort vectors used for the query that created this cursor. - /// - public float[] SortVector { get; private set; } - - /// - /// Synchronously moves the cursor to the next batch of results. - /// - /// True if there are more batches, false otherwise. - /// - /// The asynchronous version is recommended to avoid potential deadlocks. - /// - public bool MoveNext() - { - _semaphore.Wait(); - try - { - return MoveNextAsync(true).GetAwaiter().GetResult(); - } - finally - { - _semaphore.Release(); - } - } - - /// - /// Asynchronously moves the cursor to the next batch of results. - /// - /// An optional cancellation token to cancel the operation. - /// True if there are more batches, false otherwise. - public async Task MoveNextAsync(CancellationToken cancellationToken = default) - { - await _semaphore.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - return await MoveNextAsync(false, cancellationToken).ConfigureAwait(false); - } - finally - { - _semaphore.Release(); - } - } - - private async Task MoveNextAsync(bool runSynchronously, CancellationToken cancellationToken = default) - { - if (_currentBatch != null && string.IsNullOrEmpty(_currentBatch.NextPageState)) - { - return false; - } - - _parentCursor?.SetStarted(); - - var nextPageState = _currentBatch?.NextPageState; - var nextBatch = await _fetchNextBatch(nextPageState, cancellationToken, runSynchronously).ConfigureAwait(false); - if (nextBatch.Data == null || nextBatch.Data.Items == null || nextBatch.Data.Items.Count == 0) - { - return false; - } - - var nextSortVector = nextBatch.Status?.SortVector; - _parentCursor?.SetSortVector(nextSortVector); - - _currentBatch = nextBatch.Data; - return true; - } - - /// - /// Releases all resources used by the cursor. - /// - public void Dispose() - { - _semaphore.Dispose(); - } - - /// - /// Converts the cursor to an IAsyncEnumerator, starting from the first batch. - /// - /// An optional cancellation token. - /// An IAsyncEnumerator containing all documents in the cursor. - public async IAsyncEnumerator ToAsyncEnumerator(CancellationToken cancellationToken = default) - { - using var newCursor = new Cursor(_fetchNextBatch, this); - while (await newCursor.MoveNextAsync(cancellationToken).ConfigureAwait(false)) - { - foreach (var item in newCursor.Current) - { - cancellationToken.ThrowIfCancellationRequested(); - yield return item; - } - } - } - - /// - /// Converts the cursor to an IEnumerable, starting from the first batch. - /// - /// An IEnumerable containing all documents in the cursor. - public IEnumerable ToEnumerable() - { - using var newCursor = new Cursor(_fetchNextBatch, this); - while (newCursor.MoveNext()) - { - foreach (var item in newCursor.Current) - { - yield return item; - } - } - } - - void IParentCursor.SetSortVector(float[] sortVector) - { - if ((SortVector == null || SortVector.Length == 0) && sortVector != null && sortVector.Length > 0) - { - SortVector = sortVector; - } - } - - void IParentCursor.SetStarted() - { - IsStarted = true; - } - -} diff --git a/src/DataStax.AstraDB.DataApi/Core/Enumeration/AbstractCursor.cs b/src/DataStax.AstraDB.DataApi/Core/Enumeration/AbstractCursor.cs new file mode 100644 index 00000000..cc187a0e --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Enumeration/AbstractCursor.cs @@ -0,0 +1,285 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Utils; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace DataStax.AstraDB.DataApi.Core.Enumeration; + +/// +/// Represents the current state of a cursor. +/// +public enum CursorState +{ + /// + /// The cursor has not been started yet. + /// + Idle, + /// + /// The cursor has been started and is actively iterating. + /// + Started, + /// + /// The cursor has been closed and can no longer be used. + /// + Closed +} + +/// +/// Exception thrown when a cursor operation is attempted in an invalid state. +/// +public class CursorException : Exception +{ + /// + /// Initializes a new instance of the class. + /// + /// The error message. + /// The cursor state when the error occurred. + public CursorException(string message, CursorState state) : base(message) + { + CursorState = state; + } + + /// + /// Gets the cursor state when the error occurred. + /// + public CursorState CursorState { get; } +} + +/// +/// Abstract base class for cursors that iterate over query results in a streaming manner. +/// +/// This cursor provides state management, buffering, and both synchronous and asynchronous enumeration +/// capabilities. Results are fetched in batches from the underlying API and buffered for efficient iteration. +/// +/// The type of the items or rows in the cursor. +/// The type of the cursor. +public abstract class AbstractCursor : IDisposable, IEnumerable, IAsyncEnumerable + where TCursor : AbstractCursor +{ + /// + /// Gets the internal buffer containing fetched results. + /// + protected abstract List _buffer { get; } + private bool _isNextPage = true; + + /// + /// Gets the current state of the cursor. + /// + public CursorState State { get; protected set; } = CursorState.Idle; + + /// + /// Returns the number of items currently buffered in memory. + /// + /// The count of buffered items. + public int Buffered() => _buffer?.Count ?? 0; + + /// + /// Gets the total number of items consumed from the cursor so far. + /// + public int Consumed { get; protected set; } + + /// + /// Consumes and returns items from the buffer. + /// + /// The maximum number of items to consume. If 0 or omitted, consumes all buffered items. + /// A read-only list of consumed items. + public IReadOnlyList ConsumeBuffer(int max = 0) + { + if (_buffer == null || _buffer.Count == 0) + { + return new List(); + } + + var count = max > 0 ? Math.Min(max, _buffer.Count) : _buffer.Count; + var result = _buffer.Take(count).ToList(); + _buffer.RemoveRange(0, count); + Consumed += count; + + return result; + } + + /// + /// Synchronously checks if there are more items available without consuming them. + /// + /// True if there are more items available, false otherwise. + /// + /// The asynchronous version is recommended to avoid potential deadlocks. + /// + public bool HasNext() + { + return MoveNextAsync(CancellationToken.None, peek: true, runSynchronously: true).ResultSync() != null; + } + + /// + /// Asynchronously checks if there are more items available without consuming them. + /// + /// An optional cancellation token to cancel the operation. + /// True if there are more items available, false otherwise. + public async Task HasNextAsync(CancellationToken cancellationToken = default) + { + return await MoveNextAsync(cancellationToken, peek: true, runSynchronously: false) != null; + } + + /// + /// Synchronously moves to and returns the next item in the cursor. + /// + /// The next item, or null if there are no more items. + /// + /// The asynchronous version is recommended to avoid potential deadlocks. + /// + public T MoveNext() + { + return MoveNextAsync(CancellationToken.None, peek: false, runSynchronously: true).ResultSync(); + } + + /// + /// Asynchronously moves to and returns the next item in the cursor. + /// + /// An optional cancellation token to cancel the operation. + /// The next item, or null if there are no more items. + public async Task MoveNextAsync(CancellationToken cancellationToken = default) + { + return await MoveNextAsync(cancellationToken, peek: false, runSynchronously: false); + } + + /// + /// Returns an async enumerator to iterate over all items in the cursor. + /// + /// An optional cancellation token to cancel the operation. + /// An async enumerator for the cursor. + /// Thrown when attempting to iterate over a closed cursor. + public async IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) + { + if (State == CursorState.Closed) + { + throw new CursorException("Cannot iterate over a closed cursor", State); + } + + T doc; + while ((doc = await MoveNextAsync(cancellationToken)) != null) + { + cancellationToken.ThrowIfCancellationRequested(); + yield return doc; + } + } + + /// + /// Returns an enumerator to iterate over all items in the cursor. + /// + /// An enumerator for the cursor. + /// Thrown when attempting to iterate over a closed cursor. + public IEnumerator GetEnumerator() + { + if (State == CursorState.Closed) + { + throw new CursorException("Cannot iterate over a closed cursor", State); + } + + T doc; + while ((doc = MoveNext()) != null) + { + yield return doc; + } + } + + /// + /// Returns an enumerator to iterate over all items in the cursor. + /// + /// An enumerator for the cursor. + IEnumerator IEnumerable.GetEnumerator() + { + return GetEnumerator(); + } + + /// + /// Creates a new cursor instance with the same configuration. + /// + /// A new cursor instance. + public abstract TCursor Clone(); + + /// + /// Releases all resources used by the cursor and sets its state to . + /// + public virtual void Dispose() + { + State = CursorState.Closed; + _isNextPage = false; + _buffer?.Clear(); + } + + /// + /// Resets the cursor to its initial state, allowing iteration to start over from the beginning. + /// + /// + /// This resets the consumed count and cursor state, but does not refetch data. + /// The next iteration will start from the first page again. + /// + public virtual void Rewind() + { + Consumed = 0; + State = CursorState.Idle; + _isNextPage = true; + } + + /// + /// Fetches the next page of results from the underlying data source. + /// + /// A cancellation token to cancel the operation. + /// Whether to run the operation synchronously. + /// True if more pages are available, false otherwise. + protected abstract Task FetchMoreAsync(CancellationToken cancellationToken, bool runSynchronously); + + /// + /// Internal method to move to the next item, with options to peek or consume. + /// + /// A cancellation token to cancel the operation. + /// If true, returns the next item without consuming it. + /// Whether to run the operation synchronously. + /// The next item, or null if there are no more items. + protected async Task MoveNextAsync(CancellationToken cancellationToken, bool peek, bool runSynchronously) + { + if (State == CursorState.Closed) + { + return default; + } + + State = CursorState.Started; + + while (_buffer == null || _buffer.Count == 0) + { + if (!_isNextPage) + { + State = CursorState.Closed; + return default; + } + + _isNextPage = await FetchMoreAsync(cancellationToken, runSynchronously).ConfigureAwait(false); + } + + if (peek) + { + return _buffer.FirstOrDefault(); + } + + return ConsumeBuffer(1).FirstOrDefault(); + } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Enumeration/CollectionFindCursor.cs b/src/DataStax.AstraDB.DataApi/Core/Enumeration/CollectionFindCursor.cs new file mode 100644 index 00000000..cfd89d0d --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Enumeration/CollectionFindCursor.cs @@ -0,0 +1,120 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Collections; +using DataStax.AstraDB.DataApi.Core.Query; + +namespace DataStax.AstraDB.DataApi.Core.Enumeration; + +/// +/// A cursor for finding and enumerating documents in a collection. +/// This is a convenience class that uses the same type for both the document and result. +/// +/// The type of the documents in the collection. +/// +/// This cursor is returned by and provides a fluent API +/// for filtering, sorting, limiting, and projecting documents. It supports both synchronous +/// and asynchronous iteration patterns. +/// +/// +/// +/// // Basic usage with foreach +/// var cursor = collection.Find() +/// .Filter(Builders<MyDocument>.Filter.Eq(d => d.Status, "active")) +/// .Sort(Builders<MyDocument>.Sort.Ascending(d => d.Name)) +/// .Limit(10); +/// +/// foreach (var doc in cursor) +/// { +/// Console.WriteLine(doc.Name); +/// } +/// +/// // Async iteration +/// await foreach (var doc in cursor) +/// { +/// await ProcessDocumentAsync(doc); +/// } +/// +/// +public class CollectionFindCursor : CollectionFindCursor where T : class +{ + internal CollectionFindCursor(IFindManyOptions> options, CommandOptions commandOptions, FetchPageFunc> fetchPage) + : base(options, commandOptions, fetchPage) { } +} + +/// +/// A cursor for finding and enumerating documents in a collection with projection support. +/// This class allows you to specify a different result type than the document type, useful for projections. +/// +/// The type of the documents in the collection. +/// The type to deserialize the results to (e.g., when using projections). +/// +/// This cursor is returned by and provides a fluent API +/// for filtering, sorting, limiting, and projecting documents into a different result type. +/// +/// +/// +/// // Using projection to return only specific fields +/// public class MyDocumentProjection +/// { +/// public string Name { get; set; } +/// public string Email { get; set; } +/// } +/// +/// var cursor = collection.Find<MyDocumentProjection>() +/// .Project(Builders<MyDocument>.Projection +/// .Include(d => d.Name) +/// .Include(d => d.Email)) +/// .Limit(100); +/// +/// var results = await cursor.ToListAsync(); +/// +/// +public class CollectionFindCursor : FindCursor, CollectionFindCursor> + where T : class + where TResult : class +{ + /// + /// Initializes a new instance of the class. + /// + /// The find options to use. + /// The command options to use. + /// The function to fetch pages of results. + internal CollectionFindCursor( + IFindManyOptions> options, + CommandOptions commandOptions, + FetchPageFunc> fetchPage + ) : base(options, commandOptions, fetchPage) { } + + /// + /// Creates a new cursor instance with the same configuration. + /// + /// A new cursor instance. + public override CollectionFindCursor Clone() + { + return new(FindOptions.Clone(), CommandOptions, FetchPageFunc); + } + + /// + /// Creates a new cursor instance with updated find options. + /// + /// The updated find options. + /// A new cursor instance with the updated options. + internal override CollectionFindCursor CloneWithOptions(IFindManyOptions> options) + { + return new(options, CommandOptions, FetchPageFunc); + } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Enumeration/FindCursor.cs b/src/DataStax.AstraDB.DataApi/Core/Enumeration/FindCursor.cs new file mode 100644 index 00000000..fdee59d2 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Enumeration/FindCursor.cs @@ -0,0 +1,393 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Core.Query; +using DataStax.AstraDB.DataApi.Utils; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace DataStax.AstraDB.DataApi.Core.Enumeration; + +/// +/// Represents a page of results from a find operation. +/// +/// The type of records in the page. +public class FindPage +{ + /// + /// Initializes a new instance of the class. + /// + /// The page state token for fetching the next page, or null if this is the last page. + /// The records in this page. + /// The sort vector used for vector similarity searches, if applicable. + public FindPage(string nextPageState, List result, float[] sortVector) + { + NextPageState = nextPageState; + Results = new List(result); + SortVector = sortVector; + } + + /// + /// Gets the page state token for fetching the next page, or null if this is the last page. + /// + public string NextPageState { get; } + + /// + /// Gets the list of records in this page. + /// + public List Results { get; internal set; } + + /// + /// Gets the sort vector used for vector similarity searches, if applicable. + /// + public float[] SortVector { get; } +} + +/// +/// Delegate for fetching a page of results. +/// +/// The type of records in the page. +/// The type of the cursor. +/// The cursor instance. +/// Whether to run the operation synchronously. +/// A task that returns the fetched page. +delegate Task> FetchPageFunc(TCursor cursor, string nextPageState, bool runSynchronously); + +/// +/// A fluent API cursor for finding and enumerating records or rows with filtering, sorting, and projection capabilities. +/// +/// This cursor extends to provide query-specific operations like filtering, sorting, +/// limiting, skipping, and projecting results. It supports both synchronous and asynchronous iteration patterns. +/// +/// Use the fluent methods to refine your query, then iterate using foreach, LINQ, or manual cursor navigation. +/// +/// The type representing the record or row being queried. +/// The type to deserialize the results to (e.g., when using projections). +/// The type of sort builder to use (e.g., or ). +/// The concrete cursor type for fluent method chaining. +public abstract class FindCursor : AbstractCursor + where T : class + where TResult : class + where TSort : SortBuilder + where TCursor : FindCursor +{ + /// + /// Gets the find options used for this cursor. + /// + internal IFindManyOptions FindOptions { get; } + + /// + /// Gets the command options used for this cursor. + /// + internal CommandOptions CommandOptions { get; } + + /// + /// Gets the function used to fetch pages of results. + /// + internal readonly FetchPageFunc FetchPageFunc; + + private FindPage _currentPage; + + /// + /// Gets the internal buffer containing the current page of results. + /// + protected override List _buffer => _currentPage?.Results; + + /// + /// Initializes a new instance of the class. + /// + /// The find options to use. + /// The command options to use. + /// The function to fetch pages of results. + internal FindCursor(IFindManyOptions options, CommandOptions commandOptions, FetchPageFunc fetchPage) + { + FindOptions = options.Clone(); + CommandOptions = commandOptions; + FetchPageFunc = fetchPage; + + if (options.PageState != null) + { + _currentPage = new FindPage(options.PageState, new List(), null); + } + } + + /// + /// Specifies a filter to apply to the query. + /// + /// The filter to apply. + /// A new cursor instance with the updated filter. + /// + /// + /// var filter = Builders<MyRecord>.Filter.Eq(d => d.Status, "active"); + /// var cursor = collection.Find().Filter(filter); + /// + /// + public TCursor Filter(Filter filter) + { + return UpdateOptions(options => options.Filter = filter); + } + + /// + /// Specifies a sort to apply to the query results. + /// + /// The sort to apply. + /// A new cursor instance with the updated sort. + /// + /// + /// var sort = Builders<MyRecord>.Sort.Ascending(d => d.Name); + /// var cursor = collection.Find().Sort(sort); + /// + /// + public TCursor Sort(TSort sort) + { + return UpdateOptions(options => options.Sort = sort); + } + + /// + /// Specifies the maximum number of records to return. + /// + /// The maximum number of records to return. + /// A new cursor instance with the updated limit. + public TCursor Limit(int limit) + { + return UpdateOptions(options => options.Limit = limit); + } + + /// + /// Specifies the number of records to skip before starting to return records. + /// Use in conjunction with to determine the order before skipping. + /// + /// The number of records to skip. + /// A new cursor instance with the updated skip value. + public TCursor Skip(int skip) + { + return UpdateOptions(options => options.Skip = skip); + } + + /// + /// Specifies a projection to apply to the results. + /// + /// The projection to apply. + /// A new cursor instance with the updated projection. + /// + /// + /// var projection = Builders<MyRecord>.Projection.Include(d => d.Name).Include(d => d.Email); + /// var cursor = collection.Find().Project(projection); + /// + /// + public TCursor Project(IProjectionBuilder projection) + { + return UpdateOptions(options => options.Projection = projection); + } + + /// + /// Specifies whether to include the similarity score in the results. + /// + /// Whether to include the similarity score. Defaults to true. + /// A new cursor instance with the updated setting. + /// + /// Use the with + /// to map the similarity score to a property in your result class. + /// + /// + /// + /// public class MyDocumentWithSimilarity : MyDocument + /// { + /// [DocumentMapping(DocumentMappingField.Similarity)] + /// public double? Similarity { get; set; } + /// } + /// + /// var cursor = collection.Find<MyDocumentWithSimilarity>() + /// .Sort(Builders<MyDocument>.Sort.Vector(vectorQuery)) + /// .IncludeSimilarity(); + /// + /// + public TCursor IncludeSimilarity(bool include = true) + { + return UpdateOptions(options => options.IncludeSimilarity = include); + } + + /// + /// Specifies whether to include the sort vector in the results. + /// + /// Whether to include the sort vector. Defaults to true. + /// A new cursor instance with the updated setting. + /// + /// When enabled, you can retrieve the sort vector using or . + /// This is useful for vector similarity searches where you want to access the vector used for sorting. + /// + /// + /// + /// var cursor = collection.Find() + /// .Sort(Builders<MyRecord>.Sort.Vectorize("search query")) + /// .IncludeSortVector(); + /// + /// await foreach (var doc in cursor) + /// { + /// // Process records + /// } + /// + /// var sortVector = await cursor.GetSortVectorAsync(); + /// + /// + public TCursor IncludeSortVector(bool include = true) + { + return UpdateOptions(options => options.IncludeSortVector = include); + } + + /// + /// Sets the initial page state used to resume pagination from a previous find operation. + /// + /// The page state to resume from, or null to start from the beginning. + /// A new cursor instance with the updated initial page state. + public TCursor InitialPageState(string initialPageState) + { + return UpdateOptions(options => options.PageState = initialPageState); + } + + /// + /// Synchronously fetches the next complete page of results from the server. + /// + /// The next page of results. + /// Thrown when the cursor is closed or the current buffer is not empty. + public FindPage FetchNextPage() + { + return FetchNextPageAsync(CancellationToken.None, true).ResultSync(); + } + + /// + /// Asynchronously fetches the next complete page of results from the server. + /// + /// An optional cancellation token to cancel the operation. + /// The next page of results. + /// Thrown when the cursor is closed or the current buffer is not empty. + public async Task> FetchNextPageAsync(CancellationToken cancellationToken = default) + { + return await FetchNextPageAsync(cancellationToken, false).ConfigureAwait(false); + } + + private async Task> FetchNextPageAsync(CancellationToken cancellationToken, bool runSynchronously) + { + if ((_buffer?.Count ?? 0) > 0) + { + throw new CursorException("Cannot fetch next page when the current page (the buffer) is not empty", State); + } + + if (State == CursorState.Closed) + { + throw new CursorException("Cannot fetch next page on a closed cursor", State); + } + + await MoveNextAsync(cancellationToken, peek: true, runSynchronously).ConfigureAwait(false); + + var buffer = _currentPage.Results; + _currentPage.Results = new List(); + + return new FindPage(_currentPage.NextPageState, buffer, _currentPage.SortVector); + } + + /// + /// Synchronously retrieves the sort vector used for the query. + /// + /// The sort vector, or null if not available. + /// + /// The cursor must have been started (at least one record fetched) and must be enabled. + /// The asynchronous version is recommended to avoid potential deadlocks. + /// + public float[] GetSortVector() + { + return GetSortVectorAsync(CancellationToken.None, true).ResultSync(); + } + + /// + /// Asynchronously retrieves the sort vector used for the query. + /// + /// An optional cancellation token to cancel the operation. + /// The sort vector, or null if not available. + /// + /// The cursor must have been started (at least one record fetched) and must be enabled. + /// If the cursor hasn't been started yet, this method will automatically fetch the first page to retrieve the sort vector. + /// + public async Task GetSortVectorAsync(CancellationToken cancellationToken = default) + { + return await GetSortVectorAsync(cancellationToken, false); + } + + private async Task GetSortVectorAsync(CancellationToken cancellationToken, bool runSynchronously) + { + if (_currentPage == null && FindOptions.IncludeSortVector == true) + { + await MoveNextAsync(cancellationToken, peek: true, runSynchronously).ConfigureAwait(false); + } + return _currentPage?.SortVector; + } + + /// + /// Resets the cursor to its initial state, clearing the current page. + /// + public override void Rewind() + { + base.Rewind(); + _currentPage = null; + } + + /// + /// Releases all resources used by the cursor and clears the current page. + /// + public override void Dispose() + { + base.Dispose(); + _currentPage = null; + } + + /// + /// Fetches the next page of results from the underlying data source. + /// + /// A cancellation token to cancel the operation. + /// Whether to run the operation synchronously. + /// True if more pages are available, false otherwise. + protected override async Task FetchMoreAsync(CancellationToken cancellationToken, bool runSynchronously) + { + if (cancellationToken != CancellationToken.None) + { + CommandOptions.BulkOperationCancellationToken = cancellationToken; + } + + _currentPage = await FetchPageFunc((TCursor)this, _currentPage?.NextPageState, runSynchronously).ConfigureAwait(false); + + return _currentPage.NextPageState != null; + } + + private TCursor UpdateOptions(Action> optionsUpdater) + { + if (State != CursorState.Idle) + { + throw new CursorException("Cursors must be idle when building their options", State); + } + var newOptions = FindOptions.Clone(); + optionsUpdater(newOptions); + return CloneWithOptions(newOptions); + } + + /// + /// Creates a new cursor instance with updated find options. + /// + /// The updated find options. + /// A new cursor instance with the updated options. + internal abstract TCursor CloneWithOptions(IFindManyOptions options); +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Enumeration/TableFindCursor.cs b/src/DataStax.AstraDB.DataApi/Core/Enumeration/TableFindCursor.cs new file mode 100644 index 00000000..0da2b29c --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Enumeration/TableFindCursor.cs @@ -0,0 +1,122 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using DataStax.AstraDB.DataApi.Core.Query; +using DataStax.AstraDB.DataApi.Tables; + +namespace DataStax.AstraDB.DataApi.Core.Enumeration; + +/// +/// A cursor for finding and enumerating rows in a table. +/// This is a convenience class that uses the same type for both the row and result. +/// +/// The type of the rows in the table. +/// +/// This cursor is returned by and provides a fluent API +/// for filtering, sorting, limiting, and projecting rows. It supports both synchronous +/// and asynchronous iteration patterns. +/// +/// +/// +/// // Basic usage with foreach +/// var cursor = table.Find() +/// .Filter(Builders<MyRow>.Filter.Eq(r => r.Status, "active")) +/// .Sort(Builders<MyRow>.Sort.Ascending(r => r.CreatedAt)) +/// .Limit(10); +/// +/// foreach (var row in cursor) +/// { +/// Console.WriteLine(row.Name); +/// } +/// +/// // Async iteration +/// await foreach (var row in cursor) +/// { +/// await ProcessRowAsync(row); +/// } +/// +/// +public class TableFindCursor : TableFindCursor where T : class +{ + internal TableFindCursor(IFindManyOptions> options, CommandOptions commandOptions, FetchPageFunc> fetchPage) + : base(options, commandOptions, fetchPage) { } +} + +/// +/// A cursor for finding and enumerating rows in a table with projection support. +/// This class allows you to specify a different result type than the row type, useful for projections. +/// +/// The type of the rows in the table. +/// The type to deserialize the results to (e.g., when using projections). +/// +/// This cursor is returned by and provides a fluent API +/// for filtering, sorting, limiting, and projecting rows into a different result type. +/// +/// +/// +/// // Using projection to return only specific columns +/// public class MyRowProjection +/// { +/// public string Name { get; set; } +/// public string Email { get; set; } +/// } +/// +/// var cursor = table.Find<MyRowProjection>() +/// .Project(Builders<MyRow>.Projection +/// .Include(r => r.Name) +/// .Include(r => r.Email)) +/// .Limit(100); +/// +/// var results = await cursor.ToListAsync(); +/// +/// +public class TableFindCursor : FindCursor, TableFindCursor> + where T : class + where TResult : class +{ + /// + /// Initializes a new instance of the class. + /// + /// The find options to use. + /// The command options to use. + /// The function to fetch pages of results. + internal TableFindCursor( + IFindManyOptions> options, + CommandOptions commandOptions, + FetchPageFunc> fetchPage + ) : base(options, commandOptions, fetchPage) + { + } + + /// + /// Creates a new cursor instance with the same configuration. + /// + /// A new cursor instance. + public override TableFindCursor Clone() + { + return new(FindOptions.Clone(), CommandOptions, FetchPageFunc); + } + + /// + /// Creates a new cursor instance with updated find options. + /// + /// The updated find options. + /// A new cursor instance with the updated options. + internal override TableFindCursor CloneWithOptions(IFindManyOptions> options) + { + return new(options, CommandOptions, FetchPageFunc); + } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/CollectionFindManyOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/CollectionFindManyOptions.cs new file mode 100644 index 00000000..345b5c1d --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/CollectionFindManyOptions.cs @@ -0,0 +1,151 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +/// +/// Options for finding multiple documents in a collection. +/// +/// The type of the document. +public class CollectionFindManyOptions : IFindManyOptions> +{ + /// The projection to apply to the results. + [JsonIgnore] + public IProjectionBuilder Projection { get; set; } + + /// + /// Whether to include a similarity score in the results or not (when performing a vector sort). + /// + [JsonIgnore] + public bool? IncludeSimilarity { get; set; } + + /// The sort to apply when running the query. + [JsonIgnore] + public CollectionSortBuilder Sort { get; set; } + + /// + /// The initial page state used to resume pagination from a previous find-many operation. + /// + [JsonIgnore] + public string InitialPageState { get => PageState; set => PageState = value; } + + [JsonIgnore] + public int? Skip { get; set; } + + [JsonIgnore] + public int? Limit { get; set; } + + [JsonIgnore] + internal bool? IncludeSortVector { get; set; } + + bool? IFindManyOptions>.IncludeSortVector { get => IncludeSortVector; set => IncludeSortVector = value; } + + internal Filter Filter { get; set; } + + [JsonIgnore] + internal string PageState { get; set; } + + string IFindOptions>.PageState { get => PageState; set => PageState = value; } + + Filter IFindOptions>.Filter { get => Filter; set => Filter = value; } + + [JsonInclude] + [JsonPropertyName("filter")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + internal Dictionary FilterMap => Filter == null ? null : Filter.Serialize(); + + CollectionSortBuilder IFindOptions>.Sort { get => Sort; set => Sort = value; } + + IProjectionBuilder IFindOptions>.Projection { get => Projection; set => Projection = value; } + + bool? IFindOptions>.IncludeSimilarity { get => IncludeSimilarity; set => IncludeSimilarity = value; } + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("sort")] + internal Dictionary SortMap => Sort == null ? null : Sort.Sorts.ToDictionary(x => x.Name, x => x.Value); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("projection")] + internal Dictionary ProjectionMap => Projection == null ? null : Projection.Projections.ToDictionary(x => x.FieldName, x => x.Value); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("options")] + internal Dictionary Options + { + get + { + var options = new Dictionary() + { + { "includeSimilarity", IncludeSimilarity }, + { "includeSortVector", IncludeSortVector }, + { "pageState", PageState }, + { "skip", Skip }, + { "limit", Limit }, + }; + options = options.Where(pair => pair.Value != null).ToDictionary(pair => pair.Key, pair => pair.Value); + if (options.Count == 0) + { + return null; + } + return options; + } + } + + internal CollectionFindManyOptions Clone() + { + return new CollectionFindManyOptions + { + Filter = Filter != null ? Filter.Clone() : null, + InitialPageState = InitialPageState, + Skip = Skip, + Limit = Limit, + IncludeSortVector = IncludeSortVector, + IncludeSimilarity = IncludeSimilarity, + Projection = Projection != null ? Projection.Clone() : null, + Sort = Sort != null ? Sort.Clone() : null + }; + } + + IFindManyOptions> IFindManyOptions>.Clone() + { + return Clone(); + } + + internal CollectionFindManyOptions WithFilterParam(CollectionFilter filter) + { + if (filter == null) + { + return this; + } + + if (Filter != null) + { + throw new ArgumentException("Cannot pass a filter both within FindOptions and as stand-alone argument"); + } + + var cloned = Clone(); + cloned.Filter = filter; + return cloned; + } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/CollectionFindOneOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/CollectionFindOneOptions.cs new file mode 100644 index 00000000..c3c1a812 --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/CollectionFindOneOptions.cs @@ -0,0 +1,108 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +/// +/// Options for finding a single document in a collection. +/// +/// The type of the document. +public class CollectionFindOneOptions +{ + /// The projection to apply to the results. + [JsonIgnore] + public IProjectionBuilder Projection { get; set; } + + /// + /// Whether to include a similarity score in the results or not (when performing a vector sort). + /// + [JsonIgnore] + public bool? IncludeSimilarity { get; set; } + + /// The sort to apply when running the query. + [JsonIgnore] + public CollectionSortBuilder Sort { get; set; } + + internal Filter Filter { get; set; } + + [JsonInclude] + [JsonPropertyName("filter")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + internal Dictionary FilterMap => Filter == null ? null : Filter.Serialize(); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("sort")] + internal Dictionary SortMap => Sort == null ? null : Sort.Sorts.ToDictionary(x => x.Name, x => x.Value); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("projection")] + internal Dictionary ProjectionMap => Projection == null ? null : Projection.Projections.ToDictionary(x => x.FieldName, x => x.Value); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("options")] + internal Dictionary Options + { + get + { + var options = new Dictionary() + { + { "includeSimilarity", IncludeSimilarity } + }; + options = options.Where(pair => pair.Value != null).ToDictionary(pair => pair.Key, pair => pair.Value); + if (options.Count == 0) + { + return null; + } + return options; + } + } + + internal CollectionFindOneOptions Clone() + { + return new CollectionFindOneOptions + { + Filter = Filter != null ? Filter.Clone() : null, + IncludeSimilarity = IncludeSimilarity, + Projection = Projection != null ? Projection.Clone() : null, + Sort = Sort != null ? Sort.Clone() : null + }; + } + + internal CollectionFindOneOptions WithFilterParam(CollectionFilter filter) + { + if (filter == null) + { + return this; + } + + if (Filter != null) + { + throw new ArgumentException("Cannot pass a filter both within FindOptions and as stand-alone argument"); + } + + var cloned = Clone(); + cloned.Filter = filter; + return cloned; + } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/DocumentFindManyOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/DocumentFindManyOptions.cs index 1f4191b7..b476fb45 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/DocumentFindManyOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/DocumentFindManyOptions.cs @@ -20,6 +20,12 @@ namespace DataStax.AstraDB.DataApi.Core.Query; internal class DocumentFindManyOptions : DocumentFindOptions, IFindManyOptions> { + /// + /// The initial page state used to resume pagination from a previous find-many operation. + /// + [JsonIgnore] + public string InitialPageState { get => PageState; set => PageState = value; } + [JsonIgnore] public int? Skip { get => _skip; set => _skip = value; } @@ -31,12 +37,12 @@ internal class DocumentFindManyOptions : DocumentFindOptions, IFindManyOpt bool? IFindManyOptions>.IncludeSortVector { get => IncludeSortVector; set => IncludeSortVector = value; } - IFindManyOptions> IFindManyOptions>.Clone() + internal virtual DocumentFindOptions Clone() { - var clone = new DocumentFindManyOptions + return new DocumentFindManyOptions { Filter = Filter != null ? Filter.Clone() : null, - PageState = PageState, + InitialPageState = InitialPageState, Skip = Skip, Limit = Limit, IncludeSortVector = IncludeSortVector, @@ -44,7 +50,10 @@ IFindManyOptions> IFindManyOptions> IFindManyOptions>.Clone() + { + return (DocumentFindManyOptions)Clone(); + } } diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs index 67a77885..3d24bf1d 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FilterBuilder.cs @@ -54,6 +54,11 @@ private TFilter Make(LogicalOperator op, object value) protected TFilter MakeOp(string fieldName, string op, object value) => Make(fieldName, Make(op, value)); + /// + /// Create an empty filter with no conditions (matching everything) + /// + public TFilter Empty() => Make(string.Empty, new Dictionary()); + /// /// Logical AND operator for combining multiple filters. /// diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FindEnumerator.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FindEnumerator.cs deleted file mode 100644 index 3507a876..00000000 --- a/src/DataStax.AstraDB.DataApi/Core/Query/FindEnumerator.cs +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright DataStax, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -using DataStax.AstraDB.DataApi.Core.Results; -using System; -using System.Collections; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; - -namespace DataStax.AstraDB.DataApi.Core.Query; - -/// -/// A Fluent API for finding and enumerating documents or rows. -/// -/// The type representing the document or row. -/// The type to deserialize the results to (i.e. if using ). -/// The type to use for sorting. -public class FindEnumerator : IAsyncEnumerable, IEnumerable - where T : class - where TResult : class - where TSort : SortBuilder -{ - private readonly IQueryRunner _queryRunner; - private readonly IFindManyOptions _findOptions; - private readonly CommandOptions _commandOptions; - private Cursor _cursor; - - internal FindEnumerator(IQueryRunner queryRunner, IFindManyOptions findOptions, CommandOptions commandOptions) - { - _queryRunner = queryRunner; - _findOptions = findOptions.Clone(); - _commandOptions = commandOptions; - } - - /// - /// Specify a Projection to apply to the results of the operation. - /// - /// The projection to apply. - /// The FindEnumerator instance to continue specifying the find options. - /// - /// - /// // Inclusive Projection, return only the nested Properties.PropertyOne field - /// var projectionBuilder = Builders<SimpleObject>.Projection; - /// var projection = projectionBuilder.Include(p => p.Properties.PropertyOne); - /// - /// - public FindEnumerator Project(IProjectionBuilder projection) - { - return UpdateOptions(options => options.Projection = projection); - } - - /// - /// Specify the maximum number of documents to return. - /// - /// The maximum number of documents to return. - /// The FindEnumerator instance to continue specifying the find options. - public FindEnumerator Limit(int limit) - { - return UpdateOptions(options => options.Limit = limit); - } - - /// - /// The number of documents to skip before starting to return documents. - /// Use in conjuction with to determine the order to apply before skipping. - /// - /// The number of documents to skip. - /// The FindEnumerator instance to continue specifying the find options. - public FindEnumerator Skip(int skip) - { - return UpdateOptions(options => options.Skip = skip); - } - - /// - /// Specify a Sort to use when running the find. - /// - /// The sort to apply. - /// The FindEnumerator instance to continue adding options. - /// - /// - /// // Sort by the nested Properties.PropertyOne field - /// var sortBuilder = Builders<SimpleObject>.CollectionSort; - /// var sort = sortBuilder.Ascending(p => p.Properties.PropertyOne); - /// - /// - public FindEnumerator Sort(TSort sortBuilder) - { - return UpdateOptions(options => options.Sort = sortBuilder); - } - - /// - /// Whether to include the similarity score in the result or not. - /// - /// Whether to include the similarity score in the result or not. - /// The FindEnumerator instance to continue specifying the find options. - /// - /// You can use the attribute to map the similarity score to the result class. - /// - /// public class SimpleObjectWithVectorizeResult : SimpleObjectWithVectorize - /// { - /// [DocumentMapping(DocumentMappingField.Similarity)] - /// public double? Similarity { get; set; } - /// } - /// - /// var FindEnumerator = collection.Find<SimpleObjectWithVectorizeResult>() - /// .Sort(Builders<SimpleObjectWithVectorize>.CollectionSort.Vectorize(dogQueryVectorString)) - /// .IncludeSimilarity(true); - /// var cursor = FindEnumerator.ToCursor(); - /// var list = cursor.ToList(); - /// var result = list.First(); - /// var similarity = result.Similarity; - /// - /// - /// - /// When searching on Tables, the field in the row class should be given the - /// attribute instead. - /// - public FindEnumerator IncludeSimilarity(bool includeSimilarity) - { - return UpdateOptions(options => options.IncludeSimilarity = includeSimilarity); - } - - /// - /// Whether to include the sort vector in the result or not. - /// - /// Whether to include the sort vector in the result or not. - /// The FindEnumerator instance to continue specifying the find options. - /// - /// - /// var finder = collection.Find<SimpleObjectWithVectorizeResult>() - /// .Sort(Builders<SimpleObjectWithVectorize>.CollectionSort.Vectorize(dogQueryVectorString)) - /// .IncludeSortVector(true); - /// //enumerate the results - /// var results = await finder.ToList(); - /// var sortVector = finder.GetSortVector(); - /// - /// - public FindEnumerator IncludeSortVector(bool includeSortVector) - { - return UpdateOptions(options => options.IncludeSortVector = includeSortVector); - } - - /// - /// Returns the sort vector created by the vectorize sort. - /// - /// The sort vector. - /// Thrown when the enumerator has not been started. - public float[] GetSortVector() - { - var cursor = ToCursor(); - if (!cursor.IsStarted) - { - throw new InvalidOperationException("Enumerator has not been started. Enumerate the results first, call ToList(), or manually manage paging by calling ToCursor() and using MoveNextAsync() to iterate over the results."); - } - return cursor.SortVector; - } - - /// - /// Returns a cursor to iterate over the results of the find operation page by page. - /// - /// NOTE: It is recommended to use the find results as an IEnumerable or IAsyncEnumerable instead of using the Cursor directly. - /// The only situation where you would need to use the Cursor is when you need to access the sort vectors (see ) - /// - /// A cursor to iterate over the results of the find operation page by page. - public Cursor ToCursor() - { - if (_cursor != null) - { - return _cursor; - } - _cursor = new Cursor((string pageState, CancellationToken cancellationToken, bool runSynchronously) => RunAsync(pageState, cancellationToken, runSynchronously)); - return _cursor; - } - - /// - /// Returns an async enumerator to iterate over the results of the find operation. - /// - /// An optional cancellation token to use for the operation. - /// An async enumerator - /// - /// Timeouts passed in the ( - /// and ) will be used for each batched request to the API. - /// If you need to enforce a timeout for the entire operation, you can pass a to this method. - /// BulkOperationCancellationToken settings are ignored for this operation. - /// - public IAsyncEnumerator GetAsyncEnumerator(CancellationToken cancellationToken = default) - { - var cursor = ToCursor(); - return cursor.ToAsyncEnumerator(cancellationToken); - } - - /// - /// Returns an enumerator to iterate over the results of the find operation. - /// - /// An enumerator - public IEnumerator GetEnumerator() - { - var cursor = ToCursor(); - return cursor.ToEnumerable().GetEnumerator(); - } - - IEnumerator IEnumerable.GetEnumerator() - { - return GetEnumerator(); - } - - private Task, FindStatusResult>> RunAsync(string pageState = null, CancellationToken cancellationToken = default, bool runSynchronously = false) - { - _findOptions.PageState = pageState; - if (cancellationToken != default && _commandOptions.BulkOperationCancellationToken == null) - { - _commandOptions.BulkOperationCancellationToken = cancellationToken; - } - return _queryRunner.RunFindManyAsync(_findOptions.Filter, _findOptions, _commandOptions, runSynchronously); - } - - private FindEnumerator UpdateOptions(Action> optionsUpdater) - { - optionsUpdater(_findOptions); - return new FindEnumerator(_queryRunner, _findOptions, _commandOptions); - } - -} \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/FindOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/FindOptions.cs index 408d8ce6..dba77ab0 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/FindOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/FindOptions.cs @@ -86,7 +86,7 @@ internal Dictionary Options { "includeSortVector", _includeSortVector }, { "pageState", PageState }, { "skip", _skip }, - { "limit", _limit } + { "limit", _limit }, }; options = options.Where(pair => pair.Value != null).ToDictionary(pair => pair.Key, pair => pair.Value); if (options.Count == 0) @@ -103,5 +103,4 @@ internal Dictionary Options IProjectionBuilder IFindOptions.Projection { get => Projection; set => Projection = value; } [JsonIgnore] bool? IFindOptions.IncludeSimilarity { get => IncludeSimilarity; set => IncludeSimilarity = value; } - -} \ No newline at end of file +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/IQueryRunner.cs b/src/DataStax.AstraDB.DataApi/Core/Query/IQueryRunner.cs deleted file mode 100644 index ab392f15..00000000 --- a/src/DataStax.AstraDB.DataApi/Core/Query/IQueryRunner.cs +++ /dev/null @@ -1,11 +0,0 @@ -using DataStax.AstraDB.DataApi.Core.Results; -using System.Threading.Tasks; - -namespace DataStax.AstraDB.DataApi.Core.Query; - -internal interface IQueryRunner where TSort : SortBuilder -{ - internal Task, FindStatusResult>> RunFindManyAsync( - Filter filter, IFindManyOptions findOptions, CommandOptions commandOptions, bool runSynchronously) - where TProjected : class; -} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/TableFindManyOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/TableFindManyOptions.cs index cca73b65..156d03e4 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/TableFindManyOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/TableFindManyOptions.cs @@ -14,6 +14,9 @@ * limitations under the License. */ +using System; +using System.Collections.Generic; +using System.Linq; using System.Text.Json.Serialization; namespace DataStax.AstraDB.DataApi.Core.Query; @@ -22,21 +25,42 @@ namespace DataStax.AstraDB.DataApi.Core.Query; /// A set of options to be used when finding rows in a table. /// /// -public class TableFindManyOptions : TableFindOptions, IFindManyOptions> +public class TableFindManyOptions : IFindManyOptions> { + /// The projection to apply to the results. + [JsonIgnore] + public IProjectionBuilder Projection { get; set; } + + /// + /// Whether to include a similarity score in the results or not (when performing a vector sort). + /// + [JsonIgnore] + public bool? IncludeSimilarity { get; set; } + + /// + /// The builder used to define the sort to apply when running the query. + /// + [JsonIgnore] + public TableSortBuilder Sort { get; set; } + + /// + /// The initial page state used to resume pagination from a previous find-many operation. + /// + [JsonIgnore] + public string InitialPageState { get => PageState; set => PageState = value; } /// /// The number of documents to skip before starting to return documents. /// Use in conjuction with to determine the order to apply before skipping. /// [JsonIgnore] - public int? Skip { get => _skip; set => _skip = value; } + public int? Skip { get; set; } /// /// The number of documents to return. /// [JsonIgnore] - public int? Limit { get => _limit; set => _limit = value; } + public int? Limit { get; set; } /// /// Whether to include the sort vector in the result or not @@ -62,22 +86,119 @@ public class TableFindManyOptions : TableFindOptions, IFindManyOptions /// [JsonIgnore] - internal bool? IncludeSortVector { get => _includeSortVector; set => _includeSortVector = value; } + internal bool? IncludeSortVector { get; set; } + bool? IFindManyOptions>.IncludeSortVector { get => IncludeSortVector; set => IncludeSortVector = value; } - IFindManyOptions> IFindManyOptions>.Clone() + internal Filter Filter { get; set; } + + [JsonIgnore] + internal string PageState { get; set; } + + string IFindOptions>.PageState { get => PageState; set => PageState = value; } + + Filter IFindOptions>.Filter { get => Filter; set => Filter = value; } + + [JsonInclude] + [JsonPropertyName("filter")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + internal Dictionary FilterMap => Filter == null ? null : Filter.Serialize(); + + TableSortBuilder IFindOptions>.Sort { get => Sort; set => Sort = value; } + + IProjectionBuilder IFindOptions>.Projection { get => Projection; set => Projection = value; } + + bool? IFindOptions>.IncludeSimilarity { get => IncludeSimilarity; set => IncludeSimilarity = value; } + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("sort")] + internal Dictionary SortMap => Sort == null ? null : Sort.Sorts.ToDictionary(x => x.Name, x => x.Value); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("projection")] + internal Dictionary ProjectionMap => Projection == null ? null : Projection.Projections.ToDictionary(x => x.FieldName, x => x.Value); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("options")] + internal Dictionary Options + { + get + { + var options = new Dictionary() + { + { "includeSimilarity", IncludeSimilarity }, + { "includeSortVector", IncludeSortVector }, + { "pageState", PageState }, + { "skip", Skip }, + { "limit", Limit }, + }; + options = options.Where(pair => pair.Value != null).ToDictionary(pair => pair.Key, pair => pair.Value); + if (options.Count == 0) + { + return null; + } + return options; + } + } + + internal TableFindManyOptions Clone() { - var clone = new TableFindManyOptions + return new TableFindManyOptions { Filter = Filter != null ? Filter.Clone() : null, - PageState = PageState, + InitialPageState = InitialPageState, Skip = Skip, Limit = Limit, + IncludeSortVector = IncludeSortVector, IncludeSimilarity = IncludeSimilarity, Projection = Projection != null ? Projection.Clone() : null, Sort = Sort != null ? Sort.Clone() : null }; - return clone; } -} + IFindManyOptions> IFindManyOptions>.Clone() + + { + + return Clone(); + + } + + + + internal TableFindManyOptions WithFilterParam(TableFilter filter) + + { + + if (filter == null) + + { + + return this; + + } + + + + if (Filter != null) + + { + + throw new ArgumentException("Cannot pass a filter both within FindOptions and as stand-alone argument"); + + } + + + + var cloned = Clone(); + + cloned.Filter = filter; + + return cloned; + + } + + } \ No newline at end of file diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/TableFindOneOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/TableFindOneOptions.cs new file mode 100644 index 00000000..43a904fb --- /dev/null +++ b/src/DataStax.AstraDB.DataApi/Core/Query/TableFindOneOptions.cs @@ -0,0 +1,110 @@ +/* + * Copyright DataStax, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json.Serialization; + +namespace DataStax.AstraDB.DataApi.Core.Query; + +/// +/// A set of options to be used when finding a row in a table. +/// +/// The type of the row in the table. +public class TableFindOneOptions +{ + /// The projection to apply to the results. + [JsonIgnore] + public IProjectionBuilder Projection { get; set; } + + /// + /// Whether to include a similarity score in the results or not (when performing a vector sort). + /// + [JsonIgnore] + public bool? IncludeSimilarity { get; set; } + + /// + /// The builder used to define the sort to apply when running the query. + /// + [JsonIgnore] + public TableSortBuilder Sort { get; set; } + + internal Filter Filter { get; set; } + + [JsonInclude] + [JsonPropertyName("filter")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + internal Dictionary FilterMap => Filter == null ? null : Filter.Serialize(); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("sort")] + internal Dictionary SortMap => Sort == null ? null : Sort.Sorts.ToDictionary(x => x.Name, x => x.Value); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("projection")] + internal Dictionary ProjectionMap => Projection == null ? null : Projection.Projections.ToDictionary(x => x.FieldName, x => x.Value); + + [JsonInclude] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + [JsonPropertyName("options")] + internal Dictionary Options + { + get + { + var options = new Dictionary() + { + { "includeSimilarity", IncludeSimilarity } + }; + options = options.Where(pair => pair.Value != null).ToDictionary(pair => pair.Key, pair => pair.Value); + if (options.Count == 0) + { + return null; + } + return options; + } + } + + internal TableFindOneOptions Clone() + { + return new TableFindOneOptions + { + Filter = Filter != null ? Filter.Clone() : null, + IncludeSimilarity = IncludeSimilarity, + Projection = Projection != null ? Projection.Clone() : null, + Sort = Sort != null ? Sort.Clone() : null + }; + } + + internal TableFindOneOptions WithFilterParam(TableFilter filter) + { + if (filter == null) + { + return this; + } + + if (Filter != null) + { + throw new ArgumentException("Cannot pass a filter both within FindOptions and as stand-alone argument"); + } + + var cloned = Clone(); + cloned.Filter = filter; + return cloned; + } +} diff --git a/src/DataStax.AstraDB.DataApi/Core/Query/TableFindOptions.cs b/src/DataStax.AstraDB.DataApi/Core/Query/TableFindOptions.cs index 0615683d..adec670c 100644 --- a/src/DataStax.AstraDB.DataApi/Core/Query/TableFindOptions.cs +++ b/src/DataStax.AstraDB.DataApi/Core/Query/TableFindOptions.cs @@ -30,7 +30,7 @@ public class TableFindOptions : FindOptions> [JsonIgnore] public override TableSortBuilder Sort { get; set; } - internal TableFindOptions Clone() + internal virtual TableFindOptions Clone() { return new TableFindOptions { diff --git a/src/DataStax.AstraDB.DataApi/DataStax.AstraDB.DataApi.csproj b/src/DataStax.AstraDB.DataApi/DataStax.AstraDB.DataApi.csproj index 0869731e..df4a679d 100644 --- a/src/DataStax.AstraDB.DataApi/DataStax.AstraDB.DataApi.csproj +++ b/src/DataStax.AstraDB.DataApi/DataStax.AstraDB.DataApi.csproj @@ -31,6 +31,7 @@ + diff --git a/src/DataStax.AstraDB.DataApi/Tables/Table.cs b/src/DataStax.AstraDB.DataApi/Tables/Table.cs index fd54e030..2aa01ce0 100644 --- a/src/DataStax.AstraDB.DataApi/Tables/Table.cs +++ b/src/DataStax.AstraDB.DataApi/Tables/Table.cs @@ -16,6 +16,7 @@ using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.Core.Commands; +using DataStax.AstraDB.DataApi.Core.Enumeration; using DataStax.AstraDB.DataApi.Core.Query; using DataStax.AstraDB.DataApi.Core.Results; using DataStax.AstraDB.DataApi.SerDes; @@ -37,7 +38,7 @@ namespace DataStax.AstraDB.DataApi.Tables; /// This is the main entry point for interacting with a table in the Astra DB Data API. /// /// The type to use for rows in the table (when not specified, defaults to -public class Table : IQueryRunner> where T : class +public class Table where T : class { private readonly string _tableName; private readonly Database _database; @@ -846,18 +847,17 @@ private async Task InsertOneAsync(T row, CommandOptions co /// /// Find rows in the table. /// - /// The Find() methods return a object that can be used to further structure the query + /// The Find() methods return a object that can be used to further structure the query /// by adding Sort, Projection, Skip, Limit, etc. to affect the final results. /// - /// The object can be directly enumerated both synchronously and asynchronously. - /// Secondarily, the results can be paged through manually by using the results of . + /// The object can be directly enumerated both synchronously and asynchronously. /// /// /// /// Synchronous Enumeration: /// - /// var FindEnumerator = table.Find(); - /// foreach (var row in FindEnumerator) + /// var cursor = table.Find(); + /// foreach (var row in cursor) /// { /// // Process row /// } @@ -879,7 +879,7 @@ private async Task InsertOneAsync(T row, CommandOptions co /// however BulkOperationCancellationToken settings are ignored due to the nature of Enumeration. /// If you need to enforce a timeout for the entire operation, you can pass a to GetAsyncEnumerator. /// - public FindEnumerator> Find() + public TableFindCursor Find() { return Find(null, null); } @@ -898,52 +898,79 @@ public FindEnumerator> Find() /// } /// /// - public FindEnumerator> Find(TableFilter filter) + public TableFindCursor Find(TableFilter filter) { return Find(filter, null); } + /// + /// + public TableFindCursor Find(TableFindManyOptions findOptions) + { + return Find(null, findOptions); + } + /// /// - /// - public FindEnumerator> Find(TableFilter filter, CommandOptions commandOptions) + /// + public TableFindCursor Find(TableFilter filter, TableFindManyOptions findOptions) { - return Find(filter, commandOptions); + findOptions ??= new TableFindManyOptions(); + return new(findOptions.WithFilterParam(filter), null, RunFindManyAsync); } - /// - /// - /// + /// /// - /// This overload of Find() allows you to specify a different result class type - /// which the resultant rows will be deserialized into. This is generally used along with .Project() to limit the fields returned + /// The Find alternatives that accept a TResult type parameter allow for deserializing the row as a different type + /// (most commonly used when using projection to return a subset of fields) /// - public FindEnumerator> Find(TableFilter filter, CommandOptions commandOptions) where TResult : class + public TableFindCursor Find() where TResult : class { - var findOptions = new TableFindManyOptions - { - Filter = filter - }; - return new FindEnumerator>(this, findOptions, commandOptions); + return Find(null, null); } - internal async Task, TableFindStatusResult>> RunFindManyAsync( - Filter filter, - IFindManyOptions> findOptions, - CommandOptions commandOptions, - bool runSynchronously - ) - where TResult : class + /// + /// + /// The Find alternatives that accept a TResult type parameter allow for deserializing the row as a different type + /// (most commonly used when using projection to return a subset of fields) + /// + public TableFindCursor Find(TableFilter filter) where TResult : class { - findOptions.Filter = filter; - commandOptions = SetRowSerializationOptions(commandOptions, false); - var command = CreateCommand("find").WithPayload(findOptions).AddCommandOptions(commandOptions); + return Find(filter, null); + } + + /// + /// + /// The Find alternatives that accept a TResult type parameter allow for deserializing the row as a different type + /// (most commonly used when using projection to return a subset of fields) + /// + public TableFindCursor Find(TableFindManyOptions findOptions) where TResult : class + { + return Find(null, findOptions); + } + + /// + /// + /// + public TableFindCursor Find(TableFilter filter, TableFindManyOptions findOptions) where TResult : class + { + findOptions ??= new TableFindManyOptions(); + return new(findOptions.WithFilterParam(filter), null, RunFindManyAsync); + } + + internal async Task> RunFindManyAsync(TableFindCursor cursor, string nextPageState, bool runSynchronously) where TResult : class + { + var options = cursor.FindOptions.Clone(); + options.PageState = nextPageState; + + var commandOptions = SetRowSerializationOptions(cursor.CommandOptions, false); + var command = CreateCommand("find").WithPayload(options).AddCommandOptions(commandOptions); var response = await command.RunAsyncReturnData, TableFindStatusResult>(runSynchronously).ConfigureAwait(false); + if (typeof(Row).IsAssignableFrom(typeof(TResult))) { var columnsInResult = response.Status.ProjectionSchema; - if (response != null && response.Data != null && response.Data.Items != null) + if (response.Data is { Items: not null }) { foreach (var row in response.Data.Items) { @@ -951,7 +978,12 @@ bool runSynchronously } } } - return response; + + return new FindPage( + response.Data.NextPageState, + response.Data.Items, + response.Status?.SortVector + ); } /// @@ -975,23 +1007,23 @@ public T FindOne(TableFilter filter, CommandOptions commandOptions) return FindOne(filter, null, commandOptions); } - /// - /// Synchronous version of - public T FindOne(TableFindOptions findOptions) + /// + /// Synchronous version of + public T FindOne(TableFindOneOptions findOptions) { return FindOne(null, findOptions, null); } - /// - /// Synchronous version of - public T FindOne(TableFilter filter, TableFindOptions findOptions) + /// + /// Synchronous version of + public T FindOne(TableFilter filter, TableFindOneOptions findOptions) { return FindOne(filter, findOptions, null); } - /// - /// Synchronous version of - public T FindOne(TableFilter filter, TableFindOptions findOptions, CommandOptions commandOptions) + /// + /// Synchronous version of + public T FindOne(TableFilter filter, TableFindOneOptions findOptions, CommandOptions commandOptions) { return FindOne(filter, findOptions, commandOptions); } @@ -1017,16 +1049,16 @@ public TResult FindOne(TableFilter filter, CommandOptions commandOpt return FindOne(filter, null, commandOptions); } - /// - /// Synchronous version of - public TResult FindOne(TableFilter filter, TableFindOptions findOptions) where TResult : class + /// + /// Synchronous version of + public TResult FindOne(TableFilter filter, TableFindOneOptions findOptions) where TResult : class { return FindOne(filter, findOptions, null); } - /// - /// Synchronous version of - public TResult FindOne(TableFilter filter, TableFindOptions findOptions, CommandOptions commandOptions) where TResult : class + /// + /// Synchronous version of + public TResult FindOne(TableFilter filter, TableFindOneOptions findOptions, CommandOptions commandOptions) where TResult : class { return FindOneAsync(filter, findOptions, commandOptions, true).ResultSync(); } @@ -1063,7 +1095,7 @@ public Task FindOneAsync(TableFilter filter, CommandOptions commandOptions /// /// Specify Sort options for the find operation. /// - public Task FindOneAsync(TableFindOptions findOptions) + public Task FindOneAsync(TableFindOneOptions findOptions) { return FindOneAsync(null, findOptions, null); } @@ -1071,16 +1103,16 @@ public Task FindOneAsync(TableFindOptions findOptions) /// /// /// Specify Sort options for the find operation. - public Task FindOneAsync(TableFilter filter, TableFindOptions findOptions) + public Task FindOneAsync(TableFilter filter, TableFindOneOptions findOptions) { return FindOneAsync(filter, findOptions, null); } - /// + /// /// /// /// - public Task FindOneAsync(TableFilter filter, TableFindOptions findOptions, CommandOptions commandOptions) + public Task FindOneAsync(TableFilter filter, TableFindOneOptions findOptions, CommandOptions commandOptions) { return FindOneAsync(filter, findOptions, commandOptions); } @@ -1115,24 +1147,23 @@ public Task FindOneAsync(TableFilter filter, CommandOptions /// /// /// Specify Sort options for the find operation. - public Task FindOneAsync(TableFilter filter, TableFindOptions findOptions) where TResult : class + public Task FindOneAsync(TableFilter filter, TableFindOneOptions findOptions) where TResult : class { return FindOneAsync(filter, findOptions, null); } - /// + /// /// /// /// - public Task FindOneAsync(TableFilter filter, TableFindOptions findOptions, CommandOptions commandOptions) where TResult : class + public Task FindOneAsync(TableFilter filter, TableFindOneOptions findOptions, CommandOptions commandOptions) where TResult : class { return FindOneAsync(filter, findOptions, commandOptions, false); } - internal async Task FindOneAsync(TableFilter filter, TableFindOptions findOptions, CommandOptions commandOptions, bool runSynchronously) - where TResult : class + internal async Task FindOneAsync(TableFilter filter, TableFindOneOptions findOptions, CommandOptions commandOptions, bool runSynchronously) where TResult : class { - findOptions = findOptions != null ? findOptions.Clone() : new TableFindOptions(); + findOptions = findOptions != null ? findOptions.Clone() : new TableFindOneOptions(); if (filter != null) { if (findOptions.Filter == null) @@ -1148,7 +1179,7 @@ internal async Task FindOneAsync(TableFilter filter, TableF var response = await command.RunAsyncReturnData, TableFindStatusResult>(runSynchronously).ConfigureAwait(false); if (typeof(Row).IsAssignableFrom(typeof(TResult))) { - if (response != null && response.Data != null && response.Data.Document != null) + if (response is { Data.Document: not null }) { ProcessUntypedRow(response.Data.Document as Row, response.Status.ProjectionSchema); } @@ -1656,16 +1687,4 @@ internal Command CreateCommand(string name) var optionsTree = GetOptionsTree().ToArray(); return new Command(name, _database.Client, optionsTree, new DatabaseCommandUrlBuilder(_database, _tableName)); } - - async Task, FindStatusResult>> IQueryRunner>.RunFindManyAsync(Filter filter, IFindManyOptions> findOptions, CommandOptions commandOptions, bool runSynchronously) - { - var result = await RunFindManyAsync(filter, findOptions, commandOptions, runSynchronously).ConfigureAwait(false); - return new ApiResponseWithData, FindStatusResult> - { - Data = result.Data, - Status = result.Status, - Errors = result.Errors, - Warnings = result.Warnings - }; - } } diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/CollectionCursorFixture.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/CollectionCursorFixture.cs new file mode 100644 index 00000000..4f7ab639 --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Fixtures/CollectionCursorFixture.cs @@ -0,0 +1,101 @@ +using DataStax.AstraDB.DataApi.Collections; +using DataStax.AstraDB.DataApi.Core; +using DataStax.AstraDB.DataApi.Core.Query; +using Xunit; + +namespace DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; + +[CollectionDefinition("CollectionCursor")] +public class CollectionCursorCollection : ICollectionFixture, ICollectionFixture +{ +} + +public class CollectionCursorFixture : BaseFixture, IAsyncLifetime +{ + public CollectionCursorFixture(AssemblyFixture assemblyFixture) : base(assemblyFixture, "collectionCursor") + { + } + + public Collection FilledCollection { get; private set; } + public int FilledCollectionCount { get; private set; } + public Collection FilledPaginationCollection { get; private set; } + public int FilledPaginationCollectionCount { get; private set; } + + public async ValueTask InitializeAsync() + { + await CreateFilledCollection(); + await CreateFilledPaginationCollection(); + } + + public async ValueTask DisposeAsync() + { + await Database.DropCollectionAsync(_collectionName); + await Database.DropCollectionAsync(_paginationCollectionName); + } + + private const string _collectionName = "collectionTestCursorFilled"; + private const string _paginationCollectionName = "collectionPaginationTestCursorFilled"; + private const int NUM_DOCS = 25; // keep this between 21 and 39 (must be 1 full + 1 partial page in size) + private const int NUM_DOCS_PAGINATION = 90; // keep this above 2 * (2 * 20) and below 2 * (3 * 20) + + private async Task CreateFilledCollection() + { + + // TODO replace with from-object creation when PR 136 gets merged and makes it into this branch + var collectionDefinition = new CollectionDefinition + { + Vector = new VectorOptions { Dimension = 2 } + }; + var collection = await Database.CreateCollectionAsync(_collectionName, collectionDefinition); + + await collection.DeleteManyAsync(Builders.CollectionFilter.Empty()); + + var testDocuments = new List(); + for (int i = 0; i < NUM_DOCS; i++) + { + testDocuments.Add(new CursorTestDocument + { + Id = $"doc_{i + 1}", + PText = "pA", + PInt = i, + Vector = new float[] { i, 1.0f } + }); + } + + await collection.InsertManyAsync(testDocuments); + + FilledCollection = collection; + FilledCollectionCount = NUM_DOCS; + } + + private async Task CreateFilledPaginationCollection() + { + + // TODO replace with from-object creation when PR 136 gets merged and makes it into this branch + var collectionDefinition = new CollectionDefinition + { + Vector = new VectorOptions { Dimension = 2 } + }; + var collection = await Database.CreateCollectionAsync(_paginationCollectionName, collectionDefinition); + + await collection.DeleteManyAsync(Builders.CollectionFilter.Empty()); + + var testDocuments = new List(); + for (int i = 0; i < NUM_DOCS_PAGINATION; i++) + { + testDocuments.Add(new CursorPaginationTestDocument + { + id = i, + text = $"doc number {i}", + even = i % 2 == 0, + vector = new float[] { i, 1.0f } + }); + } + + await collection.InsertManyAsync(testDocuments); + + FilledPaginationCollection = collection; + FilledPaginationCollectionCount = NUM_DOCS_PAGINATION; + } + +} diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/TestObjects.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/TestObjects.cs index 5040c672..250fbe54 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/TestObjects.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/TestObjects.cs @@ -2,7 +2,6 @@ using DataStax.AstraDB.DataApi.Core; using DataStax.AstraDB.DataApi.SerDes; using DataStax.AstraDB.DataApi.Tables; -using DataStax.AstraDB.DataApi.Collections; using MongoDB.Bson; using System.Text.Json.Serialization; @@ -711,6 +710,15 @@ public class TripleMapObject public Dictionary map_v { get; set; } } +public class CursorTestDocument { + [DocumentId] + public string Id { get; set; } + public string PText { get; set; } + public int PInt { get; set; } + [DocumentMapping(DocumentMappingField.Vector)] + public float[] Vector { get; set; } +} + [CollectionName("testColl_vecEncoding_Typed")] [CollectionVector(3)] public class VectorObjectAsLst @@ -767,3 +775,12 @@ public class SimpleTwoColumnRow public int Id { get; set; } public string Name { get; set; } } + +public class CursorPaginationTestDocument { + [DocumentId] + public int id { get; set; } + public string text { get; set; } + public bool even { get; set; } + [DocumentMapping(DocumentMappingField.Vector)] + public float[] vector { get; set; } +} diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalCollectionTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalCollectionTests.cs index 0faf07e1..bc4096df 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalCollectionTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalCollectionTests.cs @@ -451,10 +451,10 @@ await createdCollection.InsertOneAsync(new Document }); // Reads: - var findOptionsLst = new DocumentFindOptions() + var findOptionsLst = new CollectionFindOneOptions() { Projection = Builders.Projection.Include("$vector") }; - var findOptionsBin = new DocumentFindOptions() + var findOptionsBin = new CollectionFindOneOptions() { Projection = Builders.Projection.Include("$vector") }; var lstRead = await createdCollection.FindOneAsync( @@ -520,11 +520,11 @@ await collection.InsertManyAsync(new List { // findOne through FINDOPTIONS ONLY: // - var findOpt_id1 = new DocumentFindOptions() + var findOpt_id1 = new CollectionFindOneOptions() { Filter = Builders.CollectionFilter.Eq(d => d.Id, 1) }; - var findOpt_id2 = new DocumentFindOptions() + var findOpt_id2 = new CollectionFindOneOptions() { Filter = Builders.CollectionFilter.Eq(d => d.Id, 2) }; @@ -536,11 +536,11 @@ await collection.InsertManyAsync(new List { Assert.Equal("two", find_o_id2.Name); // findOne through BOTH FILTER AND FINDOPTIONS (should throw): - var findOpt_id991 = new DocumentFindOptions() + var findOpt_id991 = new CollectionFindOneOptions() { Filter = Builders.CollectionFilter.Eq(d => d.Id, 991) }; - var findOpt_id992 = new DocumentFindOptions() + var findOpt_id992 = new CollectionFindOneOptions() { Filter = Builders.CollectionFilter.Eq(d => d.Id, 992) }; diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalTableTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalTableTests.cs index 67ed6d1b..649c9bcb 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalTableTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/AdditionalTableTests.cs @@ -50,7 +50,7 @@ public async Task Test_Arrays() await table.CreateIndexAsync("StringArray_idx", (b) => b.StringArray); var insertResult = await table.InsertManyAsync(items); Assert.Equal(items.Count, insertResult.InsertedIdTuples.Count); - var findOptions = new TableFindOptions() + var findOptions = new TableFindOneOptions() { Filter = Builders.TableFilter.In(x => x.StringArray, new string[] { "five" }), }; @@ -235,7 +235,7 @@ public async Task TestDictionaryTypes() Assert.Equal(5, result.InsertedCount); // Query the data - var findOptions = new TableFindOptions() + var findOptions = new TableFindOneOptions() { Filter = Builders.TableFilter.ValuesIn((t) => t.IntKey, new string[] { "IntValue 1A" }), }; @@ -245,7 +245,7 @@ public async Task TestDictionaryTypes() Assert.Equal(1, findResult.Id); Assert.Equal("DecimalValue 1A", findResult.DecimalKey[2.1m]); - findOptions = new TableFindOptions() + findOptions = new TableFindOneOptions() { Filter = Builders.TableFilter.ValuesIn((t) => t.IntKey, new string[] { "IntValue 1ABC" }), }; @@ -552,7 +552,7 @@ public async Task FindOne_Vector() var sortBuilder = Builders.TableSort; var sort = sortBuilder.Vector(b => b.VectorEmbeddings, dogQueryVector); var result = await table.FindOneAsync(null, - new TableFindOptions() { Sort = sort, IncludeSimilarity = true }); + new TableFindOneOptions() { Sort = sort, IncludeSimilarity = true }); Assert.NotNull(result.Similarity); Assert.True(result.Similarity > 0); @@ -597,7 +597,7 @@ public async Task FindOne_Vector_UntypedFind() var tableUntyped = fixture.Database.GetTable(tableName); - var findOptions = new TableFindOptions() + var findOptions = new TableFindOneOptions() { Sort = Builders.TableSort.Vector("VectorEmbeddings", dogQueryVector), IncludeSimilarity = true, @@ -968,7 +968,7 @@ await table.InsertOneAsync(new TripleMapObject() }, }); - var projectingOptions = new TableFindOptions() + var projectingOptions = new TableFindOneOptions() { Projection = Builders.Projection.Include(r => r.id) }; @@ -1164,11 +1164,11 @@ await table.InsertManyAsync(new List { // findOne through FINDOPTIONS ONLY: // - var findOpt_id1 = new TableFindOptions() + var findOpt_id1 = new TableFindOneOptions() { Filter = Builders.TableFilter.Eq(d => d.Id, 1) }; - var findOpt_id2 = new TableFindOptions() + var findOpt_id2 = new TableFindOneOptions() { Filter = Builders.TableFilter.Eq(d => d.Id, 2) }; @@ -1180,11 +1180,11 @@ await table.InsertManyAsync(new List { Assert.Equal("two", find_o_id2.Name); // findOne through BOTH FILTER AND FINDOPTIONS (should throw): - var findOpt_id991 = new TableFindOptions() + var findOpt_id991 = new TableFindOneOptions() { Filter = Builders.TableFilter.Eq(d => d.Id, 991) }; - var findOpt_id992 = new TableFindOptions() + var findOpt_id992 = new TableFindOneOptions() { Filter = Builders.TableFilter.Eq(d => d.Id, 992) }; diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionCursorTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionCursorTests.cs new file mode 100644 index 00000000..ffc18dc1 --- /dev/null +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/CollectionCursorTests.cs @@ -0,0 +1,394 @@ +using DataStax.AstraDB.DataApi.Collections; +using DataStax.AstraDB.DataApi.Core; +using DataStax.AstraDB.DataApi.Core.Commands; +using DataStax.AstraDB.DataApi.Core.Enumeration; +using DataStax.AstraDB.DataApi.Core.Results; +using DataStax.AstraDB.DataApi.IntegrationTests.Fixtures; +using DataStax.AstraDB.DataApi.SerDes; +using System.Linq; +using MongoDB.Bson; +using System.Text; +using UUIDNext; +using Xunit; + +namespace DataStax.AstraDB.DataApi.IntegrationTests; + +[Collection("CollectionCursor")] +public class CollectionCursorTests +{ + + private readonly CollectionCursorFixture _fixture; + Collection filledCollection; + + public CollectionCursorTests(AssemblyFixture assemblyFixture, CollectionCursorFixture fixture) + { + _fixture = fixture; + } + + [Fact] + public async Task Test_CollectionCursor_IdleProperties() + { + var filledCollection = _fixture.FilledCollection; + + // pristine cursor properties + var cur = filledCollection.Find(); + Assert.Equal(CursorState.Idle, cur.State); + Assert.Equal(0, cur.Consumed); + Assert.Empty(cur.ConsumeBuffer(3)); + Assert.Equal(0, cur.Buffered()); + Assert.Equal(0, cur.Consumed); + + var toClose = cur.Clone(); + toClose.Dispose(); + toClose.Dispose(); + Assert.Equal(CursorState.Closed, toClose.State); + Assert.Equal(CursorState.Idle, cur.State); + + await Assert.ThrowsAsync( async () => + { + await foreach (var item in toClose) { /* moot */ } + }); + Assert.Throws(() => + { + foreach (var item in toClose) { /* moot */ } + }); + await Assert.ThrowsAsync( async () => + await toClose.ToListAsync() + ); + Assert.Throws( () => + toClose.ToList() + ); + // TODO: c# does not error on "MoveNextAsync" for closed cursor (python does). Do we care? + + // rewinding + cur.Rewind(); + Assert.Equal(CursorState.Idle, cur.State); + Assert.Equal(0, cur.Consumed); + Assert.Equal(0, cur.Buffered()); + + // various fluent-api methods + cur.Filter(Builders.CollectionFilter.Eq(d => d.PInt, 789)); + cur.Project(Builders.Projection.Include(d => d.PInt)); + cur.Sort(Builders.CollectionSort.Ascending(d => d.PInt)); + cur.Limit(1); + cur.IncludeSimilarity(false); + cur.IncludeSortVector(true); + cur.Skip(1); + } + + [Fact] + public async Task Test_CollectionCursor_ClosedProperties() + { + var filledCollection = _fixture.FilledCollection; + + var cur0 = filledCollection.Find(); + cur0.Dispose(); + cur0.Rewind(); + Assert.Equal(CursorState.Idle, cur0.State); + + var cur1 = filledCollection.Find(); + Assert.Equal(0, cur1.Consumed); + await foreach (var item in cur1) { /* moot */ } + Assert.Equal(CursorState.Closed, cur1.State); + Assert.Empty(cur1.ConsumeBuffer(2)); + Assert.Equal(_fixture.FilledCollectionCount, cur1.Consumed); + Assert.Equal(0, cur1.Buffered()); + var cloned = cur1.Clone(); + Assert.Equal(0, cloned.Consumed); + Assert.Equal(0, cloned.Buffered()); + Assert.Equal(CursorState.Idle, cloned.State); + + // Closed cursors can't receive fluent-API edits: + + // TODO does not throw (and should) + // Assert.Throws(() => { cur1.Filter( + // Builders.CollectionFilter.Empty() + // ); }); + // TODO does not throw (and should) + // Assert.Throws(() => { cur1.Project( + // Builders.Projection.Include(d => d.PText) + // ); }); + // TODO does not throw (and should) + // Assert.Throws(() => { cur1.Sort( + // Builders.Sort.Ascending(d => d.PText) + // ); }); + // TODO does not throw (and should) + // Assert.Throws(() => { cur1.Limit(1); }); + // TODO does not throw (and should) + // Assert.Throws(() => { cur1.Skip(1); }); + // TODO does not throw (and should) + // Assert.Throws(() => { cur1.IncludeSimilarity(true); }); + // TODO does not throw (and should) + // Assert.Throws(() => { cur1.IncludeSortVector(true); }); + // TODO does not throw (and should) + // Assert.Throws(() => { cur1.InitialPageState("blaaaa"); }); + + // (note the full prefix required otherwise ambiguous a/sync unresolved) + await Assert.ThrowsAsync(async () => { + await System.Linq.AsyncEnumerable.Select(cur1, doc => doc.PText).ToListAsync(); + }); + + } + + [Fact] + public async Task Test_CollectionCursor_StartedProperties() + { + var filledCollection = _fixture.FilledCollection; + + var cur = filledCollection.Find(); + await cur.MoveNextAsync(); + // now: 19 in buffer, one consumed: + Assert.Equal(1, cur.Consumed); + Assert.Equal(19, cur.Buffered()); + Assert.Equal(3, cur.ConsumeBuffer(3).Count); + // 16 in buffer, 4 consumed: + Assert.Equal(4, cur.Consumed); + Assert.Equal(16, cur.Buffered()); + // from time to time the buffer is empty: + for(int i=0; i<16; i++){ + await cur.MoveNextAsync(); + } + Assert.Equal(0, cur.Buffered()); + Assert.Equal(0, cur.ConsumeBuffer(3).Count); + Assert.Equal(20, cur.Consumed); + Assert.Equal(0, cur.Buffered()); + + // Started cursors can't receive fluent-API edits: + + // TODO does not throw (and should) + // Assert.Throws(() => { cur.Filter( + // Builders.CollectionFilter.Empty() + // ); }); + // TODO does not throw (and should) + // Assert.Throws(() => { cur.Project( + // Builders.Projection.Include(d => d.PText) + // ); }); + // TODO does not throw (and should) + // Assert.Throws(() => { cur.Sort( + // Builders.Sort.Ascending(d => d.PText) + // ); }); + // TODO does not throw (and should) + // Assert.Throws(() => { cur.Limit(1); }); + // TODO does not throw (and should) + // Assert.Throws(() => { cur.Skip(1); }); + // TODO does not throw (and should) + // Assert.Throws(() => { cur.IncludeSimilarity(true); }); + // TODO does not throw (and should) + // Assert.Throws(() => { cur.IncludeSortVector(true); }); + // TODO does not throw (and should) + // Assert.Throws(() => { cur.InitialPageState("blaaaa"); }); + + // TODO: this one *does not throw* (other clients don't admit setting mapping *after* started) + // But in this case "we're in LINQ's hands" and can't really do much about it. Are we ok? + // + // (note the full prefix required otherwise ambiguous a/sync unresolved) + //await Assert.ThrowsAsync(async () => { + await System.Linq.AsyncEnumerable.Select(cur, doc => doc.PText).ToListAsync(); + // }); + + } + + [Fact] + public async Task Test_CollectionCursor_HasNext() + { + var filledCollection = _fixture.FilledCollection; + var cur = filledCollection.Find(); + + Assert.Equal(CursorState.Idle, cur.State); + Assert.Equal(0, cur.Consumed); + Assert.True(await cur.HasNextAsync()); + Assert.Equal(CursorState.Started, cur.State); + Assert.Equal(0, cur.Consumed); + await cur.MoveNextAsync(); + Assert.Equal(CursorState.Started, cur.State); + await foreach (var item in cur) { /* moot */ }; + Assert.Equal(CursorState.Closed, cur.State); + Assert.Equal(_fixture.FilledCollectionCount, cur.Consumed); + + var curMf = filledCollection.Find(); + await curMf.MoveNextAsync(); + await curMf.MoveNextAsync(); + Assert.Equal(2, curMf.Consumed); + Assert.Equal(CursorState.Started, curMf.State); + Assert.True(await curMf.HasNextAsync()); + Assert.Equal(2, curMf.Consumed); + Assert.Equal(CursorState.Started, curMf.State); + for(int i=0; i<18; i++){ + await curMf.MoveNextAsync(); + } + Assert.True(await curMf.HasNextAsync()); + Assert.Equal(20, curMf.Consumed); + Assert.Equal(CursorState.Started, curMf.State); + Assert.Equal(_fixture.FilledCollectionCount - 20, curMf.Buffered()); + + var cur0 = filledCollection.Find(); + cur0.Dispose(); + Assert.False(await cur0.HasNextAsync()); + } + + [Fact] + public async Task Test_CollectionCursor_ZeroMatches() + { + var filledCollection = _fixture.FilledCollection; + var cur = filledCollection.Find().Filter( + Builders.CollectionFilter.Eq(d => d.PText, "ZZ")); + + Assert.False(await cur.HasNextAsync()); + await Assert.ThrowsAsync( async () => + { + await cur.ToListAsync(); + }); + } + + [Fact] + public async Task Test_CollectionCursor_EarlyClosing() + { + var filledCollection = _fixture.FilledCollection; + var cur = filledCollection.Find(); + for (int i = 0; i < 12; i++){ + await cur.MoveNextAsync(); + } + cur.Dispose(); + Assert.Equal(CursorState.Closed, cur.State); + Assert.Equal(0, cur.Buffered()); + Assert.Equal(12, cur.Consumed); + + cur.Rewind(); + Assert.Equal(_fixture.FilledCollectionCount, cur.ToList().Count); + } + + [Fact] + public async Task Test_CollectionCursor_CollectiveMethods() + { + var filledCollection = _fixture.FilledCollection; + var baseRows = await filledCollection.Find().ToListAsync(); + + // full ToList (list equalities projected on scalar lists for conciseness) + var tlCur = filledCollection.Find(); + Assert.Equal(baseRows.Select(d => d.Id), (await tlCur.ToListAsync()).Select(d => d.Id)); + Assert.Equal(CursorState.Closed, tlCur.State); + + // partially-consumed ToList + var ptlCur = filledCollection.Find(); + for (int i = 0; i < 15; i++){ + await ptlCur.MoveNextAsync(); + } + Assert.Equal(baseRows.Skip(15).Select(d => d.Id), (await ptlCur.ToListAsync()).Select(d => d.Id)); + Assert.Equal(CursorState.Closed, ptlCur.State); + + // Tests on *mapped cursors + ToList* are omitted, as such logic occurs not in cursor territory anymore (rather LINQ's). + + // Full ForEach + /* TODO (this section): + c# differs greatly from other clients. There's no ForEach on cursors (and arguably there shouldn't be). + As is now, this part passes but adds little. Even the client pattern of a "ForEach(callback)", with the + callback returning whether to stop or not, probably is not needed here, nor is it idiomatic. + I think we should be ok with there not being any ForEach fancy thing other than the LINQ stuff (so dropping this part of test?) + */ + var accum0 = new List(); + var feCur = filledCollection.Find(); + await foreach (var row in feCur) + { + accum0.Add(row); + } + Assert.Equal(baseRows.Select(d => d.Id), accum0.Select(d => d.Id)); + Assert.Equal(CursorState.Closed, feCur.State); + + /* TODO in the same spirit, porting from Python I am skipping: + 1. same as above with a coroutine callback (does not apply here) + 2. foreach on a partially-consumed cursor (trivial once exited cursor to LINQ-land) + 3. 1+2 + 4. mapped ForEach (we're not testing LINQ after all) + 5. mapped ForEach with coroutine + 6. early-break ForEach & coroutine forms, various return types thereof (don't apply) + */ + } + + [Fact] + public async Task Test_CollectionCursor_InitialPageState() + { + const int PAGE_SIZE = 20; + + var filledPagCollection = _fixture.FilledPaginationCollection; + var cur = filledPagCollection.Find( + Builders.CollectionFilter.Eq(d => d.even, true)); + + // TODO This test is made of two parts (in its Python inspiration) + // Part 1: accesses the cursor private page state and puts it to test in a few ways. + // This cannot be done here. The rest is just this (which could be moved to next test): + + // Part 2: tests an overload `Rewind(initialPageState)` which is not available. + // TODO should it be added? (Python has it) + } + + [Fact] + public async Task Test_CollectionCursor_FetchNextPage() + { + var filledPagCollection = _fixture.FilledPaginationCollection; + + var cur0 = filledPagCollection.Find( + Builders.CollectionFilter.Eq(d => d.even, true)); + var page0 = await cur0.FetchNextPageAsync(); + // TODO this should be `Results` (see yellow doc for 'specs') and not `Results` + var ids0 = page0.Results.Select(d => d.id).ToList(); + var nps0 = page0.NextPageState; + Assert.IsType(nps0); + + // TODO: are we ok that there is no 'options' to Find, alternative to fluent way to set initialPS? + var cur1 = filledPagCollection.Find( + Builders.CollectionFilter.Eq(d => d.even, true) + ).InitialPageState(nps0); + var page1 = await cur1.FetchNextPageAsync(); + var ids1 = page1.Results.Select(d => d.id).ToList(); + var nps1 = page1.NextPageState; + Assert.IsType(nps1); + + var cur2 = filledPagCollection.Find( + Builders.CollectionFilter.Eq(d => d.even, true) + ).InitialPageState(nps1); + var page2 = await cur2.FetchNextPageAsync(); + var ids2 = page2.Results.Select(d => d.id).ToList(); + Assert.Null(page2.NextPageState); + + var expectedIds = new List(); + for (int id = 0; id < _fixture.FilledPaginationCollectionCount; id+=2){ expectedIds.Add(id); } + var retrievedIds = ids0.Concat(ids1).Concat(ids2).ToList(); + Assert.Equal(expectedIds, retrievedIds.OrderBy(x => x).ToList()); + + // Fetching consecutive pages on a given cursor + var cur0x = filledPagCollection.Find( + Builders.CollectionFilter.Eq(d => d.even, true)); + await cur0x.FetchNextPageAsync(); + var page1x = await cur0x.FetchNextPageAsync(); + Assert.Equal(page1x.NextPageState, page1.NextPageState); + Assert.Equivalent(page1x.Results, page1.Results); + Assert.Equal(page1x.SortVector, page1.SortVector); + + // Forbidden: mixing pagination and ordinary usage + var cur0y = filledPagCollection.Find( + Builders.CollectionFilter.Eq(d => d.even, true)); + await cur0y.MoveNextAsync(); + await Assert.ThrowsAsync( async () => + { + // errors because we're in mid-page + await cur0y.FetchNextPageAsync(); + }); + await cur0y.ToListAsync(); + await Assert.ThrowsAsync( async () => + { + // errors because closed already (by ToList) + await cur0y.FetchNextPageAsync(); + }); + + // Include-sort-vector: + var vcur0 = filledPagCollection.Find() + .IncludeSortVector(true) + .Limit(15) + .Sort(Builders.CollectionSort.Vector(new float[] { 1.0f, 1.0f })); + var vpage0 = await vcur0.FetchNextPageAsync(); + Assert.Null(vpage0.NextPageState); + Assert.Equal(15, vpage0.Results.Count); + Assert.IsType(vpage0.SortVector); + } + +} diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs index 6f3e3ba1..a836ffd3 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/SearchTests.cs @@ -202,7 +202,7 @@ public async Task VectorizeFindOne() var collection = await fixture.Database.CreateCollectionAsync(collectionName, options); var insertResult = await collection.InsertManyAsync(items); Assert.Equal(items.Count, insertResult.InsertedIds.Count); - var findOptions = new DocumentFindOptions() + var findOptions = new CollectionFindOneOptions() { Sort = Builders.CollectionSort.Vectorize("dog"), IncludeSimilarity = true @@ -945,7 +945,7 @@ public async Task LexicalFindOne() var collection = await fixture.Database.CreateCollectionAsync(collectionName); var insertResult = await collection.InsertManyAsync(items); Assert.Equal(items.Count, insertResult.InsertedIds.Count); - var findOptions = new DocumentFindOptions() + var findOptions = new CollectionFindOneOptions() { Sort = Builders.CollectionSort.Lexical("dog"), Filter = Builders.CollectionFilter.LexicalMatch("dog"), diff --git a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableTests.cs b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableTests.cs index 3c28d97a..c19de569 100644 --- a/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableTests.cs +++ b/test/DataStax.AstraDB.DataApi.IntegrationTests/Tests/TableTests.cs @@ -178,7 +178,7 @@ public async Task FindOne_Sort() var sorter = Builders.TableSort; var sort = sorter.Descending(b => b.Title); var projection = Builders.Projection.Include(b => b.Title); - var result = await table.FindOneAsync(null, new TableFindOptions() { Sort = sort, Projection = projection }); + var result = await table.FindOneAsync(null, new TableFindOneOptions() { Sort = sort, Projection = projection }); Assert.Equal("Title 99", result.Title); Assert.Null(result.Author); } @@ -190,7 +190,7 @@ public void FindOne_Sort_Skip_Exclude() var sorter = Builders.TableSort; var sort = sorter.Ascending(b => b.Title); var projection = Builders.Projection.Exclude(b => b.DueDate); - var results = table.Find().Sort(sort).Project(projection).Skip(2).Limit(5); + var results = table.Find().Sort(sort).Project(projection).Skip(2).Limit(5).ToList(); Assert.Equal(5, results.Count()); // due to 'Computed...' and 'Desert...', the third is 'Title 0' here Assert.Equal("Title 0", results.First().Title); @@ -208,11 +208,12 @@ public void FindOne_Find_SortByVector() var findResults = table.Find().Sort(sort).Project(projection).Limit(1); Assert.Equal(1, findResults.Count()); + findResults.Rewind(); Assert.Equal("last_component", findResults.First().Id); - + var findOneResult = table.FindOne( null, - new TableFindOptions() { + new TableFindOneOptions() { Sort = sort, Projection = projection } ); @@ -545,13 +546,13 @@ public async Task FindOne_Vectorize_Untyped() var sorter = Builders.TableSort; var sort = sorter.Vectorize("Vectorize", "String To Vectorize 22"); var results = await fixture.UntypedTableSinglePrimaryKey.FindOneAsync(null, - new TableFindOptions() { Sort = sort, IncludeSimilarity = true }); + new TableFindOneOptions() { Sort = sort, IncludeSimilarity = true }); Assert.Equal("Name_22", results["Name"].ToString()); results = await fixture.UntypedTableCompositePrimaryKey.FindOneAsync(null, - new TableFindOptions() { Sort = sort, IncludeSimilarity = true }); + new TableFindOneOptions() { Sort = sort, IncludeSimilarity = true }); Assert.Equal("Name_22", results["Name"].ToString()); results = await fixture.UntypedTableCompoundPrimaryKey.FindOneAsync(null, - new TableFindOptions() { Sort = sort, IncludeSimilarity = true }); + new TableFindOneOptions() { Sort = sort, IncludeSimilarity = true }); Assert.Equal("Name_22", results["Name"].ToString()); } @@ -561,11 +562,11 @@ public async Task FindOne_Sort_Untyped() var sorter = Builders.TableSort; var sort = sorter.Descending("Name"); var projection = Builders.Projection.Include("Name"); - var result = await fixture.UntypedTableSinglePrimaryKey.FindOneAsync(null, new TableFindOptions() { Sort = sort, Projection = projection }); + var result = await fixture.UntypedTableSinglePrimaryKey.FindOneAsync(null, new TableFindOneOptions() { Sort = sort, Projection = projection }); Assert.Equal("Name_9", result["Name"].ToString()); - result = await fixture.UntypedTableCompositePrimaryKey.FindOneAsync(null, new TableFindOptions() { Sort = sort, Projection = projection }); + result = await fixture.UntypedTableCompositePrimaryKey.FindOneAsync(null, new TableFindOneOptions() { Sort = sort, Projection = projection }); Assert.Equal("Name_9", result["Name"].ToString()); - result = await fixture.UntypedTableCompoundPrimaryKey.FindOneAsync(null, new TableFindOptions() { Sort = sort, Projection = projection }); + result = await fixture.UntypedTableCompoundPrimaryKey.FindOneAsync(null, new TableFindOneOptions() { Sort = sort, Projection = projection }); Assert.Equal("Name_9", result["Name"].ToString()); } @@ -646,7 +647,7 @@ public async Task FindOne_Lexical() await table.CreateTextIndexAsync("b_idx", (b) => b.LexicalValue, Builders.TableIndex.Text()); var insertResult = await table.InsertManyAsync(items); Assert.Equal(items.Count, insertResult.InsertedIdTuples.Count); - var findOptions = new TableFindOptions() + var findOptions = new TableFindOneOptions() { Sort = Builders.TableSort.Lexical((b) => b.LexicalValue, "dog"), Filter = Builders.TableFilter.LexicalMatch((b) => b.LexicalValue, "dog"),