diff --git a/storage/Directory.Build.props b/storage/Directory.Build.props new file mode 100644 index 000000000..c9c5763c6 --- /dev/null +++ b/storage/Directory.Build.props @@ -0,0 +1,9 @@ + + + + Duende.$(MSBuildProjectName) + Duende.$(MSBuildProjectName) + Duende.$(MSBuildProjectName) + + + diff --git a/storage/Directory.Build.targets b/storage/Directory.Build.targets new file mode 100644 index 000000000..8efea5859 --- /dev/null +++ b/storage/Directory.Build.targets @@ -0,0 +1,9 @@ + + + + + + $([System.String]::Copy('%(Filename)').Replace('.g', '.cs')) + + + diff --git a/storage/src/Directory.Build.props b/storage/src/Directory.Build.props new file mode 100644 index 000000000..9cb0617c1 --- /dev/null +++ b/storage/src/Directory.Build.props @@ -0,0 +1,10 @@ + + + + + 0.1 + storage- + Duende Storage + + + diff --git a/storage/src/Directory.Build.targets b/storage/src/Directory.Build.targets new file mode 100644 index 000000000..822e9c81c --- /dev/null +++ b/storage/src/Directory.Build.targets @@ -0,0 +1,6 @@ + + + + + diff --git a/storage/src/Storage.CliPlugin/AssemblyInfo.cs b/storage/src/Storage.CliPlugin/AssemblyInfo.cs new file mode 100644 index 000000000..fa2b03ae4 --- /dev/null +++ b/storage/src/Storage.CliPlugin/AssemblyInfo.cs @@ -0,0 +1,7 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Cli.PluginAbstractions; +using Duende.Storage.CliPlugin; + +[assembly: CliPlugin(typeof(StorageCliPlugin))] diff --git a/storage/src/Storage.CliPlugin/Commands/MigrateCommand.cs b/storage/src/Storage.CliPlugin/Commands/MigrateCommand.cs new file mode 100644 index 000000000..fa3e90d78 --- /dev/null +++ b/storage/src/Storage.CliPlugin/Commands/MigrateCommand.cs @@ -0,0 +1,23 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.CommandLine; + +namespace Duende.Storage.CliPlugin.Commands; + +/// +/// Provides the duende storage migrate command. +/// +internal static class MigrateCommand +{ + internal static Command Create() + { + var command = new Command("migrate", "Apply pending Duende Storage schema migrations."); + command.SetAction(async (_, ct) => + { + Console.WriteLine("Storage migrate: not yet implemented."); + await Task.CompletedTask; + }); + return command; + } +} diff --git a/storage/src/Storage.CliPlugin/Storage.CliPlugin.csproj b/storage/src/Storage.CliPlugin/Storage.CliPlugin.csproj new file mode 100644 index 000000000..128ca17f0 --- /dev/null +++ b/storage/src/Storage.CliPlugin/Storage.CliPlugin.csproj @@ -0,0 +1,47 @@ + + + + + Duende.Storage.CliPlugin + Duende.Storage.CliPlugin + Duende.Storage.CliPlugin + + + true + + + + + + + + + + + + + + $(TargetsForTfmSpecificBuildOutput);CollectPluginDependencies + + + + + + + + + + + diff --git a/storage/src/Storage.CliPlugin/StorageCliPlugin.cs b/storage/src/Storage.CliPlugin/StorageCliPlugin.cs new file mode 100644 index 000000000..04fd5eb8d --- /dev/null +++ b/storage/src/Storage.CliPlugin/StorageCliPlugin.cs @@ -0,0 +1,26 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.CommandLine; +using Duende.Cli.PluginAbstractions; +using Duende.Storage.CliPlugin.Commands; + +namespace Duende.Storage.CliPlugin; + +/// +/// Entry point for the Duende Storage CLI plugin. +/// Provides the storage subcommand tree to the duende CLI host. +/// +public sealed class StorageCliPlugin : ICliPlugin +{ + /// + public string Name => "storage"; + + /// + public Command GetCommand() + { + var storageCommand = new Command("storage", "Commands for managing Duende Storage."); + storageCommand.Subcommands.Add(MigrateCommand.Create()); + return storageCommand; + } +} diff --git a/storage/src/Storage.MsSql/CreateSqlConnection.cs b/storage/src/Storage.MsSql/CreateSqlConnection.cs new file mode 100644 index 000000000..beea1f721 --- /dev/null +++ b/storage/src/Storage.MsSql/CreateSqlConnection.cs @@ -0,0 +1,12 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Microsoft.Data.SqlClient; + +namespace Duende.Storage.MsSql; + +/// +/// A delegate that creates an unopened . +/// Register as a keyed service matching the store's service key. +/// +public delegate SqlConnection CreateSqlConnection(); diff --git a/storage/src/Storage.MsSql/Internal/Log.cs b/storage/src/Storage.MsSql/Internal/Log.cs new file mode 100644 index 000000000..55beb61d4 --- /dev/null +++ b/storage/src/Storage.MsSql/Internal/Log.cs @@ -0,0 +1,90 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Microsoft.Extensions.Logging; + +namespace Duende.Storage.MsSql.Internal; + +internal static partial class Log +{ + [LoggerMessage(Level = LogLevel.Information, Message = $"Checking schema version")] + internal static partial void CheckingSchemaVersion(ILogger logger); + + [LoggerMessage(Level = LogLevel.Information, Message = $"Creating schema {{{Parameters.SchemaName}}}")] + internal static partial void CreatingSchema(ILogger logger, string schemaName); + + [LoggerMessage(Level = LogLevel.Information, Message = $"Migrating schema {{{Parameters.SchemaName}}}")] + internal static partial void MigratingSchema(ILogger logger, string schemaName); + + [LoggerMessage(Level = LogLevel.Information, Message = $"Verifying schema {{{Parameters.SchemaName}}}")] + internal static partial void VerifyingSchema(ILogger logger, string schemaName); + + [LoggerMessage(Level = LogLevel.Information, Message = $"Executing migration step V{{{Parameters.FromVersion}}}→V{{{Parameters.ToVersion}}}")] + internal static partial void ExecutingMigrationStep(ILogger logger, int fromVersion, int toVersion); + + [LoggerMessage(Level = LogLevel.Warning, Message = $"Error While creating schema")] + internal static partial void ErrorWhileCreatingSchema(ILogger logger, Exception e); + + [LoggerMessage(Level = LogLevel.Debug, Message = $"Executing sql {{{Parameters.Sql}}}")] + internal static partial void ExecutingSql(ILogger logger, string sql); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Creating DSO: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Id)}={{{Parameters.Id}}}, {nameof(Parameters.DsoSchemaVersion)}={{{Parameters.DsoSchemaVersion}}}")] + internal static partial void CreatingDso(ILogger logger, EntityType entityType, Guid id, uint dsoSchemaVersion); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Deleting DSO: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Id)}={{{Parameters.Id}}}")] + internal static partial void DeletingDso(ILogger logger, EntityType entityType, Guid id); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Reading DSO: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Id)}={{{Parameters.Id}}}")] + internal static partial void ReadingDso(ILogger logger, EntityType entityType, Guid id); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Reading DSOs: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Count)}={{{Parameters.Count}}}")] + internal static partial void ReadingDsos(ILogger logger, EntityType entityType, int count); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Updating DSO: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Id)}={{{Parameters.Id}}}, {nameof(Parameters.DsoSchemaVersion)}={{{Parameters.DsoSchemaVersion}}}, {nameof(Parameters.ExpectedEntityVersion)}={{{Parameters.ExpectedEntityVersion}}}")] + internal static partial void UpdatingDso(ILogger logger, EntityType entityType, Guid id, uint dsoSchemaVersion, int expectedEntityVersion); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Querying DSOs: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Skip)}={{{Parameters.Skip}}}, {nameof(Parameters.Take)}={{{Parameters.Take}}}")] + internal static partial void QueryingDsos(ILogger logger, EntityType entityType, int skip, int take); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Querying DSO fields: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.FieldCount)}={{{Parameters.FieldCount}}}, {nameof(Parameters.Skip)}={{{Parameters.Skip}}}, {nameof(Parameters.Take)}={{{Parameters.Take}}}")] + internal static partial void QueryingFieldsDsos(ILogger logger, EntityType entityType, int fieldCount, int skip, int take); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Executing query: {nameof(Parameters.Query)}={{{Parameters.Query}}}")] + internal static partial void ExecutingQuery(ILogger logger, string query); + + private static class Parameters + { + internal const string FromVersion = nameof(FromVersion); + internal const string ToVersion = nameof(ToVersion); + internal const string Count = nameof(Count); + internal const string DsoSchemaVersion = nameof(DsoSchemaVersion); + internal const string EntityType = nameof(EntityType); + internal const string ExpectedEntityVersion = nameof(ExpectedEntityVersion); + internal const string FieldCount = nameof(FieldCount); + internal const string Id = nameof(Id); + internal const string Skip = nameof(Skip); + internal const string Take = nameof(Take); + internal const string Query = nameof(Query); + internal const string SchemaName = nameof(SchemaName); + internal const string Sql = nameof(Sql); + } + + +} diff --git a/storage/src/Storage.MsSql/Internal/MigrationScriptLoader.cs b/storage/src/Storage.MsSql/Internal/MigrationScriptLoader.cs new file mode 100644 index 000000000..9f5f21d73 --- /dev/null +++ b/storage/src/Storage.MsSql/Internal/MigrationScriptLoader.cs @@ -0,0 +1,46 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Globalization; +using System.Reflection; +using System.Text.RegularExpressions; + +namespace Duende.Storage.MsSql.Internal; + +internal static class MigrationScriptLoader +{ + private static readonly Regex VersionPattern = new(@"\.Migrations\.V(\d+)_", RegexOptions.Compiled); + + public static IEnumerable<(int TargetVersion, string Sql)> GetScripts( + Assembly assembly, + DatabaseSchemaVersion fromVersion, + string schemaName) + { + var assemblyName = assembly.GetName().Name; + var prefix = $"{assemblyName}.Migrations.V"; + + return assembly.GetManifestResourceNames() + .Where(name => name.StartsWith(prefix, StringComparison.Ordinal) && name.EndsWith(".sql", StringComparison.Ordinal)) + .Select(name => (Name: name, Version: ParseVersion(name))) + .Where(x => x.Version > fromVersion.Value) + .OrderBy(x => x.Version) + .Select(x => (x.Version, ApplySchema(ReadResource(assembly, x.Name), schemaName))); + } + + private static int ParseVersion(string resourceName) + { + var match = VersionPattern.Match(resourceName); + return match.Success ? int.Parse(match.Groups[1].Value, CultureInfo.InvariantCulture) : 0; + } + + private static string ReadResource(Assembly assembly, string resourceName) + { + using var stream = assembly.GetManifestResourceStream(resourceName) + ?? throw new InvalidOperationException($"Embedded resource '{resourceName}' not found."); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } + + private static string ApplySchema(string sql, string schemaName) => + sql.Replace("[[schemaname]]", schemaName, StringComparison.Ordinal); +} diff --git a/storage/src/Storage.MsSql/Internal/MsSqlDialect.cs b/storage/src/Storage.MsSql/Internal/MsSqlDialect.cs new file mode 100644 index 000000000..d60a7a056 --- /dev/null +++ b/storage/src/Storage.MsSql/Internal/MsSqlDialect.cs @@ -0,0 +1,57 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Data; +using System.Data.Common; +using Duende.Storage.Internal.Querying; +using Microsoft.Data.SqlClient; + +namespace Duende.Storage.MsSql.Internal; + +/// +/// SQL Server-specific SQL dialect implementation. +/// +internal sealed class MsSqlDialect : ISqlDialect +{ + public string CaseInsensitiveLikeOperator => "LIKE"; + + public string TrueLiteral => "1=1"; + + public string FalseLiteral => "1=0"; + + public string EscapeLikeWildcards(string value) + { + if (string.IsNullOrEmpty(value)) + { + return value; + } + + // SQL Server uses brackets to escape special characters + return value + .Replace("[", "[[]", StringComparison.OrdinalIgnoreCase) + .Replace("%", "[%]", StringComparison.OrdinalIgnoreCase) + .Replace("_", "[_]", StringComparison.OrdinalIgnoreCase); + } + + public void AddParameter(DbCommand command, string name, object value) + { + var sqlCommand = (SqlCommand)command; + + // Handle DateTimeOffset with explicit type + if (value is DateTimeOffset dto) + { + var param = sqlCommand.Parameters.AddWithValue(name, dto); + param.SqlDbType = SqlDbType.DateTimeOffset; + } + else if (value is DateTime dt) + { + // Convert to DateTimeOffset assuming UTC + var param = sqlCommand.Parameters.AddWithValue(name, new DateTimeOffset(dt, TimeSpan.Zero)); + param.SqlDbType = SqlDbType.DateTimeOffset; + } + else + { + _ = sqlCommand.Parameters.AddWithValue(name, value); + } + } +} diff --git a/storage/src/Storage.MsSql/Internal/MsSqlStore.cs b/storage/src/Storage.MsSql/Internal/MsSqlStore.cs new file mode 100644 index 000000000..ebe671c29 --- /dev/null +++ b/storage/src/Storage.MsSql/Internal/MsSqlStore.cs @@ -0,0 +1,2812 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Data; +using System.Globalization; +using System.Text; +using System.Text.Json; +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Outbox; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Expressions; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Internal.Querying.SearchFields; +using Duende.Storage.Internal.Querying.Sorting; +using Duende.Storage.Pagination; +using Duende.Storage.Querying; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.Logging; +using CursorToken = Duende.Storage.Internal.Querying.CursorToken; +using OutboxEventId = Duende.Storage.Internal.Outbox.OutboxEventId; +using OutboxEventName = Duende.Storage.Internal.Outbox.OutboxEventName; +using SubscriberName = Duende.Storage.Internal.Outbox.SubscriberName; + +namespace Duende.Storage.MsSql.Internal; + +#pragma warning disable CA1812 // Avoid uninstantiated internal classes +#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities +internal sealed class MsSqlStore( + CreateSqlConnection createConnection, + MsSqlStoreOptions options, + DataStorageTypeRegistry dataStorageTypeRegistry, + TimeProvider timeProvider, + OutboxSubscribers outboxSubscribers, + ILogger logger) : StoreBase, IStore, IDatabaseSchema +{ + private const int RequiredSchemaVersion = 1; + private readonly string _schemaName = options.SchemaName; + + private SqlConnection OpenConnection() => createConnection(); + + async Task IDatabaseSchema.CheckVersionAsync(Ct ct) + { + Log.CheckingSchemaVersion(logger); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + + await using var cmd = connection.CreateCommand(); + cmd.CommandType = CommandType.Text; + cmd.CommandText = $""" + SELECT CAST(JSON_VALUE(CAST(value AS NVARCHAR(MAX)), '$.Version') AS INT) as Version + FROM sys.extended_properties ep + WHERE ep.class = 3 + AND ep.name = N'SchemaVersion' + AND ep.major_id = SCHEMA_ID('{_schemaName}') + """; + + Log.ExecutingSql(logger, cmd.CommandText); + var scalar = await cmd.ExecuteScalarAsync(ct); + + if (scalar is null or DBNull) + { + return new CheckSchemaVersionResult(0, RequiredSchemaVersion); + } + + var version = (uint)Convert.ToInt32(scalar, CultureInfo.InvariantCulture); + return new CheckSchemaVersionResult(version, RequiredSchemaVersion); + } + + async Task IDatabaseSchema.MigrateAsync(Ct ct) + { + Log.MigratingSchema(logger, _schemaName); + + var versionResult = await ((IDatabaseSchema)this).CheckVersionAsync(ct); + var currentVersion = new DatabaseSchemaVersion((int)versionResult.CurrentVersion); + + var scripts = MigrationScriptLoader.GetScripts(typeof(MsSqlStore).Assembly, currentVersion, _schemaName).ToList(); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + + foreach (var (_, sql) in scripts) + { + await using var transaction = await connection.BeginTransactionAsync(ct); + try + { + // Acquire application lock for safe concurrent migration + await using (var lockCmd = connection.CreateCommand()) + { + lockCmd.Transaction = (SqlTransaction)transaction; + lockCmd.CommandType = CommandType.Text; + lockCmd.CommandText = """ + DECLARE @result INT; + EXEC @result = sp_getapplock + @Resource = N'__schema_migration__', + @LockMode = N'Exclusive', + @LockOwner = N'Transaction', + @LockTimeout = 60000; + + IF @result < 0 + BEGIN + THROW 50000, 'Failed to acquire application lock for migration', 1; + END + """; + Log.ExecutingSql(logger, lockCmd.CommandText); + _ = await lockCmd.ExecuteNonQueryAsync(ct); + } + + // Execute the migration script (version gate and version bump are inside the SQL) + await using (var stepCmd = connection.CreateCommand()) + { + stepCmd.Transaction = (SqlTransaction)transaction; + stepCmd.CommandType = CommandType.Text; + stepCmd.CommandText = sql; + Log.ExecutingSql(logger, stepCmd.CommandText); + _ = await stepCmd.ExecuteNonQueryAsync(ct); + } + + await transaction.CommitAsync(ct); + } + catch (SqlException e) + { + Log.ErrorWhileCreatingSchema(logger, e); + await transaction.RollbackAsync(ct); + throw; + } + } + + var verifyResult = await ((IDatabaseSchema)this).VerifySchemaAsync(ct); + if (!verifyResult.IsValid) + { + var errors = string.Join("; ", verifyResult.Errors.Select(e => e.ErrorMessage)); + throw new InvalidOperationException($"Schema verification failed after migration: {errors}"); + } + } + + async Task IDatabaseSchema.VerifySchemaAsync(Ct ct) + { + Log.VerifyingSchema(logger, _schemaName); + + var errors = new List(); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + + // Expected tables and their required columns (table -> column -> data_type) + var expectedColumns = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + ["entities"] = new(StringComparer.OrdinalIgnoreCase) + { + ["pool_id"] = "int", + ["entity_type_id"] = "int", + ["entity_id"] = "uniqueidentifier", + ["original_entity_id"] = "uniqueidentifier", + ["entity_type_name"] = "nvarchar", + ["value"] = "nvarchar", + ["dso_type_schema_version"] = "int", + ["value_version"] = "int", + ["created_at"] = "datetimeoffset", + ["last_updated_at"] = "datetimeoffset", + ["expires_at"] = "datetimeoffset", + }, + ["entity_keys"] = new(StringComparer.OrdinalIgnoreCase) + { + ["pool_id"] = "int", + ["entity_type_id"] = "int", + ["key_type_id"] = "int", + ["entity_id"] = "uniqueidentifier", + ["key_type_name"] = "nvarchar", + ["key_type_version"] = "int", + ["key_value"] = "uniqueidentifier", + ["key_json"] = "nvarchar", + ["timestamp"] = "datetimeoffset", + }, + ["search_values"] = new(StringComparer.OrdinalIgnoreCase) + { + ["pool_id"] = "int", + ["entity_type_id"] = "int", + ["entity_id"] = "uniqueidentifier", + ["field_path"] = "uniqueidentifier", + ["field_path_text"] = "nvarchar", + ["item_index"] = "int", + ["string_value"] = "nvarchar", + ["number_value"] = "decimal", + ["datetime_value"] = "datetimeoffset", + ["boolean_value"] = "bit", + ["guid_value"] = "uniqueidentifier", + }, + ["entity_links"] = new(StringComparer.OrdinalIgnoreCase) + { + ["pool_id"] = "int", + ["link_type_id"] = "int", + ["left_entity_type_id"] = "int", + ["left_entity_id"] = "uniqueidentifier", + ["right_entity_type_id"] = "int", + ["right_entity_id"] = "uniqueidentifier", + ["created_at"] = "datetimeoffset", + }, + ["outbox_subscriber_queue"] = new(StringComparer.OrdinalIgnoreCase) + { + ["sequence_number"] = "bigint", + ["message_id"] = "uniqueidentifier", + ["event_id"] = "uniqueidentifier", + ["timestamp"] = "datetimeoffset", + ["event_name"] = "nvarchar", + ["subject_id"] = "uniqueidentifier", + ["entity_type_id"] = "int", + ["entity_type_name"] = "nvarchar", + ["pool_id"] = "int", + ["payload"] = "nvarchar", + ["subscriber_name"] = "nvarchar", + }, + }; + + // Query actual columns from INFORMATION_SCHEMA + await using (var cmd = connection.CreateCommand()) + { + cmd.CommandType = CommandType.Text; + cmd.CommandText = $""" + SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE + FROM INFORMATION_SCHEMA.COLUMNS + WHERE TABLE_SCHEMA = '{_schemaName}' + ORDER BY TABLE_NAME, COLUMN_NAME + """; + Log.ExecutingSql(logger, cmd.CommandText); + + var actualColumns = new Dictionary>(StringComparer.OrdinalIgnoreCase); + var actualColumnTypes = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + var tableName = reader.GetString(0); + var columnName = reader.GetString(1); + var dataType = reader.GetString(2); + + if (!actualColumns.TryGetValue(tableName, out var cols)) + { + cols = new HashSet(StringComparer.OrdinalIgnoreCase); + actualColumns[tableName] = cols; + actualColumnTypes[tableName] = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + _ = cols.Add(columnName); + actualColumnTypes[tableName][columnName] = dataType; + } + + foreach (var (tableName, columns) in expectedColumns) + { + if (!actualColumns.ContainsKey(tableName)) + { + errors.Add(new SchemaVerificationError( + tableName, null, + $"Table '{_schemaName}.{tableName}' is missing.", + SchemaVerificationErrorKind.MissingTable)); + continue; + } + + foreach (var (columnName, expectedType) in columns) + { + if (!actualColumns[tableName].Contains(columnName)) + { + errors.Add(new SchemaVerificationError( + tableName, columnName, + $"Column '{columnName}' is missing from table '{_schemaName}.{tableName}'.", + SchemaVerificationErrorKind.MissingColumn)); + } + else if (!string.Equals(actualColumnTypes[tableName][columnName], expectedType, StringComparison.OrdinalIgnoreCase)) + { + errors.Add(new SchemaVerificationError( + tableName, columnName, + $"Column '{columnName}' in table '{_schemaName}.{tableName}' has type '{actualColumnTypes[tableName][columnName]}' but expected '{expectedType}'.", + SchemaVerificationErrorKind.WrongType)); + } + } + } + } + + // Verify required indexes + var expectedIndexes = new[] + { + ("entities", $"IX_{_schemaName}_entities_expires_at"), + ("entities", $"IX_{_schemaName}_entities_entity_type_name"), + ("entities", $"IX_{_schemaName}_entities_created_at"), + ("entities", $"IX_{_schemaName}_entities_last_updated_at"), + ("entity_keys", $"IX_{_schemaName}_entity_keys_entity_type_id_entity_id"), + ("search_values", $"IX_{_schemaName}_search_values_string_value"), + ("search_values", $"IX_{_schemaName}_search_values_number_value"), + ("search_values", $"IX_{_schemaName}_search_values_datetime_value"), + ("search_values", $"IX_{_schemaName}_search_values_boolean_value"), + ("search_values", $"IX_{_schemaName}_search_values_array_string_value"), + ("search_values", $"IX_{_schemaName}_search_values_array_number_value"), + ("search_values", $"IX_{_schemaName}_search_values_array_datetime_value"), + ("search_values", $"IX_{_schemaName}_search_values_array_boolean_value"), + ("search_values", $"IX_{_schemaName}_search_values_guid_value"), + ("search_values", $"IX_{_schemaName}_search_values_array_guid_value"), + ("entity_links", $"IX_{_schemaName}_entity_links_left_entity"), + ("entity_links", $"IX_{_schemaName}_entity_links_right_entity"), + ("entity_links", $"IX_{_schemaName}_entity_links_left_cascade"), + ("entity_links", $"IX_{_schemaName}_entity_links_right_cascade"), + ("outbox_subscriber_queue", $"IX_{_schemaName}_outbox_subscriber_queue_subscriber"), + }; + + await using (var cmd = connection.CreateCommand()) + { + cmd.CommandType = CommandType.Text; + cmd.CommandText = $""" + SELECT t.name AS table_name, i.name AS index_name + FROM sys.indexes i + INNER JOIN sys.tables t ON i.object_id = t.object_id + INNER JOIN sys.schemas s ON t.schema_id = s.schema_id + WHERE s.name = N'{_schemaName}' + AND i.name IS NOT NULL + """; + Log.ExecutingSql(logger, cmd.CommandText); + + var actualIndexes = new HashSet<(string Table, string Index)>( + EqualityComparer<(string, string)>.Create( + (a, b) => string.Equals(a.Item1, b.Item1, StringComparison.OrdinalIgnoreCase) && + string.Equals(a.Item2, b.Item2, StringComparison.OrdinalIgnoreCase), + x => StringComparer.OrdinalIgnoreCase.GetHashCode(x.Item1) ^ StringComparer.OrdinalIgnoreCase.GetHashCode(x.Item2))); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + _ = actualIndexes.Add((reader.GetString(0), reader.GetString(1))); + } + + foreach (var (tableName, indexName) in expectedIndexes) + { + if (!actualIndexes.Contains((tableName, indexName))) + { + errors.Add(new SchemaVerificationError( + tableName, null, + $"Index '{indexName}' is missing from table '{_schemaName}.{tableName}'.", + SchemaVerificationErrorKind.MissingIndex)); + } + } + } + + // Verify required foreign keys + var expectedForeignKeys = new[] + { + ("entity_keys", $"FK_{_schemaName}_entity_keys_entities"), + ("search_values", $"FK_{_schemaName}_search_values_entities"), + }; + + await using (var cmd = connection.CreateCommand()) + { + cmd.CommandType = CommandType.Text; + cmd.CommandText = $""" + SELECT t.name AS table_name, fk.name AS fk_name + FROM sys.foreign_keys fk + INNER JOIN sys.tables t ON fk.parent_object_id = t.object_id + INNER JOIN sys.schemas s ON t.schema_id = s.schema_id + WHERE s.name = N'{_schemaName}' + """; + Log.ExecutingSql(logger, cmd.CommandText); + + var actualForeignKeys = new HashSet<(string Table, string Fk)>( + EqualityComparer<(string, string)>.Create( + (a, b) => string.Equals(a.Item1, b.Item1, StringComparison.OrdinalIgnoreCase) && + string.Equals(a.Item2, b.Item2, StringComparison.OrdinalIgnoreCase), + x => StringComparer.OrdinalIgnoreCase.GetHashCode(x.Item1) ^ StringComparer.OrdinalIgnoreCase.GetHashCode(x.Item2))); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + _ = actualForeignKeys.Add((reader.GetString(0), reader.GetString(1))); + } + + foreach (var (tableName, fkName) in expectedForeignKeys) + { + if (!actualForeignKeys.Contains((tableName, fkName))) + { + errors.Add(new SchemaVerificationError( + tableName, null, + $"Foreign key '{fkName}' is missing from table '{_schemaName}.{tableName}'.", + SchemaVerificationErrorKind.MissingForeignKey)); + } + } + } + + // Verify required user-defined table types (TVPs) + var expectedTypes = new[] + { + "KeyTableType", + "SearchValueTableType", + "EntityIdTableType", + "ExpiredEntityKeyTableType", + }; + + await using (var cmd = connection.CreateCommand()) + { + cmd.CommandType = CommandType.Text; + cmd.CommandText = $""" + SELECT t.name + FROM sys.table_types t + INNER JOIN sys.schemas s ON t.schema_id = s.schema_id + WHERE s.name = N'{_schemaName}' + """; + Log.ExecutingSql(logger, cmd.CommandText); + + var actualTypes = new HashSet(StringComparer.OrdinalIgnoreCase); + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + _ = actualTypes.Add(reader.GetString(0)); + } + + foreach (var typeName in expectedTypes) + { + if (!actualTypes.Contains(typeName)) + { + errors.Add(new SchemaVerificationError( + _schemaName, null, + $"User-defined table type '{_schemaName}.{typeName}' is missing.", + SchemaVerificationErrorKind.MissingUserDefinedType)); + } + } + } + + return new SchemaVerificationResult(errors); + } + + string IDatabaseSchema.BuildMigrationScript(DatabaseSchemaVersion fromVersion) + { + var scripts = MigrationScriptLoader.GetScripts(typeof(MsSqlStore).Assembly, fromVersion, _schemaName); + return string.Concat(scripts.Select(s => s.Sql + Environment.NewLine)); + } + + async Task IStore.CreateAsync( + Storage.UuidV7 id, + TDso value, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration expiration, + IReadOnlyList outboxEvents, + Ct ct) + { + var createOp = CreateOperation.For(id, value, keys, searchFieldCollection, expiration); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + await using var transaction = await connection.BeginTransactionAsync(ct); + + var outcome = await ExecuteCreateCoreAsync(connection, (SqlTransaction)transaction, createOp, ct); + + if (outcome == OperationOutcome.Success) + { + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(connection, (SqlTransaction)transaction, outboxEvents, ct); + } + await transaction.CommitAsync(ct); + } + + return outcome switch + { + OperationOutcome.Success => CreateResult.Success, + OperationOutcome.AlreadyExists => CreateResult.AlreadyExists, + OperationOutcome.KeyConflict => CreateResult.KeyConflict, + _ => throw new InvalidOperationException($"Unexpected outcome from create operation: {outcome}") + }; + } + + async Task IStore.TryReadAsync( + EntityType entityType, + Storage.UuidV7 id, + Ct ct) + { + Log.ReadingDso(logger, entityType, id.Value); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + + await using var cmd = connection.CreateCommand(); + cmd.CommandType = CommandType.Text; + cmd.CommandText = $""" + SELECT value, dso_type_schema_version, value_version, created_at, last_updated_at + FROM [{_schemaName}].[entities] + WHERE entity_type_id = @entityTypeId AND entity_id = @entityId AND pool_id = @poolId + """; + + _ = cmd.Parameters.AddWithValue("@entityTypeId", (int)entityType.Id); + _ = cmd.Parameters.AddWithValue("@entityId", SqlServerGuidConverter.ToSqlServer(id.Value)); + _ = cmd.Parameters.AddWithValue("@poolId", PoolId.Value); + + Log.ExecutingSql(logger, cmd.CommandText); + await using var reader = await cmd.ExecuteReaderAsync(ct); + + if (!await reader.ReadAsync(ct)) + { + return StoreGetResult.NotFound(); + } + + var jsonValue = reader.GetString(0); + var dsoTypeVersion = reader.GetInt32(1); + var valueVersion = reader.GetInt32(2); + var created = reader.GetDateTimeOffset(3); + var lastUpdated = reader.GetDateTimeOffset(4); + + var version = new DataStorageObjectVersion(entityType, (uint)dsoTypeVersion); + var dsoType = dataStorageTypeRegistry.Get(version); + var item = (IDataStorageObject)JsonSerializer.Deserialize(jsonValue, dsoType)!; + + return StoreGetResult.IsFound(item, id.Value, valueVersion, created, lastUpdated); + } + + async Task IStore.TryReadAsync( + EntityType entityType, + DataStorageKey key, + Ct ct) + { + Log.ReadingDso(logger, entityType, key.Value); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + + await using var cmd = connection.CreateCommand(); + cmd.CommandType = CommandType.Text; + cmd.CommandText = $""" + SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at + FROM [{_schemaName}].entity_keys i + INNER JOIN [{_schemaName}].[entities] v + ON i.entity_type_id = v.entity_type_id + AND i.entity_id = v.entity_id + AND i.pool_id = v.pool_id + WHERE i.entity_type_id = @entityTypeId + AND i.key_type_id = @keyTypeId + AND i.key_type_version = @keyTypeVersion + AND i.key_value = @keyValue + AND i.pool_id = @poolId + """; + + _ = cmd.Parameters.AddWithValue("@entityTypeId", (int)entityType.Id); + _ = cmd.Parameters.AddWithValue("@keyTypeId", (int)key.DskVersion.KeyType.Id); + _ = cmd.Parameters.AddWithValue("@keyTypeVersion", (int)key.DskVersion.SchemaVersion); + _ = cmd.Parameters.AddWithValue("@keyValue", key.Value); + _ = cmd.Parameters.AddWithValue("@poolId", PoolId.Value); + + Log.ExecutingSql(logger, cmd.CommandText); + await using var reader = await cmd.ExecuteReaderAsync(ct); + + if (!await reader.ReadAsync(ct)) + { + return StoreGetResult.NotFound(); + } + + var entityId = SqlServerGuidConverter.ToUuidV7(reader.GetGuid(0)); + var jsonValue = reader.GetString(1); + var dsoTypeVersion = reader.GetInt32(2); + var valueVersion = reader.GetInt32(3); + var created = reader.GetDateTimeOffset(4); + var lastUpdated = reader.GetDateTimeOffset(5); + + var version = new DataStorageObjectVersion(entityType, (uint)dsoTypeVersion); + var dsoType = dataStorageTypeRegistry.Get(version); + var item = (IDataStorageObject)JsonSerializer.Deserialize(jsonValue, dsoType)!; + + return StoreGetResult.IsFound(item, entityId, valueVersion, created, lastUpdated); + } + + async Task> IStore.TryReadManyAsync( + EntityType entityType, + IReadOnlySet ids, + int maximum, + Ct ct) + { + if (ids.Count > maximum) + { + throw new InvalidOperationException( + $"The number of requested IDs ({ids.Count}) exceeds the maximum allowed ({maximum})."); + } + + Log.ReadingDsos(logger, entityType, ids.Count); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + + await using var cmd = connection.CreateCommand(); + cmd.CommandType = CommandType.Text; + cmd.CommandText = $""" + SELECT e.entity_id, e.value, e.dso_type_schema_version, e.value_version, e.created_at, e.last_updated_at + FROM [{_schemaName}].[entities] e + INNER JOIN @entityIds ids ON e.entity_id = ids.entity_id + WHERE e.entity_type_id = @entityTypeId AND e.pool_id = @poolId + """; + + _ = cmd.Parameters.AddWithValue("@entityTypeId", (int)entityType.Id); + _ = cmd.Parameters.AddWithValue("@poolId", PoolId.Value); + + var idsTable = new DataTable(); + _ = idsTable.Columns.Add("entity_id", typeof(Guid)); + foreach (var id in ids) + { + _ = idsTable.Rows.Add(SqlServerGuidConverter.ToSqlServer(id.Value)); + } + + var idsParam = cmd.Parameters.AddWithValue("@entityIds", idsTable); + idsParam.SqlDbType = SqlDbType.Structured; + idsParam.TypeName = $"[{_schemaName}].[EntityIdTableType]"; + + Log.ExecutingSql(logger, cmd.CommandText); + await using var reader = await cmd.ExecuteReaderAsync(ct); + + var results = new List(); + while (await reader.ReadAsync(ct)) + { + var entityId = SqlServerGuidConverter.ToUuidV7(reader.GetGuid(0)); + var jsonValue = reader.GetString(1); + var dsoTypeVersion = reader.GetInt32(2); + var valueVersion = reader.GetInt32(3); + var created = reader.GetDateTimeOffset(4); + var lastUpdated = reader.GetDateTimeOffset(5); + + var version = new DataStorageObjectVersion(entityType, (uint)dsoTypeVersion); + var dsoType = dataStorageTypeRegistry.Get(version); + var item = (IDataStorageObject)JsonSerializer.Deserialize(jsonValue, dsoType)!; + + results.Add(StoreGetResult.IsFound(item, entityId, valueVersion, created, lastUpdated)); + } + + return results; + } + + async Task IStore.UpdateAsync( + Storage.UuidV7 id, + TDso dso, + int expectedEntityVersion, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration? expiration, + IReadOnlyList outboxEvents, + Ct ct) + { + var updateOp = UpdateOperation.For(id, dso, expectedEntityVersion, keys, searchFieldCollection, expiration); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + await using var transaction = await connection.BeginTransactionAsync(ct); + + var outcome = await ExecuteUpdateCoreAsync(connection, (SqlTransaction)transaction, updateOp, ct); + + if (outcome == OperationOutcome.Success) + { + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(connection, (SqlTransaction)transaction, outboxEvents, ct); + } + await transaction.CommitAsync(ct); + } + + return outcome switch + { + OperationOutcome.Success => UpdateResult.Success, + OperationOutcome.DoesNotExist => UpdateResult.DoesNotExist, + OperationOutcome.UnexpectedVersion => UpdateResult.UnexpectedVersion, + OperationOutcome.KeyConflict => UpdateResult.KeyConflict, + _ => throw new InvalidOperationException($"Unexpected outcome from update operation: {outcome}") + }; + } + + async Task IStore.DeleteAsync(EntityType entityType, Storage.UuidV7 id, IReadOnlyList outboxEvents, Ct ct) + { + var deleteOp = DeleteOperation.ById(entityType, id); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + await using var transaction = await connection.BeginTransactionAsync(ct); + + var (_, entityDeleted) = await ExecuteDeleteCoreAsync(connection, (SqlTransaction)transaction, deleteOp, ct); + + if (entityDeleted && outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(connection, (SqlTransaction)transaction, outboxEvents, ct); + } + await transaction.CommitAsync(ct); + + return DeleteResult.Success; + } + + async Task IStore.DeleteAsync(EntityType entityType, DataStorageKey key, IReadOnlyList outboxEvents, Ct ct) + { + var deleteOp = DeleteOperation.ByKey(entityType, key); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + await using var transaction = await connection.BeginTransactionAsync(ct); + + var (_, entityDeleted) = await ExecuteDeleteCoreAsync(connection, (SqlTransaction)transaction, deleteOp, ct); + + if (entityDeleted && outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(connection, (SqlTransaction)transaction, outboxEvents, ct); + } + await transaction.CommitAsync(ct); + + return DeleteResult.Success; + } + + /// + async Task IStore.LinkAsync(LinkDefinition definition, Storage.UuidV7 leftEntityId, Storage.UuidV7 rightEntityId, IReadOnlyList outboxEvents, Ct ct) + { + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + await using var transaction = await connection.BeginTransactionAsync(ct); + try + { + var outcome = await ExecuteLinkCoreAsync(connection, (SqlTransaction)transaction, LinkOperation.For(definition, leftEntityId, rightEntityId), ct); + if (outcome == OperationOutcome.Success) + { + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(connection, (SqlTransaction)transaction, outboxEvents, ct); + } + await transaction.CommitAsync(ct); + } + return outcome == OperationOutcome.AlreadyLinked ? LinkResult.AlreadyLinked : LinkResult.Success; + } + catch + { + await transaction.RollbackAsync(ct); + throw; + } + } + + /// + async Task IStore.UnlinkAsync(LinkDefinition definition, Storage.UuidV7 leftEntityId, Storage.UuidV7 rightEntityId, IReadOnlyList outboxEvents, Ct ct) + { + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + await using var transaction = await connection.BeginTransactionAsync(ct); + try + { + _ = await ExecuteUnlinkCoreAsync(connection, (SqlTransaction)transaction, UnlinkOperation.For(definition, leftEntityId, rightEntityId), ct); + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(connection, (SqlTransaction)transaction, outboxEvents, ct); + } + await transaction.CommitAsync(ct); + return UnlinkResult.Success; + } + catch + { + await transaction.RollbackAsync(ct); + throw; + } + } + + private async Task ExecuteLinkCoreAsync( + SqlConnection connection, + SqlTransaction transaction, + LinkOperation op, + Ct ct) + { + await using var cmd = connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.CommandType = CommandType.Text; + cmd.CommandText = $""" + INSERT INTO [{_schemaName}].[entity_links] (pool_id, link_type_id, left_entity_type_id, left_entity_id, right_entity_type_id, right_entity_id) + VALUES (@poolId, @linkTypeId, @leftEntityTypeId, @leftEntityId, @rightEntityTypeId, @rightEntityId) + """; + _ = cmd.Parameters.AddWithValue("@poolId", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@linkTypeId", (int)op.Definition.Link.Id); + _ = cmd.Parameters.AddWithValue("@leftEntityTypeId", (int)op.Definition.Left.Id); + _ = cmd.Parameters.AddWithValue("@leftEntityId", SqlServerGuidConverter.ToSqlServer(op.LeftEntityId.Value)); + _ = cmd.Parameters.AddWithValue("@rightEntityTypeId", (int)op.Definition.Right.Id); + _ = cmd.Parameters.AddWithValue("@rightEntityId", SqlServerGuidConverter.ToSqlServer(op.RightEntityId.Value)); + + Log.ExecutingSql(logger, cmd.CommandText); + + try + { + _ = await cmd.ExecuteNonQueryAsync(ct); + return OperationOutcome.Success; + } + catch (SqlException ex) when (ex.Number is 2627 or 2601) + { + // PK or unique constraint violation — link already exists + return OperationOutcome.AlreadyLinked; + } + } + private async Task ExecuteUnlinkCoreAsync( + SqlConnection connection, + SqlTransaction transaction, + UnlinkOperation op, + Ct ct) + { + await using var cmd = connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.CommandType = CommandType.Text; + cmd.CommandText = $""" + DELETE FROM [{_schemaName}].[entity_links] + WHERE pool_id = @poolId + AND link_type_id = @linkTypeId + AND left_entity_id = @leftEntityId + AND right_entity_id = @rightEntityId + """; + _ = cmd.Parameters.AddWithValue("@poolId", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@linkTypeId", (int)op.Definition.Link.Id); + _ = cmd.Parameters.AddWithValue("@leftEntityId", SqlServerGuidConverter.ToSqlServer(op.LeftEntityId.Value)); + _ = cmd.Parameters.AddWithValue("@rightEntityId", SqlServerGuidConverter.ToSqlServer(op.RightEntityId.Value)); + + Log.ExecutingSql(logger, cmd.CommandText); + + _ = await cmd.ExecuteNonQueryAsync(ct); + return OperationOutcome.Success; + } + + private async Task ExecuteOutboxInsertBatchCoreAsync( + SqlConnection connection, + SqlTransaction transaction, + IReadOnlyList outboxEvents, + Ct ct) + { + var rows = new List<(OutboxEvent Evt, IOutboxSubscriber Subscriber)>(); + foreach (var evt in outboxEvents) + { + foreach (var subscriber in outboxSubscribers.GetMatchingSubscribers(evt.EventName, evt.EntityTypeId)) + { + rows.Add((evt, subscriber)); + } + } + if (rows.Count == 0) + { + return; + } + + await using var cmd = connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.CommandType = CommandType.Text; + + var valueRows = new List(rows.Count); + for (var i = 0; i < rows.Count; i++) + { + var (evt, subscriber) = rows[i]; + valueRows.Add($"(@messageId{i}, @eventId{i}, @timestamp{i}, @eventName{i}, @subjectId{i}, @entityTypeId{i}, @entityTypeName{i}, @poolId, @payload{i}, @subscriberName{i})"); + _ = cmd.Parameters.AddWithValue($"@messageId{i}", Guid.CreateVersion7()); + _ = cmd.Parameters.AddWithValue($"@eventId{i}", evt.Id.Value); + _ = cmd.Parameters.AddWithValue($"@timestamp{i}", evt.Timestamp); + _ = cmd.Parameters.AddWithValue($"@eventName{i}", evt.EventName.ToString()); + _ = cmd.Parameters.AddWithValue($"@subjectId{i}", SqlServerGuidConverter.ToSqlServer(evt.SubjectId.Value)); + _ = cmd.Parameters.AddWithValue($"@entityTypeId{i}", evt.EntityTypeId); + _ = cmd.Parameters.AddWithValue($"@entityTypeName{i}", evt.EntityTypeName); + _ = cmd.Parameters.AddWithValue($"@payload{i}", evt.Payload); + _ = cmd.Parameters.AddWithValue($"@subscriberName{i}", subscriber.SubscriberName.ToString()); + } + _ = cmd.Parameters.AddWithValue("@poolId", PoolId.Value); + + cmd.CommandText = $""" + INSERT INTO [{_schemaName}].[outbox_subscriber_queue] + (message_id, event_id, timestamp, event_name, subject_id, entity_type_id, entity_type_name, pool_id, payload, subscriber_name) + VALUES + {string.Join(",\n ", valueRows)} + """; + + Log.ExecutingSql(logger, cmd.CommandText); + _ = await cmd.ExecuteNonQueryAsync(ct); + } + + async Task IStore.ExecuteBatchAsync( + IReadOnlyList operations, + IReadOnlyList outboxEvents, + Ct ct) + { + if (operations.Count == 0) + { + return new BatchResult(true, []); + } + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + await using var transaction = await connection.BeginTransactionAsync(ct); + + var results = new List(); + + try + { + for (var i = 0; i < operations.Count; i++) + { + var outcome = operations[i] switch + { + CreateOperation create => await ExecuteCreateCoreAsync(connection, (SqlTransaction)transaction, create, ct), + UpdateOperation update => await ExecuteUpdateCoreAsync(connection, (SqlTransaction)transaction, update, ct), + DeleteOperation delete => (await ExecuteDeleteCoreAsync(connection, (SqlTransaction)transaction, delete, ct)).Outcome, + LinkOperation link => await ExecuteLinkCoreAsync(connection, (SqlTransaction)transaction, link, ct), + UnlinkOperation unlink => await ExecuteUnlinkCoreAsync(connection, (SqlTransaction)transaction, unlink, ct), + _ => throw new InvalidOperationException($"Unknown operation type: {operations[i].GetType().Name}") + }; + + results.Add(new OperationResult(i, outcome)); + + if (outcome is not OperationOutcome.Success and not OperationOutcome.AlreadyLinked) + { + // Fail-fast: stop processing on first failure + // Transaction is rolled back automatically on dispose + return new BatchResult(false, results); + } + } + + // All operations succeeded — INSERT outbox events before committing + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(connection, (SqlTransaction)transaction, outboxEvents, ct); + } + + await transaction.CommitAsync(ct); + return new BatchResult(true, results); + } + catch + { + await transaction.RollbackAsync(ct); + throw; + } + } + + async Task IStore.GetOutboxEventsForSubscriberAsync(SubscriberName subscriberName, int count, Ct ct) + { + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + + await using var cmd = connection.CreateCommand(); + cmd.CommandType = CommandType.Text; + cmd.CommandText = $""" + SELECT TOP (@count) sequence_number, message_id, event_id, timestamp, event_name, subject_id, entity_type_id, entity_type_name, pool_id, payload, subscriber_name + FROM [{_schemaName}].[outbox_subscriber_queue] + WHERE subscriber_name = @subscriber_name + ORDER BY sequence_number ASC + """; + + _ = cmd.Parameters.AddWithValue("@count", count + 1); + _ = cmd.Parameters.AddWithValue("@subscriber_name", subscriberName.ToString()); + + Log.ExecutingSql(logger, cmd.CommandText); + await using var reader = await cmd.ExecuteReaderAsync(ct); + + var events = new List(); + while (await reader.ReadAsync(ct)) + { + events.Add(new PersistedOutboxEvent + { + SequenceNumber = reader.GetInt64(0), + MessageId = reader.GetGuid(1), + EventId = reader.GetGuid(2), + Timestamp = reader.GetDateTimeOffset(3), + EventName = OutboxEventName.Create(reader.GetString(4)), + SubjectId = Storage.UuidV7.From(SqlServerGuidConverter.ToUuidV7(reader.GetGuid(5))), + EntityTypeId = reader.GetInt32(6), + EntityTypeName = reader.GetString(7), + PoolId = PoolId.Load(reader.GetInt32(8)), + Payload = reader.GetString(9), + SubscriberName = SubscriberName.Create(reader.GetString(10)), + }); + } + + var hasMore = events.Count > count; + if (hasMore) + { + events.RemoveAt(events.Count - 1); + } + + return new OutboxEventsPage(events, hasMore); + } + + async Task IStore.DeleteOutboxEventsAsync(IReadOnlyList ids, Ct ct) + { + if (ids.Count == 0) + { + return; + } + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + + const int MaxBatchSize = 1000; + for (var offset = 0; offset < ids.Count; offset += MaxBatchSize) + { + var chunk = ids.Skip(offset).Take(MaxBatchSize).ToList(); + + await using var cmd = connection.CreateCommand(); + cmd.CommandType = CommandType.Text; + + var paramNames = chunk.Select((_, i) => $"@id{i}").ToList(); + cmd.CommandText = $""" + DELETE FROM [{_schemaName}].[outbox_subscriber_queue] + WHERE message_id IN ({string.Join(", ", paramNames)}) + """; + + for (var i = 0; i < chunk.Count; i++) + { + _ = cmd.Parameters.AddWithValue(paramNames[i], chunk[i].Value); + } + + Log.ExecutingSql(logger, cmd.CommandText); + _ = await cmd.ExecuteNonQueryAsync(ct); + } + } + + async Task>> IStore.QueryAsync( + EntityType entityType, + IQueryExpression filter, + SortParameter sort, + DataRange dataRange, + Ct ct) + { + if (dataRange.TokenValue is not null) + { + return await QueryCursorAsync(entityType, filter, sort, dataRange.TokenValue, ct); + } + + var (skip, take) = ResolveOffsetAndSize(dataRange); + var dsoVersion = TDso.DsoVersion; + var entityTypeId = (int)entityType.Id; + + Log.QueryingDsos(logger, entityType, skip, take); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + + await using var cmd = connection.CreateCommand(); + cmd.CommandType = CommandType.Text; + + // Build WHERE clause and ORDER BY clause + var queryClauses = BuildQueryClauses(cmd, filter, sort, skip); + + // Build main query using CTEs so total_count is correct even when page is beyond range. + // all_matches: all qualifying rows (includes sort_value when sorting); total: count of all matches; paged: the requested page. + // LEFT JOIN ensures total always returns one row even when paged is empty. + string allMatchesSelect; + string pagedOrderBy; + string outerOrderBy; + if (!sort.IsEmpty) + { + var sortColumn = GetSortColumnName(sort.Field!); + var sortDirection = sort.Direction == SortDirection.Ascending ? "ASC" : "DESC"; + allMatchesSelect = $"SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at, {sortColumn} AS sort_value"; + // MsSql doesn't support NULLS LAST — use CASE to push NULLs to end + pagedOrderBy = $"ORDER BY CASE WHEN sort_value IS NULL THEN 1 ELSE 0 END, sort_value {sortDirection}, entity_id ASC"; + outerOrderBy = $"ORDER BY CASE WHEN p.sort_value IS NULL THEN 1 ELSE 0 END, p.sort_value {sortDirection}, p.entity_id ASC"; + } + else + { + allMatchesSelect = "SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at"; + pagedOrderBy = "ORDER BY entity_id ASC"; + outerOrderBy = "ORDER BY p.entity_id ASC"; + } + + var query = $""" + WITH all_matches AS ( + {allMatchesSelect} + FROM [{_schemaName}].[entities] v + {queryClauses.JoinClause} + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({queryClauses.WhereClause}) + ), + total AS ( + SELECT COUNT(*) AS total_count FROM all_matches + ), + paged AS ( + SELECT * FROM all_matches + {pagedOrderBy} + OFFSET @offset ROWS + FETCH NEXT @limit ROWS ONLY + ) + SELECT p.entity_id, p.value, p.dso_type_schema_version, p.value_version, p.created_at, p.last_updated_at, t.total_count + FROM total t + LEFT JOIN paged p ON 1=1 + {outerOrderBy} + """; + + _ = cmd.Parameters.AddWithValue("@entity_type_id", entityTypeId); + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@offset", queryClauses.Offset); + _ = cmd.Parameters.AddWithValue("@limit", take); + + cmd.CommandText = query; + + Log.ExecutingQuery(logger, query); + + // Execute query and deserialize results. + // total_count is at column 6 (int in MsSql → reader.GetInt32). + // When page is beyond range, LEFT JOIN yields one row with p.* = NULL — skip those. + var items = new List>(); + var totalCount = 0; + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + var dsoType = dataStorageTypeRegistry.Get(dsoVersion); + while (await reader.ReadAsync(ct)) + { + if (totalCount == 0) + { + totalCount = reader.GetInt32(6); + } + + // When page is beyond range, p.entity_id is NULL — skip deserializing + if (await reader.IsDBNullAsync(0, ct)) + { + continue; + } + + var entityId = SqlServerGuidConverter.ToUuidV7(reader.GetGuid(0)); + var jsonValue = reader.GetString(1); + var valueVersion = reader.GetInt32(3); + var created = reader.GetDateTimeOffset(4); + var lastUpdated = reader.GetDateTimeOffset(5); + var item = (TDso)JsonSerializer.Deserialize(jsonValue, dsoType)!; + items.Add(new MetadataEnvelope(item, entityId, valueVersion, created, lastUpdated)); + } + } + + return new QueryResult> + { + Items = items, + TotalCount = totalCount, + TotalPages = (int)Math.Ceiling((double)totalCount / take), + HasMoreData = skip + take < totalCount + }; + } + + private async Task>> QueryCursorAsync( + EntityType entityType, + IQueryExpression filter, + SortParameter sort, + ContinuationTokenDataRange tokenRange, + Ct ct) where TDso : IDataStorageObject + { + ArgumentNullException.ThrowIfNull(sort); + if (sort.IsEmpty) + { + throw new ArgumentException("Sort parameter is required for cursor-based pagination.", nameof(sort)); + } + + var dsoVersion = TDso.DsoVersion; + var entityTypeId = (int)entityType.Id; + var pageSize = tokenRange.Size.Value; + + Log.QueryingDsos(logger, entityType, 0, pageSize); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + + await using var cmd = connection.CreateCommand(); + cmd.CommandType = CommandType.Text; + + // Build WHERE clause and ORDER BY clause for cursor-based pagination + var queryClauses = BuildCursorQueryClauses(cmd, filter, sort, tokenRange); + + // Build main query - fetch PageSize + 1 to determine if there are more pages + // We select the sort value to use in the next token + var query = $""" + SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at, {queryClauses.SortColumnName} + FROM [{_schemaName}].[entities] v + {queryClauses.JoinClause} + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({queryClauses.WhereClause}) + {queryClauses.SeekClause} + {queryClauses.OrderByClause} + OFFSET 0 ROWS + FETCH NEXT @limit ROWS ONLY + """; + + _ = cmd.Parameters.AddWithValue("@entity_type_id", entityTypeId); + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@limit", pageSize + 1); + + cmd.CommandText = query; + + Log.ExecutingQuery(logger, query); + + // Execute query and deserialize results + var items = new List<(Guid Id, TDso Item, int Version, DateTimeOffset Created, DateTimeOffset LastUpdated, object? SortValue)>(); + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + var dsoType = dataStorageTypeRegistry.Get(dsoVersion); + while (await reader.ReadAsync(ct)) + { + var entityId = SqlServerGuidConverter.ToUuidV7(reader.GetGuid(0)); + var jsonValue = reader.GetString(1); + var item = (TDso)JsonSerializer.Deserialize(jsonValue, dsoType)!; + var valueVersion = reader.GetInt32(3); + var created = reader.GetDateTimeOffset(4); + var lastUpdated = reader.GetDateTimeOffset(5); + // Get the sort value from the last column (index 6) + var sortValue = await ReadSortValueAsync(reader, sort.Field!, 6, ct); + items.Add((entityId, item, valueVersion, created, lastUpdated, sortValue)); + } + } + + // Check if there are more pages + var hasMore = items.Count > pageSize; + var pageItems = items.Take(pageSize).ToList(); + + // Generate next token if there are more pages + ContinuationToken? nextToken = null; + if (pageItems.Count > 0) + { + var lastItem = pageItems[^1]; + var token = CreateCursorToken(lastItem.Id, lastItem.SortValue); + nextToken = (ContinuationToken)token.Encode(); + } + + var resultItems = pageItems.Select(x => new MetadataEnvelope(x.Item, x.Id, x.Version, x.Created, x.LastUpdated)).ToList(); + return new QueryResult> + { + Items = resultItems, + NextToken = nextToken, + HasMoreData = hasMore + }; + } + + async Task> IStore.QueryFieldsAsync( + EntityType entityType, + IReadOnlyCollection fields, + IQueryExpression filter, + SortParameter sort, + DataRange dataRange, + Ct ct) + { + if (dataRange.TokenValue is not null) + { + return await QueryFieldsCursorAsync(entityType, fields, filter, sort, dataRange.TokenValue, ct); + } + + var (skip, take) = ResolveOffsetAndSize(dataRange); + var entityTypeId = (int)entityType.Id; + + Log.QueryingFieldsDsos(logger, entityType, fields.Count, skip, take); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + + await using var cmd = connection.CreateCommand(); + cmd.CommandType = CommandType.Text; + + // Build WHERE clause and ORDER BY clause + var queryClauses = BuildQueryClauses(cmd, filter, sort, skip); + + // Use a CTE to get filtered IDs, then join to get field values + // Use "select_field_" prefix to avoid collision with WHERE clause parameters that use "field_path_" + var fieldPaths = fields.Select(f => f.Path).ToList(); + var fieldConditions = new List(); + var paramIndex = 0; + for (var i = 0; i < fieldPaths.Count; i++) + { + if (SystemFields.IsSystemField(fieldPaths[i])) + { + continue; + } + + _ = cmd.Parameters.AddWithValue($"@select_field_{paramIndex}", DeterministicGuidGenerator.Create(fieldPaths[i].ToUpperInvariant())); + fieldConditions.Add($"field_sv.field_path = @select_field_{paramIndex}"); + paramIndex++; + } + var fieldConditionsClause = fieldConditions.Count > 0 + ? string.Join(" OR ", fieldConditions) + : "1=0"; + + // When we have sorting, we include the sort column in the CTE to enable proper ordering. + // We use ROW_NUMBER to preserve the sort order in the final results after joining with field values. + string cteSelect; + string cteJoin; + + if (!sort.IsEmpty) + { + // Determine which column to select based on field type + var sortColumn = GetSortColumnName(sort.Field!); + + // Include sort column and row number to preserve sort order + cteSelect = $"SELECT v.entity_id, v.created_at, v.last_updated_at, v.value_version, {sortColumn} AS sort_value, ROW_NUMBER() OVER ({queryClauses.OrderByClause}) AS row_num"; + cteJoin = queryClauses.JoinClause; + } + else + { + cteSelect = "SELECT v.entity_id, v.created_at, v.last_updated_at, v.value_version, ROW_NUMBER() OVER (ORDER BY v.entity_id ASC) AS row_num"; + cteJoin = ""; + } + + var query = $""" + WITH all_matches AS ( + {cteSelect} + FROM [{_schemaName}].[entities] v + {cteJoin} + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({queryClauses.WhereClause}) + ), + total AS ( + SELECT COUNT(*) AS total_count FROM all_matches + ), + filtered_ids AS ( + SELECT * FROM all_matches + ORDER BY row_num ASC + OFFSET @offset ROWS + FETCH NEXT @limit ROWS ONLY + ) + SELECT + fi.entity_id, + field_sv.field_path_text, + field_sv.string_value, + field_sv.number_value, + field_sv.datetime_value, + field_sv.boolean_value, + field_sv.guid_value, + t.total_count, + fi.created_at, + fi.last_updated_at, + fi.value_version + FROM total t + LEFT JOIN filtered_ids fi ON 1=1 + LEFT JOIN [{_schemaName}].search_values field_sv + ON fi.entity_id = field_sv.entity_id + AND field_sv.entity_type_id = @entity_type_id + AND field_sv.pool_id = @pool_id + AND field_sv.item_index = -1 + AND ({fieldConditionsClause}) + ORDER BY fi.row_num, fi.entity_id + """; + + _ = cmd.Parameters.AddWithValue("@entity_type_id", entityTypeId); + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@offset", queryClauses.Offset); + _ = cmd.Parameters.AddWithValue("@limit", take); + + cmd.CommandText = query; + + Log.ExecutingQuery(logger, query); + + // Execute query and build projected results. + // total_count is at column 7: entity_id(0), field_path(1), string_value(2), + // number_value(3), datetime_value(4), boolean_value(5), guid_value(6), total_count(7), + // created_at(8), last_updated_at(9). + // When page is beyond range, LEFT JOIN yields one row with fi.* = NULL — skip those. + var resultsByid = new Dictionary FieldValues, DateTimeOffset Created, DateTimeOffset LastUpdated, int Version)>(); + var orderedIds = new List(); + var totalCount = 0; + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + { + if (totalCount == 0) + { + totalCount = reader.GetInt32(7); + } + + // When page is beyond range, fi.entity_id is NULL — skip + if (await reader.IsDBNullAsync(0, ct)) + { + continue; + } + + var entityId = SqlServerGuidConverter.ToUuidV7(reader.GetGuid(0)); + var fieldPath = await reader.IsDBNullAsync(1, ct) ? null : reader.GetString(1); + + if (!resultsByid.TryGetValue(entityId, out var entry)) + { + var fieldValues = new Dictionary(); + orderedIds.Add(entityId); + + // Initialize all requested fields as null + foreach (var field in fields) + { + fieldValues[field.Path] = null; + } + + var created = reader.GetDateTimeOffset(8); + var lastUpdated = reader.GetDateTimeOffset(9); + var version = reader.GetInt32(10); + + // Populate system fields from entity columns + foreach (var field in fields) + { + if (string.Equals(field.Path, SystemFields.Created, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.CreatedAttributeName, StringComparison.OrdinalIgnoreCase)) + { + fieldValues[field.Path] = created; + } + else if (string.Equals(field.Path, SystemFields.LastUpdated, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.LastUpdatedAttributeName, StringComparison.OrdinalIgnoreCase)) + { + fieldValues[field.Path] = lastUpdated; + } + } + + entry = (fieldValues, created, lastUpdated, version); + resultsByid[entityId] = entry; + } + + if (fieldPath != null && entry.FieldValues.ContainsKey(fieldPath)) + { + // Find the field definition to determine which typed column to read from + var field = fields.First(f => f.Path == fieldPath); + + // Extract the value from the correct typed column based on field type + var value = await ReadFieldValueAsync(reader, field.Type, 2, ct); + entry.FieldValues[fieldPath] = value; + } + } + } + + var items = orderedIds + .Select(id => new ProjectedResult(id, resultsByid[id].FieldValues)) + .ToList(); + + return new QueryResult + { + Items = items, + TotalCount = totalCount, + TotalPages = (int)Math.Ceiling((double)totalCount / take), + HasMoreData = skip + take < totalCount + }; + } + + private async Task> QueryFieldsCursorAsync( + EntityType entityType, + IReadOnlyCollection fields, + IQueryExpression filter, + SortParameter sort, + ContinuationTokenDataRange tokenRange, + Ct ct) + { + ArgumentNullException.ThrowIfNull(sort); + if (sort.IsEmpty) + { + throw new ArgumentException("Sort parameter is required for cursor-based pagination.", nameof(sort)); + } + + var entityTypeId = (int)entityType.Id; + var pageSize = tokenRange.Size.Value; + + Log.QueryingFieldsDsos(logger, entityType, fields.Count, 0, pageSize); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + + await using var cmd = connection.CreateCommand(); + cmd.CommandType = CommandType.Text; + + // Build WHERE clause and ORDER BY clause for cursor-based pagination + var queryClauses = BuildCursorQueryClauses(cmd, filter, sort, tokenRange); + + // Use a CTE to get filtered IDs with cursor paging, then join to get field values + // Use "select_field_" prefix to avoid collision with WHERE clause parameters that use "field_path_" + var fieldPaths = fields.Select(f => f.Path).ToList(); + var fieldConditions = new List(); + var paramIndex = 0; + for (var i = 0; i < fieldPaths.Count; i++) + { + if (SystemFields.IsSystemField(fieldPaths[i])) + { + continue; + } + + _ = cmd.Parameters.AddWithValue($"@select_field_{paramIndex}", DeterministicGuidGenerator.Create(fieldPaths[i].ToUpperInvariant())); + fieldConditions.Add($"field_sv.field_path = @select_field_{paramIndex}"); + paramIndex++; + } + var fieldConditionsClause = fieldConditions.Count > 0 + ? string.Join(" OR ", fieldConditions) + : "1=0"; + + var query = $""" + WITH filtered_ids AS ( + SELECT v.entity_id, v.created_at, v.last_updated_at, v.value_version, {queryClauses.SortColumnName} AS sort_value, ROW_NUMBER() OVER ({queryClauses.OrderByClause}) AS row_num + FROM [{_schemaName}].[entities] v + {queryClauses.JoinClause} + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({queryClauses.WhereClause}) + {queryClauses.SeekClause} + {queryClauses.OrderByClause} + OFFSET 0 ROWS + FETCH NEXT @limit ROWS ONLY + ) + SELECT + fi.entity_id, + field_sv.field_path_text, + field_sv.string_value, + field_sv.number_value, + field_sv.datetime_value, + field_sv.boolean_value, + field_sv.guid_value, + fi.sort_value, + fi.created_at, + fi.last_updated_at, + fi.value_version + FROM filtered_ids fi + LEFT JOIN [{_schemaName}].search_values field_sv + ON fi.entity_id = field_sv.entity_id + AND field_sv.entity_type_id = @entity_type_id + AND field_sv.pool_id = @pool_id + AND field_sv.item_index = -1 + AND ({fieldConditionsClause}) + ORDER BY fi.row_num, fi.entity_id + """; + + _ = cmd.Parameters.AddWithValue("@entity_type_id", entityTypeId); + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@limit", pageSize + 1); + + cmd.CommandText = query; + + Log.ExecutingQuery(logger, query); + + // Execute query and build projected results + var resultsByid = new Dictionary FieldValues, DateTimeOffset Created, DateTimeOffset LastUpdated, object? SortValue, int Version)>(); + var orderedIds = new List(); + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + { + var entityId = SqlServerGuidConverter.ToUuidV7(reader.GetGuid(0)); + var fieldPath = await reader.IsDBNullAsync(1, ct) ? null : reader.GetString(1); + + if (!resultsByid.TryGetValue(entityId, out var entry)) + { + var fieldValues = new Dictionary(); + + // Initialize all requested fields as null + foreach (var field in fields) + { + fieldValues[field.Path] = null; + } + + // Get sort value from column 7, created_at from 8, last_updated_at from 9, value_version from 10 + var sortValue = await ReadSortValueAsync(reader, sort.Field!, 7, ct); + var created = reader.GetDateTimeOffset(8); + var lastUpdated = reader.GetDateTimeOffset(9); + var version = reader.GetInt32(10); + + // Populate system fields from entity columns + foreach (var field in fields) + { + if (string.Equals(field.Path, SystemFields.Created, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.CreatedAttributeName, StringComparison.OrdinalIgnoreCase)) + { + fieldValues[field.Path] = created; + } + else if (string.Equals(field.Path, SystemFields.LastUpdated, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.LastUpdatedAttributeName, StringComparison.OrdinalIgnoreCase)) + { + fieldValues[field.Path] = lastUpdated; + } + } + + entry = (fieldValues, created, lastUpdated, sortValue, version); + resultsByid[entityId] = entry; + orderedIds.Add(entityId); + } + + if (fieldPath != null && entry.FieldValues.ContainsKey(fieldPath)) + { + // Find the field definition to determine which typed column to read from + var field = fields.First(f => f.Path == fieldPath); + + // Extract the value from the correct typed column based on field type + var value = await ReadFieldValueAsync(reader, field.Type, 2, ct); + entry.FieldValues[fieldPath] = value; + } + } + } + + var itemsList = orderedIds.Select(id => (Id: id, resultsByid[id])).ToList(); + var hasMore = itemsList.Count > pageSize; + var pageItems = itemsList.Take(pageSize).ToList(); + + // Generate next token if there are more pages + ContinuationToken? nextToken = null; + if (pageItems.Count > 0) + { + var lastItem = pageItems[^1]; + var token = CreateCursorToken(lastItem.Id, lastItem.Item2.SortValue); + nextToken = (ContinuationToken)token.Encode(); + } + + var items = pageItems + .Select(item => new ProjectedResult(item.Id, item.Item2.FieldValues)) + .ToList(); + + return new QueryResult + { + Items = items, + NextToken = nextToken, + HasMoreData = hasMore + }; + } + + private async Task ExecuteCreateCoreAsync( + SqlConnection connection, + SqlTransaction transaction, + CreateOperation op, + Ct ct) + { + var dsoVersion = op.DsoVersion; + var entityType = dsoVersion.EntityType; + var jsonDso = JsonSerializer.Serialize(op.Value); + + // Resolve expiration + var expiresAt = op.Expiration.Resolve(timeProvider); + if (expiresAt.HasValue && expiresAt.Value <= timeProvider.GetUtcNow()) + { + // Already expired — noop, return success without storing + return OperationOutcome.Success; + } + + Log.CreatingDso(logger, entityType, op.Id.Value, dsoVersion.SchemaVersion); + + try + { + // Insert the main entities record + await using (var valuesCmd = connection.CreateCommand()) + { + valuesCmd.Transaction = transaction; + valuesCmd.CommandType = CommandType.Text; + valuesCmd.CommandText = $""" + INSERT INTO [{_schemaName}].[entities] ( + entity_type_id, + entity_type_name, + entity_id, + original_entity_id, + value, + dso_type_schema_version, + value_version, + created_at, + last_updated_at, + pool_id, + expires_at + ) + VALUES ( + @entityTypeId, + @entityTypeName, + @entityId, + @originalEntityId, + @value, + @dsoTypeSchemaVersion, + 1, + @now, + @now, + @poolId, + @expiresAt + ) + """; + + _ = valuesCmd.Parameters.AddWithValue("@entityTypeId", (int)entityType.Id); + _ = valuesCmd.Parameters.AddWithValue("@entityTypeName", entityType.Name); + _ = valuesCmd.Parameters.AddWithValue("@entityId", SqlServerGuidConverter.ToSqlServer(op.Id.Value)); + _ = valuesCmd.Parameters.AddWithValue("@originalEntityId", op.Id.Value); + _ = valuesCmd.Parameters.AddWithValue("@value", jsonDso); + _ = valuesCmd.Parameters.AddWithValue("@dsoTypeSchemaVersion", (int)dsoVersion.SchemaVersion); + _ = valuesCmd.Parameters.AddWithValue("@now", timeProvider.GetUtcNow()); + _ = valuesCmd.Parameters.AddWithValue("@poolId", PoolId.Value); + // expiresAt is null when Expiration.Never — DBNull.Value is needed for SQL NULL +#pragma warning disable CA1508 // Avoid dead conditional code — false positive: Expiration.Never.Resolve() returns null + _ = valuesCmd.Parameters.AddWithValue("@expiresAt", (object?)expiresAt ?? DBNull.Value); +#pragma warning restore CA1508 + + Log.ExecutingSql(logger, valuesCmd.CommandText); + _ = await valuesCmd.ExecuteNonQueryAsync(ct); + } + + // Bulk insert keys using TVP + if (op.Keys.Count > 0) + { + await using var keysCmd = connection.CreateCommand(); + keysCmd.Transaction = transaction; + keysCmd.CommandType = CommandType.Text; + keysCmd.CommandText = $""" + INSERT INTO [{_schemaName}].entity_keys ( + entity_type_id, + key_type_id, + key_type_name, + key_type_version, + key_value, + key_json, + entity_id, + pool_id + ) + SELECT + @entityTypeId, + key_type_id, + key_type_name, + key_type_version, + key_value, + key_json, + @entityId, + @poolId + FROM @keys + """; + + _ = keysCmd.Parameters.AddWithValue("@entityTypeId", (int)entityType.Id); + _ = keysCmd.Parameters.AddWithValue("@entityId", SqlServerGuidConverter.ToSqlServer(op.Id.Value)); + _ = keysCmd.Parameters.AddWithValue("@poolId", PoolId.Value); + + var keysTable = new DataTable(); + _ = keysTable.Columns.Add("key_type_id", typeof(int)); + _ = keysTable.Columns.Add("key_type_name", typeof(string)); + _ = keysTable.Columns.Add("key_type_version", typeof(int)); + _ = keysTable.Columns.Add("key_value", typeof(Guid)); + _ = keysTable.Columns.Add("key_json", typeof(string)); + + foreach (var key in op.Keys) + { + _ = keysTable.Rows.Add( + (int)key.DskVersion.KeyType.Id, + key.DskVersion.KeyType.Name, + (int)key.DskVersion.SchemaVersion, + key.Value, + (object?)key.KeyJsonValue ?? DBNull.Value + ); + } + + var keysParam = keysCmd.Parameters.AddWithValue("@keys", keysTable); + keysParam.SqlDbType = SqlDbType.Structured; + keysParam.TypeName = $"[{_schemaName}].[KeyTableType]"; + + Log.ExecutingSql(logger, keysCmd.CommandText); + _ = await keysCmd.ExecuteNonQueryAsync(ct); + } + + // Bulk insert search fields using TVP + if (op.SearchFieldCollection.Count > 0) + { + await using var searchCmd = connection.CreateCommand(); + searchCmd.Transaction = transaction; + searchCmd.CommandType = CommandType.Text; + searchCmd.CommandText = $""" + INSERT INTO [{_schemaName}].search_values ( + entity_type_id, + entity_id, + field_path, + field_path_text, + item_index, + string_value, + number_value, + datetime_value, + boolean_value, + guid_value, + pool_id + ) + SELECT + @entityTypeId, + @entityId, + field_path, + field_path_text, + item_index, + string_value, + number_value, + datetime_value, + boolean_value, + guid_value, + @poolId + FROM @searchValues + """; + + _ = searchCmd.Parameters.AddWithValue("@entityTypeId", (int)entityType.Id); + _ = searchCmd.Parameters.AddWithValue("@entityId", SqlServerGuidConverter.ToSqlServer(op.Id.Value)); + _ = searchCmd.Parameters.AddWithValue("@poolId", PoolId.Value); + + var searchTable = new DataTable(); + _ = searchTable.Columns.Add("field_path", typeof(Guid)); + _ = searchTable.Columns.Add("field_path_text", typeof(string)); + _ = searchTable.Columns.Add("item_index", typeof(int)); + _ = searchTable.Columns.Add("string_value", typeof(string)); + _ = searchTable.Columns.Add("number_value", typeof(decimal)); + _ = searchTable.Columns.Add("datetime_value", typeof(DateTimeOffset)); + _ = searchTable.Columns.Add("boolean_value", typeof(bool)); + _ = searchTable.Columns.Add("guid_value", typeof(Guid)); + + foreach (var field in op.SearchFieldCollection) + { + _ = searchTable.Rows.Add( + field.FieldPathId, + field.FieldPath, + field.ItemIndex ?? -1, + (object?)field.StringValue ?? DBNull.Value, + field.NumberValue.HasValue ? field.NumberValue.Value : DBNull.Value, + field.DateTimeValue.HasValue ? field.DateTimeValue.Value : DBNull.Value, + field.BooleanValue.HasValue ? field.BooleanValue.Value : DBNull.Value, + field.GuidValue.HasValue ? field.GuidValue.Value : DBNull.Value + ); + } + + var searchParam = searchCmd.Parameters.AddWithValue("@searchValues", searchTable); + searchParam.SqlDbType = SqlDbType.Structured; + searchParam.TypeName = $"[{_schemaName}].[SearchValueTableType]"; + + Log.ExecutingSql(logger, searchCmd.CommandText); + _ = await searchCmd.ExecuteNonQueryAsync(ct); + } + + return OperationOutcome.Success; + } + catch (SqlException ex) when (ex.Number == 2627 || ex.Number == 2601) + { + // Unique constraint violation + // Check if it's the main entities table (ID already exists) + if (ex.Message.Contains($"PK_{_schemaName}_entities", StringComparison.OrdinalIgnoreCase)) + { + return OperationOutcome.AlreadyExists; + } + + // Otherwise, it's a key conflict in the entity_keys table + if (ex.Message.Contains($"PK_{_schemaName}_entity_keys", StringComparison.OrdinalIgnoreCase)) + { + return OperationOutcome.KeyConflict; + } + + throw; + } + } + + private async Task ExecuteUpdateCoreAsync( + SqlConnection connection, + SqlTransaction transaction, + UpdateOperation op, + Ct ct) + { + var dsoVersion = op.DsoVersion; + var entityType = dsoVersion.EntityType; + var jsonDso = JsonSerializer.Serialize(op.Value); + + // Resolve expiration + DateTimeOffset? expiresAt = null; + var hasExpirationChange = op.Expiration is not null; + if (hasExpirationChange) + { + expiresAt = op.Expiration!.Resolve(timeProvider); + } + + Log.UpdatingDso(logger, entityType, op.Id.Value, dsoVersion.SchemaVersion, op.ExpectedEntityVersion); + + try + { + // Combine version check, deletes, and update into a single batch + await using (var batchCmd = connection.CreateCommand()) + { + batchCmd.Transaction = transaction; + batchCmd.CommandType = CommandType.Text; + + var expiresAtSql = hasExpirationChange + ? "expires_at = @expiresAt," + : ""; // Don't change existing expires_at when expiration is null + + batchCmd.CommandText = $""" + -- Read current version with row lock and verify + DECLARE @actualVersion INT; + SELECT @actualVersion = value_version + FROM [{_schemaName}].[entities] WITH (UPDLOCK, ROWLOCK) + WHERE entity_type_id = @entityTypeId AND entity_id = @entityId AND pool_id = @poolId; + + IF @actualVersion IS NULL + BEGIN + SELECT -1 AS Result; -- DoesNotExist + RETURN; + END + + IF @actualVersion != @expectedVersion + BEGIN + SELECT -2 AS Result; -- UnexpectedVersion + RETURN; + END + + -- Delete existing keys and search fields + DELETE FROM [{_schemaName}].entity_keys + WHERE entity_type_id = @entityTypeId AND entity_id = @entityId AND pool_id = @poolId; + + DELETE FROM [{_schemaName}].search_values + WHERE entity_type_id = @entityTypeId AND entity_id = @entityId AND pool_id = @poolId; + + -- Update the main values record + UPDATE [{_schemaName}].[entities] + SET + entity_type_name = @entityTypeName, + value = @value, + dso_type_schema_version = @dsoTypeSchemaVersion, + value_version = value_version + 1, + {expiresAtSql} + last_updated_at = @now + WHERE entity_type_id = @entityTypeId AND entity_id = @entityId AND pool_id = @poolId; + + SELECT 0 AS Result; -- Success + """; + + _ = batchCmd.Parameters.AddWithValue("@entityTypeId", (int)entityType.Id); + _ = batchCmd.Parameters.AddWithValue("@entityId", SqlServerGuidConverter.ToSqlServer(op.Id.Value)); + _ = batchCmd.Parameters.AddWithValue("@poolId", PoolId.Value); + _ = batchCmd.Parameters.AddWithValue("@expectedVersion", op.ExpectedEntityVersion); + _ = batchCmd.Parameters.AddWithValue("@entityTypeName", entityType.Name); + _ = batchCmd.Parameters.AddWithValue("@value", jsonDso); + _ = batchCmd.Parameters.AddWithValue("@dsoTypeSchemaVersion", (int)dsoVersion.SchemaVersion); + _ = batchCmd.Parameters.AddWithValue("@now", timeProvider.GetUtcNow()); + if (hasExpirationChange) + { + // expiresAt is null when Expiration.Never — DBNull.Value is needed for SQL NULL +#pragma warning disable CA1508 // Avoid dead conditional code — false positive: Expiration.Never.Resolve() returns null + _ = batchCmd.Parameters.AddWithValue("@expiresAt", (object?)expiresAt ?? DBNull.Value); +#pragma warning restore CA1508 + } + + Log.ExecutingSql(logger, batchCmd.CommandText); + var result = (int)(await batchCmd.ExecuteScalarAsync(ct))!; + + if (result == -1) + { + return OperationOutcome.DoesNotExist; + } + + if (result == -2) + { + return OperationOutcome.UnexpectedVersion; + } + } + + // Bulk insert new keys using TVP + if (op.Keys.Count > 0) + { + await using var keysCmd = connection.CreateCommand(); + keysCmd.Transaction = transaction; + keysCmd.CommandType = CommandType.Text; + keysCmd.CommandText = $""" + INSERT INTO [{_schemaName}].entity_keys ( + entity_type_id, + key_type_id, + key_type_name, + key_type_version, + key_value, + key_json, + entity_id, + pool_id + ) + SELECT + @entityTypeId, + key_type_id, + key_type_name, + key_type_version, + key_value, + key_json, + @entityId, + @poolId + FROM @keys + """; + + _ = keysCmd.Parameters.AddWithValue("@entityTypeId", (int)entityType.Id); + _ = keysCmd.Parameters.AddWithValue("@entityId", SqlServerGuidConverter.ToSqlServer(op.Id.Value)); + _ = keysCmd.Parameters.AddWithValue("@poolId", PoolId.Value); + + var keysTable = new DataTable(); + _ = keysTable.Columns.Add("key_type_id", typeof(int)); + _ = keysTable.Columns.Add("key_type_name", typeof(string)); + _ = keysTable.Columns.Add("key_type_version", typeof(int)); + _ = keysTable.Columns.Add("key_value", typeof(Guid)); + _ = keysTable.Columns.Add("key_json", typeof(string)); + + foreach (var key in op.Keys) + { + _ = keysTable.Rows.Add( + (int)key.DskVersion.KeyType.Id, + key.DskVersion.KeyType.Name, + (int)key.DskVersion.SchemaVersion, + key.Value, + (object?)key.KeyJsonValue ?? DBNull.Value + ); + } + + var keysParam = keysCmd.Parameters.AddWithValue("@keys", keysTable); + keysParam.SqlDbType = SqlDbType.Structured; + keysParam.TypeName = $"[{_schemaName}].[KeyTableType]"; + + Log.ExecutingSql(logger, keysCmd.CommandText); + _ = await keysCmd.ExecuteNonQueryAsync(ct); + } + + // Bulk insert new search fields using TVP + if (op.SearchFieldCollection.Count > 0) + { + await using var searchCmd = connection.CreateCommand(); + searchCmd.Transaction = transaction; + searchCmd.CommandType = CommandType.Text; + searchCmd.CommandText = $""" + INSERT INTO [{_schemaName}].search_values ( + entity_type_id, + entity_id, + field_path, + field_path_text, + item_index, + string_value, + number_value, + datetime_value, + boolean_value, + guid_value, + pool_id + ) + SELECT + @entityTypeId, + @entityId, + field_path, + field_path_text, + item_index, + string_value, + number_value, + datetime_value, + boolean_value, + guid_value, + @poolId + FROM @searchValues + """; + + _ = searchCmd.Parameters.AddWithValue("@entityTypeId", (int)entityType.Id); + _ = searchCmd.Parameters.AddWithValue("@entityId", SqlServerGuidConverter.ToSqlServer(op.Id.Value)); + _ = searchCmd.Parameters.AddWithValue("@poolId", PoolId.Value); + + var searchTable = new DataTable(); + _ = searchTable.Columns.Add("field_path", typeof(Guid)); + _ = searchTable.Columns.Add("field_path_text", typeof(string)); + _ = searchTable.Columns.Add("item_index", typeof(int)); + _ = searchTable.Columns.Add("string_value", typeof(string)); + _ = searchTable.Columns.Add("number_value", typeof(decimal)); + _ = searchTable.Columns.Add("datetime_value", typeof(DateTimeOffset)); + _ = searchTable.Columns.Add("boolean_value", typeof(bool)); + _ = searchTable.Columns.Add("guid_value", typeof(Guid)); + + foreach (var field in op.SearchFieldCollection) + { + _ = searchTable.Rows.Add( + field.FieldPathId, + field.FieldPath, + field.ItemIndex ?? -1, + (object?)field.StringValue ?? DBNull.Value, + field.NumberValue.HasValue ? field.NumberValue.Value : DBNull.Value, + field.DateTimeValue.HasValue ? field.DateTimeValue.Value : DBNull.Value, + field.BooleanValue.HasValue ? field.BooleanValue.Value : DBNull.Value, + field.GuidValue.HasValue ? field.GuidValue.Value : DBNull.Value + ); + } + + var searchParam = searchCmd.Parameters.AddWithValue("@searchValues", searchTable); + searchParam.SqlDbType = SqlDbType.Structured; + searchParam.TypeName = $"[{_schemaName}].[SearchValueTableType]"; + + Log.ExecutingSql(logger, searchCmd.CommandText); + _ = await searchCmd.ExecuteNonQueryAsync(ct); + } + + return OperationOutcome.Success; + } + catch (SqlException ex) when (ex.Number == 2627 || ex.Number == 2601) + { + // Key conflict during update + if (ex.Message.Contains($"PK_{_schemaName}_entity_keys", StringComparison.OrdinalIgnoreCase)) + { + return OperationOutcome.KeyConflict; + } + + throw; + } + } + + private async Task<(OperationOutcome Outcome, bool EntityDeleted)> ExecuteDeleteCoreAsync( + SqlConnection connection, + SqlTransaction transaction, + DeleteOperation op, + Ct ct) + { + var entityType = op.EntityType; + + await using var deleteCmd = connection.CreateCommand(); + deleteCmd.Transaction = transaction; + deleteCmd.CommandType = CommandType.Text; + + _ = deleteCmd.Parameters.AddWithValue("@entityTypeId", (int)entityType.Id); + _ = deleteCmd.Parameters.AddWithValue("@poolId", PoolId.Value); + + if (op.Id is not null) + { + Log.DeletingDso(logger, entityType, op.Id.Value); + + deleteCmd.CommandText = $""" + DELETE FROM [{_schemaName}].[entities] + WHERE entity_type_id = @entityTypeId AND entity_id = @entityId AND pool_id = @poolId + """; + + _ = deleteCmd.Parameters.AddWithValue("@entityId", SqlServerGuidConverter.ToSqlServer(op.Id.Value)); + } + else if (op.Key is not null) + { + var key = op.Key; + + deleteCmd.CommandText = $""" + DELETE FROM [{_schemaName}].[entities] + WHERE entity_type_id = @entityTypeId + AND pool_id = @poolId + AND entity_id = ( + SELECT entity_id + FROM [{_schemaName}].entity_keys + WHERE entity_type_id = @entityTypeId + AND key_type_id = @keyTypeId + AND key_type_version = @keyTypeVersion + AND key_value = @keyValue + AND pool_id = @poolId + ) + """; + + _ = deleteCmd.Parameters.AddWithValue("@keyTypeId", (int)key.DskVersion.KeyType.Id); + _ = deleteCmd.Parameters.AddWithValue("@keyTypeVersion", (int)key.DskVersion.SchemaVersion); + _ = deleteCmd.Parameters.AddWithValue("@keyValue", key.Value); + } + else + { + return (OperationOutcome.Success, false); + } + + Log.ExecutingSql(logger, deleteCmd.CommandText); + + // Resolve entity_id for link cleanup BEFORE deleting the entity, + // because entity_keys has ON DELETE CASCADE and will be gone after delete. + Guid? entityIdForLinks = null; + if (op.Id is not null) + { + entityIdForLinks = SqlServerGuidConverter.ToSqlServer(op.Id.Value); + } + else if (op.Key is not null) + { + entityIdForLinks = await ResolveKeyToEntityIdAsync(connection, transaction, op.EntityType, op.Key, ct); + } + + var rowsAffected = await deleteCmd.ExecuteNonQueryAsync(ct); + + // Delete entity links (no FK to entities, must be done manually) + if (entityIdForLinks.HasValue) + { + await using var linkDeleteCmd = connection.CreateCommand(); + linkDeleteCmd.Transaction = transaction; + linkDeleteCmd.CommandType = CommandType.Text; + linkDeleteCmd.CommandText = $""" + DELETE FROM [{_schemaName}].[entity_links] + WHERE pool_id = @poolId + AND (left_entity_id = @entityId OR right_entity_id = @entityId) + """; + _ = linkDeleteCmd.Parameters.AddWithValue("@poolId", PoolId.Value); + _ = linkDeleteCmd.Parameters.AddWithValue("@entityId", entityIdForLinks.Value); + Log.ExecutingSql(logger, linkDeleteCmd.CommandText); + _ = await linkDeleteCmd.ExecuteNonQueryAsync(ct); + } + + return (OperationOutcome.Success, rowsAffected > 0); + } + + private async Task ResolveKeyToEntityIdAsync( + SqlConnection connection, + SqlTransaction transaction, + EntityType entityType, + DataStorageKey key, + Ct ct) + { + await using var cmd = connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.CommandType = CommandType.Text; + cmd.CommandText = $""" + SELECT entity_id FROM [{_schemaName}].[entity_keys] + WHERE entity_type_id = @entityTypeId + AND key_type_id = @keyTypeId + AND key_type_version = @keyTypeVersion + AND key_value = @keyValue + AND pool_id = @poolId + """; + _ = cmd.Parameters.AddWithValue("@entityTypeId", (int)entityType.Id); + _ = cmd.Parameters.AddWithValue("@keyTypeId", (int)key.DskVersion.KeyType.Id); + _ = cmd.Parameters.AddWithValue("@keyTypeVersion", (int)key.DskVersion.SchemaVersion); + _ = cmd.Parameters.AddWithValue("@keyValue", key.Value); + _ = cmd.Parameters.AddWithValue("@poolId", PoolId.Value); + var result = await cmd.ExecuteScalarAsync(ct); + return result is Guid guid ? guid : null; + } + + /// + /// Builds the WHERE clause, JOIN clause, ORDER BY clause, and calculates the offset for a query. + /// + private QueryClauses BuildQueryClauses( + SqlCommand cmd, + IQueryExpression filter, + SortParameter sort, + int offset) + { + // Build WHERE clause using shared SqlWhereClauseBuilder + var whereBuilder = new SqlWhereClauseBuilder(_schemaName, cmd, Dialect); + var whereClause = whereBuilder.BuildWhereClause(filter); + + // Build JOIN clause and ORDER BY clause + string joinClause; + string orderByClause; + + if (!sort.IsEmpty) + { + var sortFieldPath = sort.Field!.Path; + + // Determine which column to sort on based on field type + var sortColumn = GetSortColumnName(sort.Field!); + + // Timestamp fields are columns on the entities table — no JOIN needed + if (SystemFields.IsSystemField(sortFieldPath)) + { + joinClause = ""; + } + else + { + // We'll use a LEFT JOIN to get the sort field value + // SQL Server doesn't support NULLS LAST syntax, so we use CASE expression + joinClause = $""" + LEFT JOIN [{_schemaName}].search_values sort_sv + ON v.entity_type_id = sort_sv.entity_type_id + AND v.entity_id = sort_sv.entity_id + AND v.pool_id = sort_sv.pool_id + AND sort_sv.field_path = @sort_field_path + AND sort_sv.item_index = -1 + """; + + _ = cmd.Parameters.AddWithValue("@sort_field_path", DeterministicGuidGenerator.Create(sortFieldPath.ToUpperInvariant())); + } + + // Use CASE expression to handle NULLS LAST behavior + if (sort.Direction == SortDirection.Ascending) + { + orderByClause = $""" + ORDER BY + CASE WHEN {sortColumn} IS NULL THEN 1 ELSE 0 END, + {sortColumn} ASC, + v.entity_id ASC + """; + } + else + { + orderByClause = $""" + ORDER BY + CASE WHEN {sortColumn} IS NULL THEN 1 ELSE 0 END, + {sortColumn} DESC, + v.entity_id ASC + """; + } + } + else + { + joinClause = ""; + orderByClause = "ORDER BY v.entity_id ASC"; + } + + + return new QueryClauses(whereClause, joinClause, orderByClause, offset); + } + + /// + /// Builds the WHERE clause, JOIN clause, ORDER BY clause, and seek clause for cursor-based pagination. + /// + private CursorQueryClauses BuildCursorQueryClauses( + SqlCommand cmd, + IQueryExpression filter, + SortParameter sort, + ContinuationTokenDataRange tokenRange) + { + // Build WHERE clause using shared SqlWhereClauseBuilder + var whereBuilder = new SqlWhereClauseBuilder(_schemaName, cmd, Dialect); + var whereClause = whereBuilder.BuildWhereClause(filter); + + var sortFieldPath = sort.Field!.Path; + + // Determine which column to sort on based on field type + var sortColumn = GetSortColumnName(sort.Field!); + + // Build JOIN clause — timestamp fields are columns on entities table, no JOIN needed + string joinClause; + if (SystemFields.IsSystemField(sortFieldPath)) + { + joinClause = ""; + } + else + { + joinClause = $""" + LEFT JOIN [{_schemaName}].search_values sort_sv + ON v.entity_type_id = sort_sv.entity_type_id + AND v.entity_id = sort_sv.entity_id + AND v.pool_id = sort_sv.pool_id + AND sort_sv.field_path = @sort_field_path + AND sort_sv.item_index = -1 + """; + + _ = cmd.Parameters.AddWithValue("@sort_field_path", DeterministicGuidGenerator.Create(sortFieldPath.ToUpperInvariant())); + } + + // Build ORDER BY clause with NULLS LAST behavior (using CASE expression) + string orderByClause; + if (sort.Direction == SortDirection.Ascending) + { + orderByClause = $""" + ORDER BY + CASE WHEN {sortColumn} IS NULL THEN 1 ELSE 0 END, + {sortColumn} ASC, + v.entity_id ASC + """; + } + else + { + orderByClause = $""" + ORDER BY + CASE WHEN {sortColumn} IS NULL THEN 1 ELSE 0 END, + {sortColumn} DESC, + v.entity_id ASC + """; + } + + // Build seek clause for cursor position (WHERE clause addition) + var seekClause = ""; + var tokenValue = tokenRange.Start.Value; + if (tokenValue != ContinuationToken.Beginning) + { + var decodedToken = CursorToken.Decode(tokenValue); + if (decodedToken != null) + { + // Use seek conditions for efficient pagination + var lastSortParam = "@last_sort_value"; + var lastIdParam = "@last_id"; + + // Add parameters based on the field type + if (decodedToken.GuidValue.HasValue) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, decodedToken.GuidValue.Value); + } + else if (decodedToken.StringValue != null) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, decodedToken.StringValue); + } + else if (decodedToken.NumberValue.HasValue) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, decodedToken.NumberValue.Value); + } + else if (decodedToken.DateTimeValue.HasValue) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, decodedToken.DateTimeValue.Value); + } + else if (decodedToken.BooleanValue.HasValue) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, decodedToken.BooleanValue.Value); + } + else + { + // NULL sort value - use DBNull + _ = cmd.Parameters.AddWithValue(lastSortParam, DBNull.Value); + } + + _ = cmd.Parameters.AddWithValue(lastIdParam, SqlServerGuidConverter.ToSqlServer(decodedToken.Id)); + + // Build the seek condition based on sort direction + // For ascending: seek rows where (sort_value > last_sort) OR (sort_value = last_sort AND id > last_id) + // For descending: seek rows where (sort_value < last_sort) OR (sort_value = last_sort AND id > last_id) + // Handle NULL values according to NULLS LAST behavior + if (sort.Direction == SortDirection.Ascending) + { + // With NULLS LAST in ascending: non-NULL values first, then NULLs + // If last value was non-NULL, include rows with greater value OR NULL values + // If last value was NULL, only include NULLs with greater ID + seekClause = $""" + AND ( + ({sortColumn} > {lastSortParam} OR ({sortColumn} = {lastSortParam} AND v.entity_id > {lastIdParam})) + OR ({sortColumn} IS NULL AND {lastSortParam} IS NOT NULL) + OR ({sortColumn} IS NULL AND {lastSortParam} IS NULL AND v.entity_id > {lastIdParam}) + ) + """; + } + else + { + // With NULLS LAST in descending: non-NULL values (descending), then NULLs + // If last value was non-NULL, include rows with lesser value OR NULL values + // If last value was NULL, only include NULLs with greater ID + seekClause = $""" + AND ( + ({sortColumn} < {lastSortParam} OR ({sortColumn} = {lastSortParam} AND v.entity_id > {lastIdParam})) + OR ({sortColumn} IS NULL AND {lastSortParam} IS NOT NULL) + OR ({sortColumn} IS NULL AND {lastSortParam} IS NULL AND v.entity_id > {lastIdParam}) + ) + """; + } + } + } + + return new CursorQueryClauses(whereClause, joinClause, orderByClause, seekClause, sortColumn); + } + + /// + /// Gets the SQL column name for sorting based on field type. + /// + private static string GetSortColumnName(Field field) + { + if (SystemFields.IsSystemField(field.Path)) + { + return field is DateTimeField + ? string.Equals(field.Path, SystemFields.Created, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.CreatedAttributeName, StringComparison.OrdinalIgnoreCase) + ? "v.created_at" + : "v.last_updated_at" + : throw new InvalidOperationException($"System field '{field.Path}' must use DateTimeField, not {field.GetType().Name}."); + } + + return field switch + { + StringField => "sort_sv.string_value", + NumberField => "sort_sv.number_value", + DateTimeField => "sort_sv.datetime_value", + BooleanField => "sort_sv.boolean_value", + GuidField or ExactMatchField => "sort_sv.guid_value", + _ => throw new InvalidOperationException($"Unsupported field type for sorting: {field.GetType().Name}") + }; + } + + /// + /// Reads a field value from the database reader. + /// The columnIndex parameter should point to the string_value column (column 2), + /// and this method will offset appropriately based on field type. + /// + private static async Task ReadFieldValueAsync(SqlDataReader reader, FieldType fieldType, int stringValueColumnIndex, Ct ct) + { + // Calculate the correct column index based on field type + // Columns are: string_value (stringValueColumnIndex), number_value (+1), datetime_value (+2), boolean_value (+3), guid_value (+4) + var columnIndex = fieldType switch + { + FieldType.String => stringValueColumnIndex, + FieldType.Number => stringValueColumnIndex + 1, + FieldType.DateTime => stringValueColumnIndex + 2, + FieldType.Boolean => stringValueColumnIndex + 3, + FieldType.Guid => stringValueColumnIndex + 4, + _ => throw new InvalidOperationException($"Unsupported field type: {fieldType}") + }; + + if (await reader.IsDBNullAsync(columnIndex, ct)) + { + return null; + } + + return fieldType switch + { + FieldType.String => reader.GetString(columnIndex), + FieldType.Number => reader.GetDecimal(columnIndex), + FieldType.DateTime => reader.GetDateTimeOffset(columnIndex), + FieldType.Boolean => reader.GetBoolean(columnIndex), + FieldType.Guid => reader.GetGuid(columnIndex), + _ => throw new InvalidOperationException($"Unsupported field type: {fieldType}") + }; + } + + /// + /// Creates a cursor token from a sort value and entity ID. + /// + private static CursorToken CreateCursorToken(Guid id, object? sortValue) => + sortValue switch + { + string s => CursorToken.Create(id, s, null, null, null, null), + decimal d => CursorToken.Create(id, null, d, null, null, null), + DateTimeOffset dto => CursorToken.Create(id, null, null, dto, null, null), + bool b => CursorToken.Create(id, null, null, null, b, null), + Guid g => CursorToken.Create(id, null, null, null, null, g), + null => CursorToken.Create(id, null, null, null, null, null), + _ => throw new InvalidOperationException($"Unsupported sort value type: {sortValue.GetType().Name}") + }; + + /// + /// Reads a sort value from a database reader for the specified field type. + /// + private static async Task ReadSortValueAsync(SqlDataReader reader, Field sortField, int columnIndex, Ct ct) + { + if (await reader.IsDBNullAsync(columnIndex, ct)) + { + return null; + } + + return sortField switch + { + StringField => reader.GetString(columnIndex), + NumberField => reader.GetDecimal(columnIndex), + DateTimeField => reader.GetDateTimeOffset(columnIndex), + BooleanField => reader.GetBoolean(columnIndex), + GuidField or ExactMatchField => reader.GetGuid(columnIndex), + _ => throw new InvalidOperationException($"Unsupported field type for sorting: {sortField.GetType().Name}") + }; + } + + private static readonly ISqlDialect Dialect = new MsSqlDialect(); + + private static (int Skip, int Take) ResolveOffsetAndSize(DataRange dataRange) + { + if (dataRange.OffsetValue is not null) + { + return ((int)dataRange.OffsetValue.Skip.Value, dataRange.OffsetValue.Take.Value); + } + + if (dataRange.PageValue is not null) + { + var page = dataRange.PageValue.Page.Value; + var size = dataRange.PageValue.PageSize.Value; + return ((page - 1) * size, size); + } + + return (0, DataRangeSize.Default.Value); + } + + /// + async Task>> IStore.QueryLinksAsync( + LinkQueryDescriptor query, + DataRange dataRange, + Ct ct) + { + if (dataRange.TokenValue is not null) + { + throw new NotSupportedException("Cursor-based pagination is not supported for link queries."); + } + + var (skip, take) = ResolveOffsetAndSize(dataRange); + var dsoVersion = TDso.DsoVersion; + var sourceEntityTypeId = (int)query.SourceEntityType.Id; + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + + await using var cmd = connection.CreateCommand(); + cmd.CommandType = CommandType.Text; + + var joinSql = new StringBuilder(); + var whereLastJoin = ""; + + for (var i = 0; i < query.Joins.Count; i++) + { + var join = query.Joins[i]; + var linkTypeParam = $"@lt{i}"; + _ = cmd.Parameters.AddWithValue(linkTypeParam, (int)join.Definition.Link.Id); + + // Which side of this link corresponds to the source (entities table / previous join)? + string sourceSide; + string filterSide; + if (join.Direction == LinkJoinDirection.LeftToRight) + { + sourceSide = "left_entity_id"; + filterSide = "right_entity_id"; + } + else + { + sourceSide = "right_entity_id"; + filterSide = "left_entity_id"; + } + + if (i == 0) + { + // First join: links entity table to first link table + _ = joinSql.AppendLine(CultureInfo.InvariantCulture, + $"JOIN [{_schemaName}].[entity_links] l0 ON l0.{sourceSide} = e.entity_id AND l0.link_type_id = {linkTypeParam} AND l0.pool_id = @pool_id"); + } + else + { + // Subsequent joins: link previous join's filter side to this join's source side + var prevJoin = query.Joins[i - 1]; + string prevFilterSide; + if (prevJoin.Direction == LinkJoinDirection.LeftToRight) + { + prevFilterSide = "right_entity_id"; + } + else + { + prevFilterSide = "left_entity_id"; + } + + _ = joinSql.AppendLine(CultureInfo.InvariantCulture, + $"JOIN [{_schemaName}].[entity_links] l{i} ON l{i}.{sourceSide} = l{i - 1}.{prevFilterSide} AND l{i}.link_type_id = {linkTypeParam} AND l{i}.pool_id = @pool_id"); + } + + if (i == query.Joins.Count - 1) + { + whereLastJoin = $"l{i}.{filterSide}"; + } + } + + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@source_entity_type_id", sourceEntityTypeId); + _ = cmd.Parameters.AddWithValue("@offset", skip); + _ = cmd.Parameters.AddWithValue("@limit", take); + + string whereClause; + if (query.WhereEntityId is not null) + { + _ = cmd.Parameters.AddWithValue("@where_entity_id", SqlServerGuidConverter.ToSqlServer(query.WhereEntityId.Value)); + whereClause = $"{whereLastJoin} = @where_entity_id"; + } + else + { + whereClause = "1=1"; + } + + var mainQuery = $""" + SELECT DISTINCT e.entity_id, e.value, e.dso_type_schema_version, e.value_version, e.created_at, e.last_updated_at + FROM [{_schemaName}].[entities] e + {joinSql} + WHERE e.entity_type_id = @source_entity_type_id + AND e.pool_id = @pool_id + AND {whereClause} + ORDER BY e.entity_id + OFFSET @offset ROWS FETCH NEXT @limit ROWS ONLY + """; + + cmd.CommandText = mainQuery; + Log.ExecutingQuery(logger, mainQuery); + + var items = new List>(); + var dsoType = dataStorageTypeRegistry.Get(dsoVersion); + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + { + var entityId = SqlServerGuidConverter.ToUuidV7(reader.GetGuid(0)); + var jsonValue = reader.GetString(1); + var valueVersion = reader.GetInt32(3); + var created = reader.GetDateTimeOffset(4); + var lastUpdated = reader.GetDateTimeOffset(5); + var item = (TDso)JsonSerializer.Deserialize(jsonValue, dsoType)!; + items.Add(new MetadataEnvelope(item, entityId, valueVersion, created, lastUpdated)); + } + } + + // Count query — reuse same join/where but count distinct entities + var countQuery = $""" + SELECT COUNT(DISTINCT e.entity_id) + FROM [{_schemaName}].[entities] e + {joinSql} + WHERE e.entity_type_id = @source_entity_type_id + AND e.pool_id = @pool_id + AND {whereClause} + """; + + await using var countCmd = connection.CreateCommand(); + countCmd.CommandType = CommandType.Text; + _ = countCmd.Parameters.AddWithValue("@source_entity_type_id", sourceEntityTypeId); + _ = countCmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + if (query.WhereEntityId is not null) + { + _ = countCmd.Parameters.AddWithValue("@where_entity_id", SqlServerGuidConverter.ToSqlServer(query.WhereEntityId.Value)); + } + + for (var i = 0; i < query.Joins.Count; i++) + { + _ = countCmd.Parameters.AddWithValue($"@lt{i}", (int)query.Joins[i].Definition.Link.Id); + } + + countCmd.CommandText = countQuery; + Log.ExecutingSql(logger, countQuery); + var totalCount = Convert.ToInt32(await countCmd.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture); + + return new QueryResult> + { + Items = items, + TotalCount = totalCount, + TotalPages = (int)Math.Ceiling((double)totalCount / take), + HasMoreData = skip + take < totalCount + }; + } + + async Task IStore.CountAsync( + EntityType entityType, + IQueryExpression? filter, + Ct ct) + { + var entityTypeId = (int)entityType.Id; + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + + await using var cmd = connection.CreateCommand(); + cmd.CommandType = CommandType.Text; + + string whereClause; + if (filter is null or AllExpression) + { + whereClause = "1=1"; + } + else + { + var whereBuilder = new SqlWhereClauseBuilder(_schemaName, cmd, Dialect); + whereClause = whereBuilder.BuildWhereClause(filter); + } + + var query = $""" + SELECT COUNT(*) + FROM [{_schemaName}].[entities] v + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({whereClause}) + """; + + _ = cmd.Parameters.AddWithValue("@entity_type_id", entityTypeId); + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + + cmd.CommandText = query; + + Log.ExecutingQuery(logger, query); + + var result = await cmd.ExecuteScalarAsync(ct); + return Convert.ToInt64(result, CultureInfo.InvariantCulture); + } + + async Task IStore.PurgeExpiredAsync(int batchSize, Ct ct) + { + ArgumentOutOfRangeException.ThrowIfLessThan(batchSize, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(batchSize, StorageConstants.TtlCleanupMaxBatchSize); + + var now = timeProvider.GetUtcNow(); + + await using var connection = OpenConnection(); + await connection.OpenAsync(ct); + await using var transaction = await connection.BeginTransactionAsync(ct); + + try + { + await using var cmd = connection.CreateCommand(); + cmd.Transaction = (SqlTransaction)transaction; + cmd.CommandType = CommandType.Text; + + var sql = new StringBuilder(); + + // Step 1: Lock expired rows into a temp table + _ = sql.AppendLine(CultureInfo.InvariantCulture, $""" + SELECT TOP (@batchSize) pool_id, entity_id, entity_type_id, entity_type_name, value, NEWID() AS event_id + INTO #_expired + FROM [{_schemaName}].[entities] WITH (UPDLOCK, ROWLOCK, READPAST) + WHERE expires_at IS NOT NULL AND expires_at <= @now; + """); + _ = cmd.Parameters.AddWithValue("@now", now); + _ = cmd.Parameters.AddWithValue("@batchSize", batchSize); + + // Step 2: Insert outbox events per matching subscriber + if (!outboxSubscribers.IsEmpty) + { + var eventName = OutboxEventName.EntityExpired; + _ = cmd.Parameters.AddWithValue("@eventName", eventName.ToString()); + + var subscriberIndex = 0; + foreach (var subscriber in outboxSubscribers.Subscribers) + { + if (subscriber.EventNames.Count > 0 && !subscriber.EventNames.Contains(eventName)) + { + continue; + } + + var subParam = $"@sub{subscriberIndex}"; + _ = cmd.Parameters.AddWithValue(subParam, subscriber.SubscriberName.ToString()); + + _ = sql.Append(CultureInfo.InvariantCulture, $""" + INSERT INTO [{_schemaName}].[outbox_subscriber_queue] + (message_id, event_id, timestamp, event_name, subject_id, entity_type_id, entity_type_name, pool_id, payload, subscriber_name) + SELECT NEWID(), event_id, @now, @eventName, entity_id, entity_type_id, entity_type_name, pool_id, value, {subParam} + FROM #_expired + """); + + if (subscriber.EntityTypeIds.Count > 0) + { + var typeIds = string.Join(", ", subscriber.EntityTypeIds.Select(id => id.ToString(CultureInfo.InvariantCulture))); + _ = sql.Append(CultureInfo.InvariantCulture, $" WHERE entity_type_id IN ({typeIds})"); + } + + _ = sql.AppendLine(";"); + subscriberIndex++; + } + } + + // Step 3: Delete entity links + _ = sql.AppendLine(CultureInfo.InvariantCulture, $""" + DELETE el FROM [{_schemaName}].[entity_links] el + INNER JOIN #_expired e ON el.pool_id = e.pool_id + AND ( + (el.left_entity_id = e.entity_id AND el.left_entity_type_id = e.entity_type_id) + OR + (el.right_entity_id = e.entity_id AND el.right_entity_type_id = e.entity_type_id) + ); + """); + + // Step 4: Delete entities, then return the count from the temp table + _ = sql.Append(CultureInfo.InvariantCulture, $""" + DELETE e FROM [{_schemaName}].[entities] e + INNER JOIN #_expired ek ON e.pool_id = ek.pool_id AND e.entity_type_id = ek.entity_type_id AND e.entity_id = ek.entity_id + WHERE e.expires_at <= @now; + SELECT COUNT(*) FROM #_expired; + """); + + cmd.CommandText = sql.ToString(); + Log.ExecutingSql(logger, cmd.CommandText); + var deleted = (int)(await cmd.ExecuteScalarAsync(ct))!; + + await transaction.CommitAsync(ct); + return deleted; + } + catch + { + await transaction.RollbackAsync(ct); + throw; + } + } + + private sealed record QueryClauses(string WhereClause, string JoinClause, string OrderByClause, int Offset); + + private sealed record CursorQueryClauses( + string WhereClause, + string JoinClause, + string OrderByClause, + string SeekClause, + string SortColumnName); + +} diff --git a/storage/src/Storage.MsSql/Internal/MsSqlStoreOptionsValidator.cs b/storage/src/Storage.MsSql/Internal/MsSqlStoreOptionsValidator.cs new file mode 100644 index 000000000..9565cb415 --- /dev/null +++ b/storage/src/Storage.MsSql/Internal/MsSqlStoreOptionsValidator.cs @@ -0,0 +1,9 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Microsoft.Extensions.Options; + +namespace Duende.Storage.MsSql.Internal; + +#pragma warning disable CA1812 // Avoid uninstantiated internal classes +public sealed class MsSqlStoreOptionsValidator(string name) : DataAnnotationValidateOptions(name); diff --git a/storage/src/Storage.MsSql/Internal/SqlServerGuidConverter.cs b/storage/src/Storage.MsSql/Internal/SqlServerGuidConverter.cs new file mode 100644 index 000000000..71055e278 --- /dev/null +++ b/storage/src/Storage.MsSql/Internal/SqlServerGuidConverter.cs @@ -0,0 +1,65 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.MsSql.Internal; + +/// +/// Converts between standard (RFC 9562) UUIDv7 byte order and a SQL Server–optimized +/// byte layout that preserves chronological sort order in UNIQUEIDENTIFIER columns. +/// +/// +/// +/// SQL Server sorts UNIQUEIDENTIFIER values using the byte-comparison semantics +/// implemented in : the last six bytes (10-15) +/// are compared first (left-to-right), then bytes 8-9, 7, 6, 3, 2, 1, 0, 5, 4. +/// UUIDv7 places the 48-bit timestamp in bytes 0-5, which SQL Server compares last. +/// This means UUIDv7 values do not sort chronologically in SQL Server indexes. +/// +/// +/// This converter swaps the byte layout so that: +/// +/// The 48-bit timestamp (originally bytes 0-5) is placed into bytes 10-15 (sorted first). +/// The version + rand_a (originally bytes 6-7) goes into bytes 8-9 (sorted second). +/// The variant + rand_b high (originally bytes 8-9) goes into bytes 6-7 (sorted third). +/// The rand_b low (originally bytes 10-15) goes into bytes 0-5 (sorted last). +/// +/// +/// +/// Since SQL Server evaluates bytes 10-15 first, placing the timestamp there ensures +/// that chronological order is preserved in clustered indexes and sort operations. +/// The conversion is its own inverse — applying it twice returns the original value. +/// +/// +internal static class SqlServerGuidConverter +{ + /// + /// Converts a standard UUIDv7 to the SQL Server–optimized byte layout. + /// + internal static Guid ToSqlServer(Guid uuidV7) + { + Span bytes = stackalloc byte[16]; + _ = uuidV7.TryWriteBytes(bytes, bigEndian: true, out _); + + Span result = stackalloc byte[16]; + + // Timestamp (bytes 0-5) → SQL Server high-priority position (bytes 10-15) + bytes[..6].CopyTo(result[10..]); + + // Version + rand_a (bytes 6-7) → bytes 8-9 + bytes[6..8].CopyTo(result[8..]); + + // Variant + rand_b high (bytes 8-9) → bytes 6-7 + bytes[8..10].CopyTo(result[6..]); + + // Rand_b low (bytes 10-15) → bytes 0-5 + bytes[10..].CopyTo(result[..6]); + + return new Guid(result, bigEndian: true); + } + + /// + /// Converts a SQL Server–optimized GUID back to the standard UUIDv7 byte layout. + /// The swap is its own inverse — the same byte permutation reverses itself. + /// + internal static Guid ToUuidV7(Guid sqlServerGuid) => ToSqlServer(sqlServerGuid); +} diff --git a/storage/src/Storage.MsSql/Migrations/V001_InitialCreate.sql b/storage/src/Storage.MsSql/Migrations/V001_InitialCreate.sql new file mode 100644 index 000000000..0b7d37736 --- /dev/null +++ b/storage/src/Storage.MsSql/Migrations/V001_InitialCreate.sql @@ -0,0 +1,226 @@ +-- Copyright (c) Duende Software. All rights reserved. +-- See LICENSE in the project root for license information. + +-- V001: Initial schema creation +-- Uses [[schemaname]] as a placeholder replaced at runtime. + +DECLARE @compatLevel INT; +SELECT @compatLevel = CAST(compatibility_level AS INT) FROM sys.databases WHERE database_id = DB_ID(); +IF @compatLevel < 140 +BEGIN + DECLARE @msg NVARCHAR(500) = CONCAT( + 'SQL Server database compatibility level ', @compatLevel, + ' is not supported. A minimum compatibility level of 140 (SQL Server 2017) is required.'); + THROW 50001, @msg, 1; +END + +-- Create schema if it doesn't exist (prerequisite for version check) +IF NOT EXISTS (SELECT 1 FROM sys.schemas WHERE name = N'[[schemaname]]') +BEGIN + EXEC('CREATE SCHEMA [[[schemaname]]]'); +END + +DECLARE @current_version INT = ISNULL(( + SELECT CAST(JSON_VALUE(CAST(value AS NVARCHAR(MAX)), '$.Version') AS INT) + FROM sys.extended_properties ep + WHERE ep.class = 3 + AND ep.name = N'SchemaVersion' + AND ep.major_id = SCHEMA_ID('[[schemaname]]') +), 0); + +IF @current_version < 1 +BEGIN + -- entities table + CREATE TABLE [[[schemaname]]].[entities] + ( + pool_id INT NOT NULL, + entity_type_id INT NOT NULL, + entity_id UNIQUEIDENTIFIER NOT NULL, + original_entity_id UNIQUEIDENTIFIER NOT NULL, + entity_type_name NVARCHAR(255) NOT NULL, + value NVARCHAR(MAX) NOT NULL, + dso_type_schema_version INT NOT NULL, + value_version INT NOT NULL, + created_at DATETIMEOFFSET NOT NULL DEFAULT SYSDATETIMEOFFSET(), + last_updated_at DATETIMEOFFSET NOT NULL DEFAULT SYSDATETIMEOFFSET(), + expires_at DATETIMEOFFSET NULL, + CONSTRAINT PK_[[schemaname]]_entities PRIMARY KEY (pool_id, entity_type_id, entity_id) + ); + + CREATE INDEX IX_[[schemaname]]_entities_expires_at + ON [[[schemaname]]].[entities] (expires_at) + WHERE expires_at IS NOT NULL; + + CREATE INDEX IX_[[schemaname]]_entities_entity_type_name + ON [[[schemaname]]].[entities] (entity_type_name); + + CREATE INDEX IX_[[schemaname]]_entities_created_at + ON [[[schemaname]]].[entities] (pool_id, entity_type_id, created_at); + + CREATE INDEX IX_[[schemaname]]_entities_last_updated_at + ON [[[schemaname]]].[entities] (pool_id, entity_type_id, last_updated_at); + + -- entity_keys table + CREATE TABLE [[[schemaname]]].[entity_keys] + ( + pool_id INT NOT NULL, + entity_type_id INT NOT NULL, + key_type_id INT NOT NULL, + entity_id UNIQUEIDENTIFIER NOT NULL, + key_type_name NVARCHAR(255) NOT NULL, + key_type_version INT NOT NULL, + key_value UNIQUEIDENTIFIER NOT NULL, + key_json NVARCHAR(MAX) NULL, + timestamp DATETIMEOFFSET NOT NULL DEFAULT SYSDATETIMEOFFSET(), + CONSTRAINT PK_[[schemaname]]_entity_keys PRIMARY KEY (pool_id, entity_type_id, key_type_id, key_type_version, key_value), + CONSTRAINT FK_[[schemaname]]_entity_keys_entities FOREIGN KEY (pool_id, entity_type_id, entity_id) + REFERENCES [[[schemaname]]].[entities] (pool_id, entity_type_id, entity_id) + ON DELETE CASCADE + ); + + CREATE INDEX IX_[[schemaname]]_entity_keys_entity_type_id_entity_id + ON [[[schemaname]]].[entity_keys] (entity_type_id, entity_id); + + -- search_values table + CREATE TABLE [[[schemaname]]].[search_values] + ( + pool_id INT NOT NULL, + entity_type_id INT NOT NULL, + entity_id UNIQUEIDENTIFIER NOT NULL, + field_path UNIQUEIDENTIFIER NOT NULL, + field_path_text NVARCHAR(500) NOT NULL, + item_index INT NOT NULL, + string_value NVARCHAR(500) NULL, + number_value DECIMAL(38,18) NULL, + datetime_value DATETIMEOFFSET NULL, + boolean_value BIT NULL, + guid_value UNIQUEIDENTIFIER NULL, + CONSTRAINT PK_[[schemaname]]_search_values PRIMARY KEY (pool_id, entity_type_id, entity_id, field_path, item_index), + CONSTRAINT FK_[[schemaname]]_search_values_entities FOREIGN KEY (pool_id, entity_type_id, entity_id) + REFERENCES [[[schemaname]]].[entities] (pool_id, entity_type_id, entity_id) + ON DELETE CASCADE + ); + + CREATE INDEX IX_[[schemaname]]_search_values_string_value + ON [[[schemaname]]].[search_values] (pool_id, entity_type_id, field_path, string_value) + WHERE string_value IS NOT NULL AND item_index = -1; + + CREATE INDEX IX_[[schemaname]]_search_values_number_value + ON [[[schemaname]]].[search_values] (pool_id, entity_type_id, field_path, number_value) + WHERE number_value IS NOT NULL AND item_index = -1; + + CREATE INDEX IX_[[schemaname]]_search_values_datetime_value + ON [[[schemaname]]].[search_values] (pool_id, entity_type_id, field_path, datetime_value) + WHERE datetime_value IS NOT NULL AND item_index = -1; + + CREATE INDEX IX_[[schemaname]]_search_values_boolean_value + ON [[[schemaname]]].[search_values] (pool_id, entity_type_id, field_path, boolean_value) + WHERE boolean_value IS NOT NULL AND item_index = -1; + + CREATE INDEX IX_[[schemaname]]_search_values_array_string_value + ON [[[schemaname]]].[search_values] (pool_id, entity_type_id, entity_id, field_path, item_index, string_value) + WHERE string_value IS NOT NULL AND item_index >= 0; + + CREATE INDEX IX_[[schemaname]]_search_values_array_number_value + ON [[[schemaname]]].[search_values] (pool_id, entity_type_id, entity_id, field_path, item_index, number_value) + WHERE number_value IS NOT NULL AND item_index >= 0; + + CREATE INDEX IX_[[schemaname]]_search_values_array_datetime_value + ON [[[schemaname]]].[search_values] (pool_id, entity_type_id, entity_id, field_path, item_index, datetime_value) + WHERE datetime_value IS NOT NULL AND item_index >= 0; + + CREATE INDEX IX_[[schemaname]]_search_values_array_boolean_value + ON [[[schemaname]]].[search_values] (pool_id, entity_type_id, entity_id, field_path, item_index, boolean_value) + WHERE boolean_value IS NOT NULL AND item_index >= 0; + + CREATE NONCLUSTERED INDEX [IX_[[schemaname]]_search_values_guid_value] + ON [[[schemaname]]].[search_values] (pool_id, entity_type_id, field_path, guid_value) + WHERE item_index = -1 AND guid_value IS NOT NULL; + + CREATE NONCLUSTERED INDEX [IX_[[schemaname]]_search_values_array_guid_value] + ON [[[schemaname]]].[search_values] (pool_id, entity_type_id, entity_id, field_path, item_index, guid_value) + WHERE item_index >= 0 AND guid_value IS NOT NULL; + + -- entity_links table + CREATE TABLE [[[schemaname]]].[entity_links] + ( + pool_id INT NOT NULL, + link_type_id INT NOT NULL, + left_entity_type_id INT NOT NULL, + left_entity_id UNIQUEIDENTIFIER NOT NULL, + right_entity_type_id INT NOT NULL, + right_entity_id UNIQUEIDENTIFIER NOT NULL, + created_at DATETIMEOFFSET NOT NULL DEFAULT SYSDATETIMEOFFSET(), + CONSTRAINT PK_[[schemaname]]_entity_links PRIMARY KEY (pool_id, link_type_id, left_entity_id, right_entity_id) + ); + + CREATE INDEX IX_[[schemaname]]_entity_links_left_entity + ON [[[schemaname]]].[entity_links] (pool_id, link_type_id, left_entity_id); + + CREATE INDEX IX_[[schemaname]]_entity_links_right_entity + ON [[[schemaname]]].[entity_links] (pool_id, link_type_id, right_entity_id); + + CREATE INDEX IX_[[schemaname]]_entity_links_left_cascade + ON [[[schemaname]]].[entity_links] (pool_id, left_entity_id); + + CREATE INDEX IX_[[schemaname]]_entity_links_right_cascade + ON [[[schemaname]]].[entity_links] (pool_id, right_entity_id); + + -- outbox_subscriber_queue table + CREATE TABLE [[[schemaname]]].[outbox_subscriber_queue] + ( + sequence_number BIGINT IDENTITY(1,1) NOT NULL, + message_id UNIQUEIDENTIFIER NOT NULL, + event_id UNIQUEIDENTIFIER NOT NULL, + timestamp DATETIMEOFFSET NOT NULL, + event_name NVARCHAR(500) NOT NULL, + subject_id UNIQUEIDENTIFIER NOT NULL, + entity_type_id INT NOT NULL, + entity_type_name NVARCHAR(255) NOT NULL, + pool_id INT NOT NULL, + payload NVARCHAR(MAX) NOT NULL, + subscriber_name NVARCHAR(255) NOT NULL, + CONSTRAINT PK_[[schemaname]]_outbox_subscriber_queue PRIMARY KEY CLUSTERED (sequence_number), + CONSTRAINT UQ_[[schemaname]]_outbox_subscriber_queue_message_id UNIQUE (message_id) + ); + + CREATE INDEX IX_[[schemaname]]_outbox_subscriber_queue_subscriber + ON [[[schemaname]]].[outbox_subscriber_queue] (subscriber_name, sequence_number); + + -- TVP types + CREATE TYPE [[[schemaname]]].[KeyTableType] AS TABLE ( + key_type_id INT NOT NULL, + key_type_name NVARCHAR(255) NOT NULL, + key_type_version INT NOT NULL, + key_value UNIQUEIDENTIFIER NOT NULL, + key_json NVARCHAR(MAX) NULL + ); + + CREATE TYPE [[[schemaname]]].[SearchValueTableType] AS TABLE ( + field_path UNIQUEIDENTIFIER NOT NULL, + field_path_text NVARCHAR(500) NOT NULL, + item_index INT NOT NULL, + string_value NVARCHAR(500) NULL, + number_value DECIMAL(38,18) NULL, + datetime_value DATETIMEOFFSET NULL, + boolean_value BIT NULL, + guid_value UNIQUEIDENTIFIER NULL + ); + + CREATE TYPE [[[schemaname]]].[EntityIdTableType] AS TABLE ( + entity_id UNIQUEIDENTIFIER NOT NULL + ); + + CREATE TYPE [[[schemaname]]].[ExpiredEntityKeyTableType] AS TABLE ( + pool_id INT NOT NULL, + entity_type_id INT NOT NULL, + entity_id UNIQUEIDENTIFIER NOT NULL + ); + + -- Version bump + EXEC sys.sp_addextendedproperty + @name = N'SchemaVersion', + @value = N'{"Version":1}', + @level0type = N'SCHEMA', + @level0name = N'[[schemaname]]'; +END diff --git a/storage/src/Storage.MsSql/MsSqlStoreOptions.cs b/storage/src/Storage.MsSql/MsSqlStoreOptions.cs new file mode 100644 index 000000000..f2006990d --- /dev/null +++ b/storage/src/Storage.MsSql/MsSqlStoreOptions.cs @@ -0,0 +1,20 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.ComponentModel.DataAnnotations; + +namespace Duende.Storage.MsSql; + +/// +/// Configuration options for the SQL Server store. +/// +public sealed class MsSqlStoreOptions +{ + /// + /// The database schema name to use for tables. + /// Default is "dbo". + /// + [Required] + [RegularExpression(@"^[a-zA-Z0-9_\-\#\@]+$", ErrorMessage = "Schema name must contain only alphanumeric characters, underscores, hyphens, hashes, and at signs.")] + public string SchemaName { get; set; } = "dbo"; +} diff --git a/storage/src/Storage.MsSql/MsSqlStoreServiceCollectionExtensions.cs b/storage/src/Storage.MsSql/MsSqlStoreServiceCollectionExtensions.cs new file mode 100644 index 000000000..e8ee0e75d --- /dev/null +++ b/storage/src/Storage.MsSql/MsSqlStoreServiceCollectionExtensions.cs @@ -0,0 +1,69 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.ComponentModel.DataAnnotations; +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.MsSql.Internal; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Duende.Storage.MsSql; + +public static class MsSqlStoreServiceCollectionExtensions +{ + extension(IStorageBuilder builder) + { + /// + /// Adds a SQL Server store with the specified service key for multi-store scenarios. + /// The caller must register a keyed with the same service key. + /// + internal IStorageBuilder AddMsSqlStore(object serviceKey, Action configure) + { + var services = builder.Services; + _ = services.AddStore(serviceKey); + _ = services.AddKeyedTransient(serviceKey, (sp, _) => + { + var createConnection = sp.GetRequiredKeyedService(serviceKey); + var outboxSubscribers = sp.GetRequiredKeyedService(serviceKey); + return BuildStore(sp, createConnection, outboxSubscribers, configure); + }); + return builder; + } + + /// + /// Adds a SQL Server store without a service key for single-store scenarios. + /// The caller must register an unkeyed . + /// + public IStorageBuilder AddMsSqlStore(Action configure) + { + var services = builder.Services; + _ = services.AddStore(); + _ = services.AddTransient(sp => + { + var createConnection = sp.GetRequiredService(); + var outboxSubscribers = sp.GetRequiredService(); + return BuildStore(sp, createConnection, outboxSubscribers, configure); + }); + return builder; + } + } + + private static MsSqlStore BuildStore( + IServiceProvider sp, + CreateSqlConnection createConnection, + OutboxSubscribers outboxSubscribers, + Action configure) + { + var options = new MsSqlStoreOptions(); + configure(options); + Validator.ValidateObject(options, new ValidationContext(options), validateAllProperties: true); + return new MsSqlStore( + createConnection, + options, + sp.GetRequiredService(), + sp.GetRequiredService(), + outboxSubscribers, + sp.GetRequiredService>()); + } +} diff --git a/storage/src/Storage.MsSql/Storage.MsSql.csproj b/storage/src/Storage.MsSql/Storage.MsSql.csproj new file mode 100644 index 000000000..1fa657645 --- /dev/null +++ b/storage/src/Storage.MsSql/Storage.MsSql.csproj @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + + + diff --git a/storage/src/Storage.PostgreSql/Internal/Log.cs b/storage/src/Storage.PostgreSql/Internal/Log.cs new file mode 100644 index 000000000..9fe9da440 --- /dev/null +++ b/storage/src/Storage.PostgreSql/Internal/Log.cs @@ -0,0 +1,82 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Microsoft.Extensions.Logging; + +namespace Duende.Storage.PostgreSql.Internal; + +internal static partial class Log +{ + [LoggerMessage(Level = LogLevel.Information, Message = $"Checking schema version")] + internal static partial void CheckingSchemaVersion(ILogger logger); + + [LoggerMessage(Level = LogLevel.Information, Message = $"Creating schema {{{Parameters.SchemaName}}}")] + internal static partial void CreatingSchema(ILogger logger, string schemaName); + + [LoggerMessage(Level = LogLevel.Information, Message = $"Migrating schema {{{Parameters.SchemaName}}}")] + internal static partial void MigratingSchema(ILogger logger, string schemaName); + + [LoggerMessage(Level = LogLevel.Information, Message = $"Executing migration step V{{{Parameters.FromVersion}}} → V{{{Parameters.ToVersion}}}")] + internal static partial void ExecutingMigrationStep(ILogger logger, int fromVersion, int toVersion); + + [LoggerMessage(Level = LogLevel.Debug, Message = $"Executing sql {{{Parameters.Sql}}}")] + internal static partial void ExecutingSql(ILogger logger, string sql); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Creating DSO: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Id)}={{{Parameters.Id}}}, {nameof(Parameters.DsoSchemaVersion)}={{{Parameters.DsoSchemaVersion}}}")] + internal static partial void CreatingDso(ILogger logger, EntityType entityType, Guid id, uint dsoSchemaVersion); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Deleting DSO: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Id)}={{{Parameters.Id}}}")] + internal static partial void DeletingDso(ILogger logger, EntityType entityType, Guid id); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Reading DSO: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Id)}={{{Parameters.Id}}}")] + internal static partial void ReadingDso(ILogger logger, EntityType entityType, Guid id); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Reading DSOs: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Count)}={{{Parameters.Count}}}")] + internal static partial void ReadingDsos(ILogger logger, EntityType entityType, int count); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Updating DSO: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Id)}={{{Parameters.Id}}}, {nameof(Parameters.DsoSchemaVersion)}={{{Parameters.DsoSchemaVersion}}}, {nameof(Parameters.ExpectedEntityVersion)}={{{Parameters.ExpectedEntityVersion}}}")] + internal static partial void UpdatingDso(ILogger logger, EntityType entityType, Guid id, uint dsoSchemaVersion, int expectedEntityVersion); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Querying DSOs: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.PageNumber)}={{{Parameters.PageNumber}}}, {nameof(Parameters.PageSize)}={{{Parameters.PageSize}}}")] + internal static partial void QueryingDsos(ILogger logger, EntityType entityType, int pageNumber, int pageSize); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Querying DSO fields: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.FieldCount)}={{{Parameters.FieldCount}}}, {nameof(Parameters.PageNumber)}={{{Parameters.PageNumber}}}, {nameof(Parameters.PageSize)}={{{Parameters.PageSize}}}")] + internal static partial void QueryingFieldsDsos(ILogger logger, EntityType entityType, int fieldCount, int pageNumber, int pageSize); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Executing query: {nameof(Parameters.Query)}={{{Parameters.Query}}}")] + internal static partial void ExecutingQuery(ILogger logger, string query); + + private static class Parameters + { + internal const string FromVersion = nameof(FromVersion); + internal const string ToVersion = nameof(ToVersion); + internal const string Count = nameof(Count); + internal const string DsoSchemaVersion = nameof(DsoSchemaVersion); + internal const string EntityType = nameof(EntityType); + internal const string ExpectedEntityVersion = nameof(ExpectedEntityVersion); + internal const string FieldCount = nameof(FieldCount); + internal const string Id = nameof(Id); + internal const string PageNumber = nameof(PageNumber); + internal const string PageSize = nameof(PageSize); + internal const string Query = nameof(Query); + internal const string SchemaName = nameof(SchemaName); + internal const string Sql = nameof(Sql); + } +} diff --git a/storage/src/Storage.PostgreSql/Internal/MigrationScriptLoader.cs b/storage/src/Storage.PostgreSql/Internal/MigrationScriptLoader.cs new file mode 100644 index 000000000..f6752329a --- /dev/null +++ b/storage/src/Storage.PostgreSql/Internal/MigrationScriptLoader.cs @@ -0,0 +1,46 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Globalization; +using System.Reflection; +using System.Text.RegularExpressions; + +namespace Duende.Storage.PostgreSql.Internal; + +internal static class MigrationScriptLoader +{ + private static readonly Regex VersionPattern = new(@"\.Migrations\.V(\d+)_", RegexOptions.Compiled); + + public static IEnumerable<(int TargetVersion, string Sql)> GetScripts( + Assembly assembly, + DatabaseSchemaVersion fromVersion, + string schemaName) + { + var assemblyName = assembly.GetName().Name; + var prefix = $"{assemblyName}.Migrations.V"; + + return assembly.GetManifestResourceNames() + .Where(name => name.StartsWith(prefix, StringComparison.Ordinal) && name.EndsWith(".sql", StringComparison.Ordinal)) + .Select(name => (Name: name, Version: ParseVersion(name))) + .Where(x => x.Version > fromVersion.Value) + .OrderBy(x => x.Version) + .Select(x => (x.Version, ApplySchema(ReadResource(assembly, x.Name), schemaName))); + } + + private static int ParseVersion(string resourceName) + { + var match = VersionPattern.Match(resourceName); + return match.Success ? int.Parse(match.Groups[1].Value, CultureInfo.InvariantCulture) : 0; + } + + private static string ReadResource(Assembly assembly, string resourceName) + { + using var stream = assembly.GetManifestResourceStream(resourceName) + ?? throw new InvalidOperationException($"Embedded resource '{resourceName}' not found."); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } + + private static string ApplySchema(string sql, string schemaName) => + sql.Replace("[[schemaname]]", schemaName, StringComparison.Ordinal); +} diff --git a/storage/src/Storage.PostgreSql/Internal/PostgreSqlDialect.cs b/storage/src/Storage.PostgreSql/Internal/PostgreSqlDialect.cs new file mode 100644 index 000000000..f4a822414 --- /dev/null +++ b/storage/src/Storage.PostgreSql/Internal/PostgreSqlDialect.cs @@ -0,0 +1,55 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Data.Common; +using Duende.Storage.Internal.Querying; +using Npgsql; +using NpgsqlTypes; + +namespace Duende.Storage.PostgreSql.Internal; + +/// +/// PostgreSQL-specific SQL dialect implementation. +/// +internal sealed class PostgreSqlDialect : ISqlDialect +{ + public string CaseInsensitiveLikeOperator => "ILIKE"; + + public string TrueLiteral => "TRUE"; + + public string FalseLiteral => "FALSE"; + + public string EscapeLikeWildcards(string value) + { + if (string.IsNullOrEmpty(value)) + { + return value; + } + + // PostgreSQL uses backslash as escape character + // Replace backslash first to avoid double-escaping + return value + .Replace("\\", "\\\\", StringComparison.OrdinalIgnoreCase) + .Replace("%", "\\%", StringComparison.OrdinalIgnoreCase) + .Replace("_", "\\_", StringComparison.OrdinalIgnoreCase); + } + + public void AddParameter(DbCommand command, string name, object value) + { + var npgsqlCommand = (NpgsqlCommand)command; + + // Handle DateTimeOffset and DateTime with explicit type + if (value is DateTime dt) + { + _ = npgsqlCommand.Parameters.AddWithValue(name, NpgsqlDbType.TimestampTz, dt); + } + else if (value is DateTimeOffset dto) + { + _ = npgsqlCommand.Parameters.AddWithValue(name, NpgsqlDbType.TimestampTz, dto.UtcDateTime); + } + else + { + _ = npgsqlCommand.Parameters.AddWithValue(name, value); + } + } +} diff --git a/storage/src/Storage.PostgreSql/Internal/PostgreSqlStore.cs b/storage/src/Storage.PostgreSql/Internal/PostgreSqlStore.cs new file mode 100644 index 000000000..6dc9ce8c0 --- /dev/null +++ b/storage/src/Storage.PostgreSql/Internal/PostgreSqlStore.cs @@ -0,0 +1,2485 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Data; +using System.Globalization; +using System.Text; +using System.Text.Json; +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Outbox; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Expressions; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Internal.Querying.SearchFields; +using Duende.Storage.Internal.Querying.Sorting; +using Duende.Storage.Pagination; +using Duende.Storage.Querying; +using Microsoft.Extensions.Logging; +using Npgsql; +using NpgsqlTypes; +using CursorToken = Duende.Storage.Internal.Querying.CursorToken; +using OutboxEventId = Duende.Storage.Internal.Outbox.OutboxEventId; +using OutboxEventName = Duende.Storage.Internal.Outbox.OutboxEventName; +using SubscriberName = Duende.Storage.Internal.Outbox.SubscriberName; + +namespace Duende.Storage.PostgreSql.Internal; + +#pragma warning disable CA1812 // Avoid uninstantiated internal classes +#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities +internal sealed class PostgreSqlStore( + NpgsqlDataSource dataSource, + PostgreSqlStoreOptions options, + DataStorageTypeRegistry dataStorageTypeRegistry, + TimeProvider timeProvider, + OutboxSubscribers outboxSubscribers, + ILogger logger) : StoreBase, IStore, IDatabaseSchema +{ + private const int RequiredSchemaVersion = 1; + private static readonly ISqlDialect Dialect = new PostgreSqlDialect(); + private readonly string _schemaName = options.SchemaName; + + async Task IDatabaseSchema.CheckVersionAsync(Ct ct) + { + Log.CheckingSchemaVersion(logger); + + await using var cmd = dataSource.CreateCommand(); + cmd.CommandType = CommandType.Text; + cmd.CommandText = $"SELECT CASE WHEN to_regnamespace('{_schemaName}') IS NOT NULL THEN obj_description(to_regnamespace('{_schemaName}')) END"; + Log.ExecutingSql(logger, cmd.CommandText); + var scalar = await cmd.ExecuteScalarAsync(ct); + + if (scalar is null or DBNull) + { + return new CheckSchemaVersionResult(0, RequiredSchemaVersion); + } + + var comment = ((string)scalar).Trim(); + + if (comment is "standard public schema" or "") + { + return new CheckSchemaVersionResult(0, RequiredSchemaVersion); + } + + try + { + var schemaComment = JsonSerializer.Deserialize(comment)!; + return new CheckSchemaVersionResult(schemaComment.Version, RequiredSchemaVersion); + } + catch (JsonException ex) + { + throw new InvalidOperationException("Invalid database schema comment", ex); + } + } + + async Task IDatabaseSchema.MigrateAsync(Ct ct) + { + Log.MigratingSchema(logger, _schemaName); + + var versionResult = await ((IDatabaseSchema)this).CheckVersionAsync(ct); + var currentVersion = new DatabaseSchemaVersion((int)versionResult.CurrentVersion); + + var scripts = MigrationScriptLoader.GetScripts(typeof(PostgreSqlStore).Assembly, currentVersion, _schemaName); + foreach (var (targetVersion, sql) in scripts) + { + Log.ExecutingMigrationStep(logger, currentVersion.Value, targetVersion); + + await using var cnn = await dataSource.OpenConnectionAsync(ct); + await using var tx = await cnn.BeginTransactionAsync(ct); + + await using var cmd = cnn.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = sql; + Log.ExecutingSql(logger, cmd.CommandText); + _ = await cmd.ExecuteNonQueryAsync(ct); + + await tx.CommitAsync(ct); + + currentVersion = new DatabaseSchemaVersion(targetVersion); + } + + var verifyResult = await ((IDatabaseSchema)this).VerifySchemaAsync(ct); + if (!verifyResult.IsValid) + { + var errors = string.Join("; ", verifyResult.Errors.Select(e => e.ErrorMessage)); + throw new InvalidOperationException($"Schema verification failed after migration: {errors}"); + } + } + + async Task IDatabaseSchema.VerifySchemaAsync(Ct ct) + { + var errors = new List(); + + // Expected tables and their columns (table -> column -> data_type) + var expectedColumns = new Dictionary>(StringComparer.OrdinalIgnoreCase) + { + ["entities"] = new(StringComparer.OrdinalIgnoreCase) + { + ["pool_id"] = "integer", + ["entity_type_id"] = "integer", + ["entity_id"] = "uuid", + ["entity_type_name"] = "text", + ["value"] = "jsonb", + ["dso_type_schema_version"] = "integer", + ["value_version"] = "integer", + ["created_at"] = "timestamp with time zone", + ["last_updated_at"] = "timestamp with time zone", + ["expires_at"] = "timestamp with time zone", + }, + ["entity_keys"] = new(StringComparer.OrdinalIgnoreCase) + { + ["pool_id"] = "integer", + ["entity_type_id"] = "integer", + ["key_type_id"] = "integer", + ["key_type_version"] = "integer", + ["key_type_name"] = "text", + ["key_value"] = "uuid", + ["key_json"] = "jsonb", + ["entity_id"] = "uuid", + ["timestamp"] = "timestamp without time zone", + }, + ["search_values"] = new(StringComparer.OrdinalIgnoreCase) + { + ["entity_type_id"] = "integer", + ["entity_id"] = "uuid", + ["field_path"] = "uuid", + ["field_path_text"] = "text", + ["item_index"] = "integer", + ["string_value"] = "text", + ["number_value"] = "numeric", + ["datetime_value"] = "timestamp with time zone", + ["boolean_value"] = "boolean", + ["guid_value"] = "uuid", + ["pool_id"] = "integer", + }, + ["entity_links"] = new(StringComparer.OrdinalIgnoreCase) + { + ["pool_id"] = "integer", + ["link_type_id"] = "integer", + ["left_entity_type_id"] = "integer", + ["left_entity_id"] = "uuid", + ["right_entity_type_id"] = "integer", + ["right_entity_id"] = "uuid", + ["created_at"] = "timestamp without time zone", + }, + ["outbox_subscriber_queue"] = new(StringComparer.OrdinalIgnoreCase) + { + ["sequence_number"] = "bigint", + ["message_id"] = "uuid", + ["event_id"] = "uuid", + ["timestamp"] = "timestamp with time zone", + ["event_name"] = "text", + ["subject_id"] = "uuid", + ["entity_type_id"] = "integer", + ["entity_type_name"] = "text", + ["pool_id"] = "integer", + ["payload"] = "jsonb", + ["subscriber_name"] = "text", + }, + }; + + // Query actual columns from information_schema + await using var colCmd = dataSource.CreateCommand( + """ + SELECT table_name, column_name, data_type + FROM information_schema.columns + WHERE table_schema = $1 + ORDER BY table_name, column_name + """); + _ = colCmd.Parameters.AddWithValue(_schemaName); + Log.ExecutingSql(logger, colCmd.CommandText); + + var actualColumns = new Dictionary>(StringComparer.OrdinalIgnoreCase); + // Key: "table|column" → data_type + var actualColumnTypes = new Dictionary(StringComparer.OrdinalIgnoreCase); + + await using (var reader = await colCmd.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + { + var tableName = reader.GetString(0); + var columnName = reader.GetString(1); + var dataType = reader.GetString(2); + + if (!actualColumns.TryGetValue(tableName, out var cols)) + { + cols = new HashSet(StringComparer.OrdinalIgnoreCase); + actualColumns[tableName] = cols; + } + + _ = cols.Add(columnName); + actualColumnTypes[$"{tableName}|{columnName}"] = dataType; + } + } + + // Check each expected table and column + foreach (var (tableName, columns) in expectedColumns) + { + if (!actualColumns.ContainsKey(tableName)) + { + errors.Add(new SchemaVerificationError( + tableName, null, + $"Table '{_schemaName}.{tableName}' is missing.", + SchemaVerificationErrorKind.MissingTable)); + continue; + } + + foreach (var (columnName, expectedType) in columns) + { + if (!actualColumns[tableName].Contains(columnName)) + { + errors.Add(new SchemaVerificationError( + tableName, columnName, + $"Column '{columnName}' is missing from table '{_schemaName}.{tableName}'.", + SchemaVerificationErrorKind.MissingColumn)); + } + else + { + var actualType = actualColumnTypes[$"{tableName}|{columnName}"]; + if (!string.Equals(actualType, expectedType, StringComparison.OrdinalIgnoreCase)) + { + errors.Add(new SchemaVerificationError( + tableName, columnName, + $"Column '{columnName}' in '{_schemaName}.{tableName}' has type '{actualType}', expected '{expectedType}'.", + SchemaVerificationErrorKind.WrongType)); + } + } + } + } + + // Check expected indexes + var expectedIndexes = new[] + { + "entities_expires_at_index", + "entities_created_at_index", + "entities_last_updated_at_index", + "entity_keys_entity_type_id_entity_id_index", + "search_values_string_value_index", + "search_values_number_value_index", + "search_values_datetime_value_index", + "search_values_boolean_value_index", + "search_values_array_string_value_index", + "search_values_array_number_value_index", + "search_values_array_datetime_value_index", + "search_values_array_boolean_value_index", + "search_values_guid_value_index", + "search_values_array_guid_value_index", + "entity_links_left_entity_index", + "entity_links_right_entity_index", + "entity_links_left_cascade_index", + "entity_links_right_cascade_index", + "outbox_subscriber_queue_subscriber_index", + }; + + await using var idxCmd = dataSource.CreateCommand( + """ + SELECT indexname + FROM pg_indexes + WHERE schemaname = $1 + """); + _ = idxCmd.Parameters.AddWithValue(_schemaName); + Log.ExecutingSql(logger, idxCmd.CommandText); + + var actualIndexes = new HashSet(StringComparer.OrdinalIgnoreCase); + await using (var reader = await idxCmd.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + { + _ = actualIndexes.Add(reader.GetString(0)); + } + } + + foreach (var indexName in expectedIndexes) + { + if (!actualIndexes.Contains(indexName)) + { + errors.Add(new SchemaVerificationError( + indexName, null, + $"Index '{indexName}' is missing from schema '{_schemaName}'.", + SchemaVerificationErrorKind.MissingIndex)); + } + } + + // Check foreign keys + var expectedForeignKeys = new[] + { + ("entity_keys", "entities"), + ("search_values", "entities"), + }; + + await using var fkCmd = dataSource.CreateCommand( + """ + SELECT tc.table_name, ccu.table_name AS foreign_table_name + FROM information_schema.table_constraints AS tc + JOIN information_schema.referential_constraints AS rc + ON tc.constraint_name = rc.constraint_name AND tc.constraint_schema = rc.constraint_schema + JOIN information_schema.constraint_column_usage AS ccu + ON ccu.constraint_name = rc.unique_constraint_name AND ccu.constraint_schema = rc.unique_constraint_schema + WHERE tc.constraint_type = 'FOREIGN KEY' + AND tc.constraint_schema = $1 + """); + _ = fkCmd.Parameters.AddWithValue(_schemaName); + Log.ExecutingSql(logger, fkCmd.CommandText); + + var actualForeignKeys = new HashSet<(string, string)>(); + await using (var reader = await fkCmd.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + { + _ = actualForeignKeys.Add((reader.GetString(0), reader.GetString(1))); + } + } + + foreach (var (fromTable, toTable) in expectedForeignKeys) + { + if (!actualForeignKeys.Contains((fromTable, toTable))) + { + errors.Add(new SchemaVerificationError( + fromTable, null, + $"Foreign key from '{_schemaName}.{fromTable}' to '{_schemaName}.{toTable}' is missing.", + SchemaVerificationErrorKind.MissingForeignKey)); + } + } + + return new SchemaVerificationResult(errors); + } + + string IDatabaseSchema.BuildMigrationScript(DatabaseSchemaVersion fromVersion) + { + var scripts = MigrationScriptLoader.GetScripts(typeof(PostgreSqlStore).Assembly, fromVersion, _schemaName); + var sb = new StringBuilder(); + foreach (var (_, sql) in scripts) + { + _ = sb.AppendLine(sql); + } + + return sb.ToString(); + } + + /// + /// Creates a new entity in the store. + /// + async Task IStore.CreateAsync( + Storage.UuidV7 id, + TDso dso, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration expiration, + IReadOnlyList outboxEvents, + Ct ct) + { + var createOp = CreateOperation.For(id, dso, keys, searchFieldCollection, expiration); + + await using var cnn = await dataSource.OpenConnectionAsync(ct); + await using var tx = await cnn.BeginTransactionAsync(ct); + + var outcome = await ExecuteCreateCoreAsync(cnn, tx, createOp, ct); + + if (outcome == OperationOutcome.Success) + { + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(cnn, tx, outboxEvents, ct); + } + await tx.CommitAsync(ct); + } + + return outcome switch + { + OperationOutcome.Success => CreateResult.Success, + OperationOutcome.AlreadyExists => CreateResult.AlreadyExists, + OperationOutcome.KeyConflict => CreateResult.KeyConflict, + _ => throw new InvalidOperationException($"Unexpected outcome from create operation: {outcome}") + }; + } + + async Task IStore.TryReadAsync( + EntityType entityType, + Storage.UuidV7 id, + Ct ct) + { + Log.ReadingDso(logger, entityType, id.Value); + + await using var cmd = dataSource.CreateCommand( + $""" + SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at + FROM {_schemaName}.entities v + WHERE v.entity_type_id = $1 AND v.entity_id = $2 AND v.pool_id = $3 + """); + + _ = cmd.Parameters.AddWithValue((int)entityType.Id); + _ = cmd.Parameters.AddWithValue(id.Value); + _ = cmd.Parameters.AddWithValue(PoolId.Value); + Log.ExecutingSql(logger, cmd.CommandText); + await using var reader = await cmd.ExecuteReaderAsync(ct); + + if (!await reader.ReadAsync(ct)) + { + return StoreGetResult.NotFound(); + } + + var entityId = reader.GetGuid(0); + var jsonValue = reader.GetString(1); + var dsoTypeVersion = reader.GetInt32(2); + var valueVersion = reader.GetInt32(3); + var created = await reader.GetFieldValueAsync(4, ct); + var lastUpdated = await reader.GetFieldValueAsync(5, ct); + + var version = new DataStorageObjectVersion(entityType, (uint)dsoTypeVersion); + var dsoType = dataStorageTypeRegistry.Get(version); + var item = (IDataStorageObject)JsonSerializer.Deserialize(jsonValue, dsoType)!; + + return StoreGetResult.IsFound(item, entityId, valueVersion, created, lastUpdated); + } + + async Task IStore.TryReadAsync( + EntityType entityType, + DataStorageKey key, + Ct ct) + { + Log.ReadingDso(logger, entityType, key.Value); + + var keyGuid = key.Value; + var keyTypeId = (int)key.DskVersion.KeyType.Id; + var keyTypeVersion = (int)key.DskVersion.SchemaVersion; + + await using var cmd = dataSource.CreateCommand( + $""" + SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at + FROM {_schemaName}.entity_keys i + INNER JOIN {_schemaName}.entities v ON i.entity_type_id = v.entity_type_id AND i.entity_id = v.entity_id + WHERE i.entity_type_id = @entity_type_id + AND i.key_type_id = @key_type_id + AND i.key_type_version = @key_type_version + AND i.key_value = @key_value + AND i.pool_id = @pool_id + AND v.pool_id = @pool_id + """); + + _ = cmd.Parameters.AddWithValue("@entity_type_id", (int)entityType.Id); + _ = cmd.Parameters.AddWithValue("@key_type_id", keyTypeId); + _ = cmd.Parameters.AddWithValue("@key_type_version", keyTypeVersion); + _ = cmd.Parameters.AddWithValue("@key_value", keyGuid); + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + + Log.ExecutingSql(logger, cmd.CommandText); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + + if (!await reader.ReadAsync(ct)) + { + return StoreGetResult.NotFound(); + } + + var entityId = reader.GetGuid(0); + var jsonValue = reader.GetString(1); + var dsoTypeVersion = reader.GetInt32(2); + var valueVersion = reader.GetInt32(3); + var created = await reader.GetFieldValueAsync(4, ct); + var lastUpdated = await reader.GetFieldValueAsync(5, ct); + + var version = new DataStorageObjectVersion(entityType, (uint)dsoTypeVersion); + var dsoType = dataStorageTypeRegistry.Get(version); + var item = (IDataStorageObject)JsonSerializer.Deserialize(jsonValue, dsoType)!; + + return StoreGetResult.IsFound(item, entityId, valueVersion, created, lastUpdated); + } + + async Task> IStore.TryReadManyAsync( + EntityType entityType, + IReadOnlySet ids, + int maximum, + Ct ct) + { + if (ids.Count > maximum) + { + throw new InvalidOperationException( + $"The number of requested IDs ({ids.Count}) exceeds the maximum allowed ({maximum})."); + } + + Log.ReadingDsos(logger, entityType, ids.Count); + + var idArray = ids.Select(id => id.Value).ToArray(); + + await using var cmd = dataSource.CreateCommand( + $""" + SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at + FROM {_schemaName}.entities v + WHERE v.entity_type_id = @entityTypeId AND v.entity_id = ANY(@ids) AND v.pool_id = @poolId + """); + + _ = cmd.Parameters.AddWithValue("@entityTypeId", (int)entityType.Id); + _ = cmd.Parameters.Add(new NpgsqlParameter("@ids", NpgsqlDbType.Array | NpgsqlDbType.Uuid) { Value = idArray }); + _ = cmd.Parameters.AddWithValue("@poolId", PoolId.Value); + + Log.ExecutingSql(logger, cmd.CommandText); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + + var results = new List(); + while (await reader.ReadAsync(ct)) + { + var entityId = reader.GetGuid(0); + var jsonValue = reader.GetString(1); + var dsoTypeVersion = reader.GetInt32(2); + var valueVersion = reader.GetInt32(3); + var created = await reader.GetFieldValueAsync(4, ct); + var lastUpdated = await reader.GetFieldValueAsync(5, ct); + + var version = new DataStorageObjectVersion(entityType, (uint)dsoTypeVersion); + var dsoType = dataStorageTypeRegistry.Get(version); + var item = (IDataStorageObject)JsonSerializer.Deserialize(jsonValue, dsoType)!; + + + results.Add(StoreGetResult.IsFound(item, entityId, valueVersion, created, lastUpdated)); + } + + return results; + } + + /// + /// Updates an existing entity in the store. + /// + async Task IStore.UpdateAsync( + Storage.UuidV7 id, + TDso dso, + int expectedEntityVersion, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration? expiration, + IReadOnlyList outboxEvents, + Ct ct) + { + var updateOp = UpdateOperation.For(id, dso, expectedEntityVersion, keys, searchFieldCollection, expiration); + + await using var cnn = await dataSource.OpenConnectionAsync(ct); + await using var tx = await cnn.BeginTransactionAsync(ct); + + var outcome = await ExecuteUpdateCoreAsync(cnn, tx, updateOp, ct); + + if (outcome == OperationOutcome.Success) + { + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(cnn, tx, outboxEvents, ct); + } + await tx.CommitAsync(ct); + } + + return outcome switch + { + OperationOutcome.Success => UpdateResult.Success, + OperationOutcome.DoesNotExist => UpdateResult.DoesNotExist, + OperationOutcome.UnexpectedVersion => UpdateResult.UnexpectedVersion, + OperationOutcome.KeyConflict => UpdateResult.KeyConflict, + _ => throw new InvalidOperationException($"Unexpected outcome from update operation: {outcome}") + }; + } + + async Task IStore.DeleteAsync(EntityType entityType, Storage.UuidV7 id, IReadOnlyList outboxEvents, Ct ct) + { + var deleteOp = DeleteOperation.ById(entityType, id); + + await using var cnn = await dataSource.OpenConnectionAsync(ct); + await using var tx = await cnn.BeginTransactionAsync(ct); + + var (_, entityDeleted) = await ExecuteDeleteCoreAsync(cnn, tx, deleteOp, ct); + + if (entityDeleted && outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(cnn, tx, outboxEvents, ct); + } + + await tx.CommitAsync(ct); + return DeleteResult.Success; + } + + async Task IStore.DeleteAsync(EntityType entityType, DataStorageKey key, IReadOnlyList outboxEvents, Ct ct) + { + var deleteOp = DeleteOperation.ByKey(entityType, key); + + await using var cnn = await dataSource.OpenConnectionAsync(ct); + await using var tx = await cnn.BeginTransactionAsync(ct); + + var (_, entityDeleted) = await ExecuteDeleteCoreAsync(cnn, tx, deleteOp, ct); + + if (entityDeleted && outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(cnn, tx, outboxEvents, ct); + } + + await tx.CommitAsync(ct); + return DeleteResult.Success; + } + + private int AddInserts(StringBuilder builder, + NpgsqlCommand cmd, + IReadOnlyCollection keys) + { + var idNumber = 0; + + foreach (var key in keys) + { + var keyTypeIdParam = $"@key_type_id{idNumber}"; + var keyTypeNameParam = $"@key_type_name{idNumber}"; + var keyTypeVersionParam = $"@key_type_version{idNumber}"; + var keyValueParam = $"@key{idNumber}"; + var keyStringParam = $"@key_json{idNumber}"; + + _ = builder.AppendLine( + CultureInfo.InvariantCulture, + $""" + INSERT INTO {_schemaName}.entity_keys (entity_type_id, key_type_id, key_type_name, key_value, key_json, key_type_version, entity_id, pool_id) + VALUES (@entity_type_id, {keyTypeIdParam}, {keyTypeNameParam}, {keyValueParam}, {keyStringParam}, {keyTypeVersionParam}, @entity_id, @pool_id); + """); + + _ = cmd.Parameters.AddWithValue(keyTypeIdParam, (int)key.DskVersion.KeyType.Id); + _ = cmd.Parameters.AddWithValue(keyTypeNameParam, key.DskVersion.KeyType.Name); + _ = cmd.Parameters.AddWithValue(keyValueParam, key.Value); + _ = cmd.Parameters.AddWithValue(keyStringParam, NpgsqlDbType.Jsonb, (object?)key.KeyJsonValue ?? DBNull.Value); + _ = cmd.Parameters.AddWithValue(keyTypeVersionParam, (int)key.DskVersion.SchemaVersion); + + ++idNumber; + } + + return idNumber; + } + + private int AddSearchFieldInserts( + StringBuilder builder, + NpgsqlCommand cmd, + SearchFieldCollection? searchFields) + { + if (searchFields is null || searchFields.Count == 0) + { + return 0; + } + + var fieldNumber = 0; + foreach (var field in searchFields) + { + var fieldPathParam = $"@field_path{fieldNumber}"; + var fieldPathTextParam = $"@field_path_text{fieldNumber}"; + var itemIndexParam = $"@item_index{fieldNumber}"; + var stringValueParam = $"@string_value{fieldNumber}"; + var numberValueParam = $"@number_value{fieldNumber}"; + var datetimeValueParam = $"@datetime_value{fieldNumber}"; + var booleanValueParam = $"@boolean_value{fieldNumber}"; + var guidValueParam = $"@guid_value{fieldNumber}"; + + _ = builder.AppendLine( + CultureInfo.InvariantCulture, + $""" + INSERT INTO {_schemaName}.search_values (entity_type_id, entity_id, field_path, field_path_text, item_index, string_value, number_value, datetime_value, boolean_value, guid_value, pool_id) + VALUES (@entity_type_id, @entity_id, {fieldPathParam}, {fieldPathTextParam}, {itemIndexParam}, {stringValueParam}, {numberValueParam}, {datetimeValueParam}, {booleanValueParam}, {guidValueParam}, @pool_id); + """); + + _ = cmd.Parameters.AddWithValue(fieldPathParam, field.FieldPathId); + _ = cmd.Parameters.AddWithValue(fieldPathTextParam, field.FieldPath); + _ = cmd.Parameters.AddWithValue(itemIndexParam, field.ItemIndex ?? -1); + _ = cmd.Parameters.AddWithValue(stringValueParam, (object?)field.StringValue ?? DBNull.Value); + _ = cmd.Parameters.AddWithValue(numberValueParam, + field.NumberValue.HasValue ? field.NumberValue.Value : DBNull.Value); + + // Convert DateTimeOffset to UTC timestamp for PostgreSQL + if (field.DateTimeValue.HasValue) + { + _ = cmd.Parameters.AddWithValue(datetimeValueParam, NpgsqlDbType.TimestampTz, + field.DateTimeValue.Value.UtcDateTime); + } + else + { + _ = cmd.Parameters.AddWithValue(datetimeValueParam, DBNull.Value); + } + + _ = cmd.Parameters.AddWithValue(booleanValueParam, + field.BooleanValue.HasValue ? field.BooleanValue.Value : DBNull.Value); + + _ = cmd.Parameters.AddWithValue(guidValueParam, + field.GuidValue.HasValue ? field.GuidValue.Value : DBNull.Value); + + ++fieldNumber; + } + + return fieldNumber; + } + + /// + /// Queries entities with the specified pagination strategy. + /// + async Task>> IStore.QueryAsync( + EntityType entityType, + IQueryExpression filter, + SortParameter sort, + DataRange dataRange, + Ct ct) + { + if (dataRange.TokenValue is not null) + { + return await QueryCursorAsync(entityType, filter, sort, dataRange.TokenValue, ct); + } + + var (skip, take) = ResolveOffsetAndSize(dataRange); + var dsoVersion = TDso.DsoVersion; + var entityTypeId = (int)entityType.Id; + + Log.QueryingDsos(logger, entityType, skip, take); + + // Build WHERE clause and ORDER BY clause + await using var cmd = dataSource.CreateCommand(); + var queryClauses = BuildQueryClauses(cmd, filter, sort); + + // Build main query using CTEs so total_count is correct even when page is beyond range. + // all_matches: all qualifying rows (includes sort_value when sorting); total: count of all matches; paged: the requested page. + // LEFT JOIN ensures total always returns one row even when paged is empty. + string allMatchesSelect; + string pagedOrderBy; + string outerOrderBy; + if (!sort.IsEmpty) + { + var sortColumn = GetSortColumnName(sort.Field!); + var sortDirection = sort.Direction == SortDirection.Ascending ? "ASC" : "DESC"; + allMatchesSelect = $"SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at, {sortColumn} AS sort_value"; + pagedOrderBy = $"ORDER BY sort_value {sortDirection} NULLS LAST, entity_id ASC"; + outerOrderBy = $"ORDER BY p.sort_value {sortDirection} NULLS LAST, p.entity_id ASC"; + } + else + { + allMatchesSelect = "SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at"; + pagedOrderBy = "ORDER BY entity_id ASC"; + outerOrderBy = "ORDER BY p.entity_id ASC"; + } + + var query = $""" + WITH all_matches AS ( + {allMatchesSelect} + FROM {_schemaName}.entities v + {queryClauses.JoinClause} + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({queryClauses.WhereClause}) + ), + total AS ( + SELECT COUNT(*) AS total_count FROM all_matches + ), + paged AS ( + SELECT * FROM all_matches + {pagedOrderBy} + OFFSET @offset LIMIT @limit + ) + SELECT p.entity_id, p.value, p.dso_type_schema_version, p.value_version, p.created_at, p.last_updated_at, t.total_count + FROM total t + LEFT JOIN paged p ON TRUE + {outerOrderBy} + """; + + _ = cmd.Parameters.AddWithValue("@entity_type_id", entityTypeId); + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@offset", skip); + _ = cmd.Parameters.AddWithValue("@limit", take); + + cmd.CommandText = query; + + Log.ExecutingQuery(logger, query); + + // Execute query and deserialize results. + // total_count is at column 6 (bigint in PG → Convert.ToInt32). + // When page is beyond range, LEFT JOIN yields one row with p.* = NULL — skip those. + var items = new List>(); + var totalCount = 0; + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + var dsoType = dataStorageTypeRegistry.Get(dsoVersion); + while (await reader.ReadAsync(ct)) + { + if (totalCount == 0) + { + totalCount = Convert.ToInt32(reader.GetInt64(6)); + } + + // When page is beyond range, p.entity_id is NULL — skip deserializing + if (await reader.IsDBNullAsync(0, ct)) + { + continue; + } + + var entityId = reader.GetGuid(0); + var jsonValue = reader.GetString(1); + var item = (TDso)JsonSerializer.Deserialize(jsonValue, dsoType)!; + var created = await reader.GetFieldValueAsync(4, ct); + var lastUpdated = await reader.GetFieldValueAsync(5, ct); + items.Add(new MetadataEnvelope(item, entityId, reader.GetInt32(3), created, lastUpdated)); + } + } + + return new QueryResult> + { + Items = items, + TotalCount = totalCount, + TotalPages = (int)Math.Ceiling((double)totalCount / take), + HasMoreData = skip + take < totalCount + }; + } + + /// + /// Queries for specific field values with the specified pagination strategy. + /// + async Task> IStore.QueryFieldsAsync( + EntityType entityType, + IReadOnlyCollection fields, + IQueryExpression filter, + SortParameter sort, + DataRange dataRange, + Ct ct) + { + if (dataRange.TokenValue is not null) + { + return await QueryFieldsCursorAsync(entityType, fields, filter, sort, dataRange.TokenValue, ct); + } + + var (skip, take) = ResolveOffsetAndSize(dataRange); + var entityTypeId = (int)entityType.Id; + + Log.QueryingFieldsDsos(logger, entityType, fields.Count, skip, take); + + // Build WHERE clause and ORDER BY clause + await using var cmd = dataSource.CreateCommand(); + var queryClauses = BuildQueryClauses(cmd, filter, sort); + + // Use a CTE to get filtered IDs, then join to get field values + // Use "select_field_" prefix to avoid collision with WHERE clause parameters that use "field_path_" + var fieldPaths = fields.Select(f => f.Path).ToList(); + var fieldConditions = new List(); + var paramIndex = 0; + for (var i = 0; i < fieldPaths.Count; i++) + { + if (SystemFields.IsSystemField(fieldPaths[i])) + { + continue; + } + + _ = cmd.Parameters.AddWithValue($"@select_field_{paramIndex}", DeterministicGuidGenerator.Create(fieldPaths[i].ToUpperInvariant())); + fieldConditions.Add($"field_sv.field_path = @select_field_{paramIndex}"); + paramIndex++; + } + var fieldConditionsClause = fieldConditions.Count > 0 + ? string.Join(" OR ", fieldConditions) + : "1=0"; + + // When we have sorting, we include the sort column in the CTE to enable proper ordering. + // We use ROW_NUMBER to preserve the sort order in the final results after joining with field values. + string cteSelect; + string cteJoin; + + if (!sort.IsEmpty) + { + // Use the JOIN and ORDER BY parts directly from queryClauses + cteJoin = queryClauses.JoinClause; + + // Determine which column to select based on field type + var sortColumn = GetSortColumnName(sort.Field!); + + // Include sort column and row number to preserve sort order + cteSelect = $"SELECT v.entity_id, v.created_at, v.last_updated_at, v.value_version, {sortColumn} AS sort_value, ROW_NUMBER() OVER ({queryClauses.OrderByClause}) AS row_num"; + } + else + { + cteSelect = "SELECT v.entity_id, v.created_at, v.last_updated_at, v.value_version, ROW_NUMBER() OVER (ORDER BY v.entity_id ASC) AS row_num"; + cteJoin = ""; + } + + var query = $""" + WITH all_matches AS ( + {cteSelect} + FROM {_schemaName}.entities v + {cteJoin} + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({queryClauses.WhereClause}) + ), + total AS ( + SELECT COUNT(*) AS total_count FROM all_matches + ), + filtered_ids AS ( + SELECT * FROM all_matches + ORDER BY row_num ASC + OFFSET @offset LIMIT @limit + ) + SELECT + fi.entity_id, + field_sv.field_path_text, + field_sv.string_value, + field_sv.number_value, + field_sv.datetime_value, + field_sv.boolean_value, + field_sv.guid_value, + t.total_count, + fi.created_at, + fi.last_updated_at, + fi.value_version + FROM total t + LEFT JOIN filtered_ids fi ON TRUE + LEFT JOIN {_schemaName}.search_values field_sv + ON fi.entity_id = field_sv.entity_id + AND field_sv.entity_type_id = @entity_type_id + AND field_sv.pool_id = @pool_id + AND field_sv.item_index = -1 + AND ({fieldConditionsClause}) + ORDER BY fi.row_num, fi.entity_id + """; + + _ = cmd.Parameters.AddWithValue("@entity_type_id", entityTypeId); + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@offset", skip); + _ = cmd.Parameters.AddWithValue("@limit", take); + + cmd.CommandText = query; + + Log.ExecutingQuery(logger, query); + + // Execute query and build projected results + var resultsById = new Dictionary FieldValues, DateTimeOffset Created, DateTimeOffset LastUpdated, int Version)>(); + var orderedIds = new List(); + var totalCount = 0; + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + { + if (totalCount == 0) + { + // total_count is at column 7: entity_id(0), field_path(1), string_value(2), + // number_value(3), datetime_value(4), boolean_value(5), guid_value(6), total_count(7) + totalCount = Convert.ToInt32(reader.GetInt64(7)); + } + + // When page is beyond range, fi.entity_id is NULL (LEFT JOIN returns no filtered rows) + if (await reader.IsDBNullAsync(0, ct)) + { + continue; + } + + var entityId = reader.GetGuid(0); + var fieldPath = await reader.IsDBNullAsync(1, ct) ? null : reader.GetString(1); + + if (!resultsById.TryGetValue(entityId, out var entry)) + { + var fieldValues = new Dictionary(); + orderedIds.Add(entityId); + + // Initialize all requested fields as null + foreach (var field in fields) + { + fieldValues[field.Path] = null; + } + + var created = await reader.GetFieldValueAsync(8, ct); + var lastUpdated = await reader.GetFieldValueAsync(9, ct); + var version = reader.GetInt32(10); + + // Populate system fields from entity columns + foreach (var field in fields) + { + if (string.Equals(field.Path, SystemFields.Created, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.CreatedAttributeName, StringComparison.OrdinalIgnoreCase)) + { + fieldValues[field.Path] = created; + } + else if (string.Equals(field.Path, SystemFields.LastUpdated, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.LastUpdatedAttributeName, StringComparison.OrdinalIgnoreCase)) + { + fieldValues[field.Path] = lastUpdated; + } + } + + entry = (fieldValues, created, lastUpdated, version); + resultsById[entityId] = entry; + } + + if (fieldPath != null && entry.FieldValues.ContainsKey(fieldPath)) + { + // Find the field definition to determine which typed column to read from + var field = fields.First(f => f.Path == fieldPath); + + // Extract the value from the correct typed column based on field type + var value = await ReadFieldValueAsync(reader, field.Type, 2, ct); + entry.FieldValues[fieldPath] = value; + } + } + } + + var items = orderedIds + .Select(id => new ProjectedResult(id, resultsById[id].FieldValues)) + .ToList(); + + return new QueryResult + { + Items = items, + TotalCount = totalCount, + TotalPages = (int)Math.Ceiling((double)totalCount / take), + HasMoreData = skip + take < totalCount + }; + } + + /// + /// Queries entities with cursor-based pagination. + /// + private async Task>> QueryCursorAsync( + EntityType entityType, + IQueryExpression filter, + SortParameter sort, + ContinuationTokenDataRange tokenRange, + Ct ct) where TDso : IDataStorageObject + { + ArgumentNullException.ThrowIfNull(sort); + if (sort.IsEmpty) + { + throw new ArgumentException("Sort parameter is required for cursor-based pagination.", nameof(sort)); + } + + var dsoVersion = TDso.DsoVersion; + var entityTypeId = (int)entityType.Id; + var pageSize = tokenRange.Size.Value; + + Log.QueryingDsos(logger, entityType, 0, pageSize); + + // Build WHERE clause and ORDER BY clause for cursor-based pagination + await using var cmd = dataSource.CreateCommand(); + var queryClauses = BuildCursorQueryClauses(cmd, entityTypeId, filter, sort, tokenRange); + + // Build main query - fetch PageSize + 1 to determine if there are more pages + // We select the sort value to use in the next token + var query = $""" + SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at, {queryClauses.SortColumnName} + FROM {_schemaName}.entities v + {queryClauses.JoinClause} + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({queryClauses.WhereClause}) + {queryClauses.SeekClause} + {queryClauses.OrderByClause} + LIMIT @limit + """; + + _ = cmd.Parameters.AddWithValue("@entity_type_id", entityTypeId); + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@limit", pageSize + 1); + + cmd.CommandText = query; + + Log.ExecutingQuery(logger, query); + + // Execute query and deserialize results + var items = new List<(Guid Id, MetadataEnvelope Item, object? SortValue)>(); + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + var dsoType = dataStorageTypeRegistry.Get(dsoVersion); + while (await reader.ReadAsync(ct)) + { + var entityId = reader.GetGuid(0); + var jsonValue = reader.GetString(1); + var item = (TDso)JsonSerializer.Deserialize(jsonValue, dsoType)!; + var created = await reader.GetFieldValueAsync(4, ct); + var lastUpdated = await reader.GetFieldValueAsync(5, ct); + var envelope = new MetadataEnvelope(item, entityId, reader.GetInt32(3), created, lastUpdated); + + // Get the sort value from column 6 + var sortValue = await ReadSortValueAsync(reader, sort.Field!, 6, ct); + items.Add((entityId, envelope, sortValue)); + } + } + + // Check if there are more pages + var hasMore = items.Count > pageSize; + var pageItems = items.Take(pageSize).ToList(); + + // Generate next token if there are more pages + ContinuationToken? nextToken = null; + if (pageItems.Count > 0) + { + var lastItem = pageItems[^1]; + var token = CreateCursorToken(lastItem.Id, lastItem.SortValue); + nextToken = (ContinuationToken)token.Encode(); + } + + var resultItems = pageItems.Select(x => x.Item).ToList(); + return new QueryResult> + { + Items = resultItems, + NextToken = nextToken, + HasMoreData = hasMore + }; + } + + /// + /// Queries for specific field values with cursor-based pagination. + /// + private async Task> QueryFieldsCursorAsync( + EntityType entityType, + IReadOnlyCollection fields, + IQueryExpression filter, + SortParameter sort, + ContinuationTokenDataRange tokenRange, + Ct ct) + { + ArgumentNullException.ThrowIfNull(sort); + if (sort.IsEmpty) + { + throw new ArgumentException("Sort parameter is required for cursor-based pagination.", nameof(sort)); + } + + var entityTypeId = (int)entityType.Id; + var pageSize = tokenRange.Size.Value; + + Log.QueryingFieldsDsos(logger, entityType, fields.Count, 0, pageSize); + + // Build WHERE clause and ORDER BY clause for cursor-based pagination + await using var cmd = dataSource.CreateCommand(); + var queryClauses = BuildCursorQueryClauses(cmd, entityTypeId, filter, sort, tokenRange); + + // Use a CTE to get filtered IDs with cursor paging, then join to get field values + // Use "select_field_" prefix to avoid collision with WHERE clause parameters that use "field_path_" + var fieldPaths = fields.Select(f => f.Path).ToList(); + var fieldConditions = new List(); + var paramIndex = 0; + for (var i = 0; i < fieldPaths.Count; i++) + { + if (SystemFields.IsSystemField(fieldPaths[i])) + { + continue; + } + + _ = cmd.Parameters.AddWithValue($"@select_field_{paramIndex}", DeterministicGuidGenerator.Create(fieldPaths[i].ToUpperInvariant())); + fieldConditions.Add($"field_sv.field_path = @select_field_{paramIndex}"); + paramIndex++; + } + var fieldConditionsClause = fieldConditions.Count > 0 + ? string.Join(" OR ", fieldConditions) + : "1=0"; + + var query = $""" + WITH filtered_ids AS ( + SELECT v.entity_id, v.created_at, v.last_updated_at, v.value_version, {queryClauses.SortColumnName} AS sort_value, ROW_NUMBER() OVER ({queryClauses.OrderByClause}) AS row_num + FROM {_schemaName}.entities v + {queryClauses.JoinClause} + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({queryClauses.WhereClause}) + {queryClauses.SeekClause} + {queryClauses.OrderByClause} + LIMIT @limit + ) + SELECT + fi.entity_id, + field_sv.field_path_text, + field_sv.string_value, + field_sv.number_value, + field_sv.datetime_value, + field_sv.boolean_value, + field_sv.guid_value, + fi.sort_value, + fi.created_at, + fi.last_updated_at, + fi.value_version + FROM filtered_ids fi + LEFT JOIN {_schemaName}.search_values field_sv + ON fi.entity_id = field_sv.entity_id + AND field_sv.entity_type_id = @entity_type_id + AND field_sv.pool_id = @pool_id + AND field_sv.item_index = -1 + AND ({fieldConditionsClause}) + ORDER BY fi.row_num, fi.entity_id + """; + + _ = cmd.Parameters.AddWithValue("@entity_type_id", entityTypeId); + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@limit", pageSize + 1); + + cmd.CommandText = query; + + Log.ExecutingQuery(logger, query); + + // Execute query and build projected results + var resultsById = new Dictionary FieldValues, object? SortValue, DateTimeOffset Created, DateTimeOffset LastUpdated, int Version)>(); + var orderedIds = new List(); + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + { + var entityId = reader.GetGuid(0); + var fieldPath = await reader.IsDBNullAsync(1, ct) ? null : reader.GetString(1); + + if (!resultsById.TryGetValue(entityId, out var entry)) + { + var fieldValues = new Dictionary(); + + // Initialize all requested fields as null + foreach (var field in fields) + { + fieldValues[field.Path] = null; + } + + var sortValue = await ReadSortValueAsync(reader, sort.Field!, 7, ct); + var created = await reader.GetFieldValueAsync(8, ct); + var lastUpdated = await reader.GetFieldValueAsync(9, ct); + var version = reader.GetInt32(10); + + // Populate system fields from entity columns + foreach (var field in fields) + { + if (string.Equals(field.Path, SystemFields.Created, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.CreatedAttributeName, StringComparison.OrdinalIgnoreCase)) + { + fieldValues[field.Path] = created; + } + else if (string.Equals(field.Path, SystemFields.LastUpdated, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.LastUpdatedAttributeName, StringComparison.OrdinalIgnoreCase)) + { + fieldValues[field.Path] = lastUpdated; + } + } + + entry = (fieldValues, sortValue, created, lastUpdated, version); + resultsById[entityId] = entry; + orderedIds.Add(entityId); + } + + if (fieldPath != null && entry.FieldValues.ContainsKey(fieldPath)) + { + // Find the field definition to determine which typed column to read from + var field = fields.First(f => f.Path == fieldPath); + + // Extract the value from the correct typed column based on field type + var value = await ReadFieldValueAsync(reader, field.Type, 2, ct); + entry.FieldValues[fieldPath] = value; + } + } + } + + var itemsList = orderedIds.Select(id => (Id: id, resultsById[id])).ToList(); + var hasMore = itemsList.Count > pageSize; + var pageItems = itemsList.Take(pageSize).ToList(); + + // Generate next token if there are more pages + ContinuationToken? nextToken = null; + if (pageItems.Count > 0) + { + var lastItem = pageItems[^1]; + var token = CreateCursorToken(lastItem.Id, lastItem.Item2.SortValue); + nextToken = (ContinuationToken)token.Encode(); + } + + var items = pageItems + .Select(item => new ProjectedResult(item.Id, item.Item2.FieldValues)) + .ToList(); + + return new QueryResult + { + Items = items, + NextToken = nextToken, + HasMoreData = hasMore + }; + } + + /// + /// Builds the WHERE clause, ORDER BY clause, and calculates the offset for a query. + /// + private QueryClauses BuildQueryClauses( + NpgsqlCommand cmd, + IQueryExpression filter, + SortParameter sort) + { + // Build WHERE clause + var whereBuilder = new SqlWhereClauseBuilder(_schemaName, cmd, Dialect); + var whereClause = whereBuilder.BuildWhereClause(filter); + + // Build JOIN and ORDER BY clauses + string joinClause; + string orderByClause; + if (!sort.IsEmpty) + { + var sortFieldPath = sort.Field!.Path; + var sortDirection = sort.Direction == SortDirection.Ascending ? "ASC" : "DESC"; + + // Determine which column to sort on based on field type + var sortColumn = GetSortColumnName(sort.Field!); + + // Timestamp fields are columns on the entities table — no JOIN needed + if (SystemFields.IsSystemField(sortFieldPath)) + { + joinClause = ""; + } + else + { + // We'll use a LEFT JOIN to get the sort field value + joinClause = $""" + LEFT JOIN {_schemaName}.search_values sort_sv + ON v.entity_type_id = sort_sv.entity_type_id + AND v.entity_id = sort_sv.entity_id + AND v.pool_id = sort_sv.pool_id + AND sort_sv.field_path = @sort_field_path + AND sort_sv.item_index = -1 + """; + + _ = cmd.Parameters.AddWithValue("@sort_field_path", DeterministicGuidGenerator.Create(sortFieldPath.ToUpperInvariant())); + } + + orderByClause = $""" + ORDER BY + {sortColumn} {sortDirection} NULLS LAST, + v.entity_id ASC + """; + } + else + { + joinClause = ""; + orderByClause = "ORDER BY v.entity_id ASC"; + } + + return new QueryClauses(whereClause, joinClause, orderByClause); + } + + /// + /// Builds the WHERE clause, JOIN clause, ORDER BY clause, and seek clause for cursor-based pagination. + /// + private CursorQueryClauses BuildCursorQueryClauses( + NpgsqlCommand cmd, + int entityTypeId, + IQueryExpression filter, + SortParameter sort, + ContinuationTokenDataRange tokenRange) + { + // Build WHERE clause + var whereBuilder = new SqlWhereClauseBuilder(_schemaName, cmd, Dialect); + var whereClause = whereBuilder.BuildWhereClause(filter); + + var sortFieldPath = sort.Field!.Path; + var sortDirection = sort.Direction == SortDirection.Ascending ? "ASC" : "DESC"; + + // Determine which column to sort on based on field type + var sortColumn = GetSortColumnName(sort.Field!); + + // Build JOIN clause — timestamp fields are columns on entities table, no JOIN needed + string joinClause; + if (SystemFields.IsSystemField(sortFieldPath)) + { + joinClause = ""; + } + else + { + joinClause = $""" + LEFT JOIN {_schemaName}.search_values sort_sv + ON v.entity_type_id = sort_sv.entity_type_id + AND v.entity_id = sort_sv.entity_id + AND v.pool_id = sort_sv.pool_id + AND sort_sv.field_path = @sort_field_path + AND sort_sv.item_index = -1 + """; + + _ = cmd.Parameters.AddWithValue("@sort_field_path", DeterministicGuidGenerator.Create(sortFieldPath.ToUpperInvariant())); + } + + // Build ORDER BY clause + var orderByClause = $""" + ORDER BY + {sortColumn} {sortDirection} NULLS LAST, + v.entity_id ASC + """; + + // Build seek clause for cursor position (WHERE clause addition) + var seekClause = ""; + var tokenValue = tokenRange.Start.Value; + if (tokenValue != ContinuationToken.Beginning) + { + var decodedToken = CursorToken.Decode(tokenValue); + if (decodedToken != null) + { + // Use row value comparison for efficient seeking + // Format: (sort_value, id) > (@last_sort, @last_id) + // This ensures we continue from the exact position + var lastSortParam = "@last_sort_value"; + var lastIdParam = "@last_id"; + + // Add parameters based on the field type + if (decodedToken.GuidValue.HasValue) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, decodedToken.GuidValue.Value); + } + else if (decodedToken.StringValue != null) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, decodedToken.StringValue); + } + else if (decodedToken.NumberValue.HasValue) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, decodedToken.NumberValue.Value); + } + else if (decodedToken.DateTimeValue.HasValue) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, NpgsqlDbType.TimestampTz, decodedToken.DateTimeValue.Value.UtcDateTime); + } + else if (decodedToken.BooleanValue.HasValue) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, decodedToken.BooleanValue.Value); + } + else + { + // NULL sort value - use a sentinel value + _ = cmd.Parameters.AddWithValue(lastSortParam, DBNull.Value); + } + + _ = cmd.Parameters.AddWithValue(lastIdParam, decodedToken.Id); + + // Build the seek condition based on sort direction + // For ascending: (sort_value, id) > (last_sort, last_id) + // For descending: (sort_value, id) < (last_sort, last_id) OR (sort_value IS NULL AND id > last_id) + if (sort.Direction == SortDirection.Ascending) + { + // Handle NULL values in sort column + seekClause = $""" + AND ( + ({sortColumn} > {lastSortParam} OR ({sortColumn} = {lastSortParam} AND v.entity_id > {lastIdParam})) + OR ({sortColumn} IS NULL AND {lastSortParam} IS NOT NULL) + ) + """; + } + else + { + seekClause = $""" + AND ( + ({sortColumn} < {lastSortParam} OR ({sortColumn} = {lastSortParam} AND v.entity_id > {lastIdParam})) + OR ({sortColumn} IS NOT NULL AND {lastSortParam} IS NULL) + ) + """; + } + } + } + + return new CursorQueryClauses(whereClause, joinClause, orderByClause, seekClause, sortColumn); + } + + /// + /// Creates a cursor token from a sort value and entity ID. + /// + private static CursorToken CreateCursorToken(Guid id, object? sortValue) => + sortValue switch + { + string s => CursorToken.Create(id, s, null, null, null, null), + decimal d => CursorToken.Create(id, null, d, null, null, null), + DateTime dt => CursorToken.Create(id, null, null, new DateTimeOffset(dt, TimeSpan.Zero), null, null), + DateTimeOffset dto => CursorToken.Create(id, null, null, dto, null, null), + bool b => CursorToken.Create(id, null, null, null, b, null), + Guid g => CursorToken.Create(id, null, null, null, null, g), + null => CursorToken.Create(id, null, null, null, null, null), + _ => throw new InvalidOperationException($"Unsupported sort value type: {sortValue.GetType().Name}") + }; + + private static (int Skip, int Take) ResolveOffsetAndSize(DataRange dataRange) + { + if (dataRange.OffsetValue is not null) + { + return ((int)dataRange.OffsetValue.Skip.Value, dataRange.OffsetValue.Take.Value); + } + + if (dataRange.PageValue is not null) + { + var page = dataRange.PageValue.Page.Value; + var size = dataRange.PageValue.PageSize.Value; + return ((page - 1) * size, size); + } + + // Fallback — should not happen since TokenValue is checked before calling this + return (0, DataRangeSize.Default.Value); + } + + /// + /// Gets the SQL column name for sorting based on field type. + /// + private static string GetSortColumnName(Field field) + { + if (SystemFields.IsSystemField(field.Path)) + { + return field is DateTimeField + ? string.Equals(field.Path, SystemFields.Created, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.CreatedAttributeName, StringComparison.OrdinalIgnoreCase) + ? "v.created_at" + : "v.last_updated_at" + : throw new InvalidOperationException($"System field '{field.Path}' must use DateTimeField, not {field.GetType().Name}."); + } + + return field switch + { + StringField => "sort_sv.string_value", + NumberField => "sort_sv.number_value", + DateTimeField => "sort_sv.datetime_value", + BooleanField => "sort_sv.boolean_value", + GuidField or ExactMatchField => "sort_sv.guid_value", + _ => throw new InvalidOperationException($"Unsupported field type for sorting: {field.GetType().Name}") + }; + } + + /// + /// Reads a field value from the database reader. + /// The columnIndex parameter should point to the string_value column (column 2), + /// and this method will offset appropriately based on field type. + /// + private static async Task ReadFieldValueAsync(NpgsqlDataReader reader, FieldType fieldType, int stringValueColumnIndex, Ct ct) + { + // Calculate the correct column index based on field type + // Columns are: string_value (stringValueColumnIndex), number_value (+1), datetime_value (+2), boolean_value (+3), guid_value (+4) + var columnIndex = fieldType switch + { + FieldType.String => stringValueColumnIndex, + FieldType.Number => stringValueColumnIndex + 1, + FieldType.DateTime => stringValueColumnIndex + 2, + FieldType.Boolean => stringValueColumnIndex + 3, + FieldType.Guid => stringValueColumnIndex + 4, + _ => throw new InvalidOperationException($"Unsupported field type: {fieldType}") + }; + + if (await reader.IsDBNullAsync(columnIndex, ct)) + { + return null; + } + +#pragma warning disable CA1849 // GetFieldValue cannot be awaited inside a switch expression + return fieldType switch + { + FieldType.String => reader.GetString(columnIndex), + FieldType.Number => reader.GetDecimal(columnIndex), + FieldType.DateTime => reader.GetFieldValue(columnIndex), + FieldType.Boolean => reader.GetBoolean(columnIndex), + FieldType.Guid => reader.GetGuid(columnIndex), + _ => throw new InvalidOperationException($"Unsupported field type: {fieldType}") + }; +#pragma warning restore CA1849 + } + + /// + /// Reads a sort value from a database reader for the specified field type. + /// + private static async Task ReadSortValueAsync(NpgsqlDataReader reader, Field sortField, int columnIndex, Ct ct) + { + if (await reader.IsDBNullAsync(columnIndex, ct)) + { + return null; + } + +#pragma warning disable CA1849 // GetFieldValue cannot be awaited inside a switch expression + return sortField switch + { + StringField => reader.GetString(columnIndex), + NumberField => reader.GetDecimal(columnIndex), + DateTimeField => reader.GetFieldValue(columnIndex), + BooleanField => reader.GetBoolean(columnIndex), + GuidField or ExactMatchField => reader.GetGuid(columnIndex), + _ => throw new InvalidOperationException($"Unsupported field type for sorting: {sortField.GetType().Name}") + }; +#pragma warning restore CA1849 + } + + private sealed record QueryClauses(string WhereClause, string JoinClause, string OrderByClause); + + private sealed record CursorQueryClauses( + string WhereClause, + string JoinClause, + string OrderByClause, + string SeekClause, + string SortColumnName); + + private sealed record SchemaComment(uint Version); + + /// + async Task IStore.LinkAsync(LinkDefinition definition, Storage.UuidV7 leftEntityId, Storage.UuidV7 rightEntityId, IReadOnlyList outboxEvents, Ct ct) + { + await using var cnn = await dataSource.OpenConnectionAsync(ct); + await using var tx = await cnn.BeginTransactionAsync(ct); + var outcome = await ExecuteLinkCoreAsync(cnn, tx, LinkOperation.For(definition, leftEntityId, rightEntityId), ct); + if (outcome == OperationOutcome.Success) + { + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(cnn, tx, outboxEvents, ct); + } + await tx.CommitAsync(ct); + } + return outcome == OperationOutcome.AlreadyLinked ? LinkResult.AlreadyLinked : LinkResult.Success; + } + + /// + async Task IStore.UnlinkAsync(LinkDefinition definition, Storage.UuidV7 leftEntityId, Storage.UuidV7 rightEntityId, IReadOnlyList outboxEvents, Ct ct) + { + await using var cnn = await dataSource.OpenConnectionAsync(ct); + await using var tx = await cnn.BeginTransactionAsync(ct); + _ = await ExecuteUnlinkCoreAsync(cnn, tx, UnlinkOperation.For(definition, leftEntityId, rightEntityId), ct); + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(cnn, tx, outboxEvents, ct); + } + await tx.CommitAsync(ct); + return UnlinkResult.Success; + } + + private async Task ExecuteLinkCoreAsync( + NpgsqlConnection cnn, + NpgsqlTransaction tx, + LinkOperation op, + Ct ct) + { + await using var cmd = cnn.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = $""" + INSERT INTO {_schemaName}.entity_links (pool_id, link_type_id, left_entity_type_id, left_entity_id, right_entity_type_id, right_entity_id) + VALUES (@pool_id, @link_type_id, @left_entity_type_id, @left_entity_id, @right_entity_type_id, @right_entity_id) + ON CONFLICT (pool_id, link_type_id, left_entity_id, right_entity_id) DO NOTHING + """; + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@link_type_id", (int)op.Definition.Link.Id); + _ = cmd.Parameters.AddWithValue("@left_entity_type_id", (int)op.Definition.Left.Id); + _ = cmd.Parameters.AddWithValue("@left_entity_id", op.LeftEntityId.Value); + _ = cmd.Parameters.AddWithValue("@right_entity_type_id", (int)op.Definition.Right.Id); + _ = cmd.Parameters.AddWithValue("@right_entity_id", op.RightEntityId.Value); + + Log.ExecutingSql(logger, cmd.CommandText); + + var rowsAffected = await cmd.ExecuteNonQueryAsync(ct); + return rowsAffected == 0 ? OperationOutcome.AlreadyLinked : OperationOutcome.Success; + } + + private async Task ExecuteUnlinkCoreAsync( + NpgsqlConnection cnn, + NpgsqlTransaction tx, + UnlinkOperation op, + Ct ct) + { + await using var cmd = cnn.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = $""" + DELETE FROM {_schemaName}.entity_links + WHERE pool_id = @pool_id + AND link_type_id = @link_type_id + AND left_entity_id = @left_entity_id + AND right_entity_id = @right_entity_id + """; + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@link_type_id", (int)op.Definition.Link.Id); + _ = cmd.Parameters.AddWithValue("@left_entity_id", op.LeftEntityId.Value); + _ = cmd.Parameters.AddWithValue("@right_entity_id", op.RightEntityId.Value); + + Log.ExecutingSql(logger, cmd.CommandText); + + _ = await cmd.ExecuteNonQueryAsync(ct); + return OperationOutcome.Success; + } + + private async Task ExecuteOutboxInsertBatchCoreAsync( + NpgsqlConnection cnn, + NpgsqlTransaction tx, + IReadOnlyList outboxEvents, + Ct ct) + { + var rows = new List<(OutboxEvent Evt, IOutboxSubscriber Subscriber)>(); + foreach (var evt in outboxEvents) + { + foreach (var subscriber in outboxSubscribers.GetMatchingSubscribers(evt.EventName, evt.EntityTypeId)) + { + rows.Add((evt, subscriber)); + } + } + + if (rows.Count == 0) + { + return; + } + + await using var cmd = cnn.CreateCommand(); + cmd.Transaction = tx; + + var valueRows = new List(rows.Count); + for (var i = 0; i < rows.Count; i++) + { + var (evt, subscriber) = rows[i]; + valueRows.Add($"(@message_id{i}, @event_id{i}, @timestamp{i}, @event_name{i}, @subject_id{i}, @entity_type_id{i}, @entity_type_name{i}, @pool_id, @payload{i}::jsonb, @subscriber_name{i})"); + _ = cmd.Parameters.AddWithValue($"@message_id{i}", Guid.CreateVersion7()); + _ = cmd.Parameters.AddWithValue($"@event_id{i}", evt.Id.Value); + _ = cmd.Parameters.Add(new NpgsqlParameter($"@timestamp{i}", NpgsqlDbType.TimestampTz) { Value = evt.Timestamp }); + _ = cmd.Parameters.AddWithValue($"@event_name{i}", evt.EventName.ToString()); + _ = cmd.Parameters.AddWithValue($"@subject_id{i}", evt.SubjectId.Value); + _ = cmd.Parameters.AddWithValue($"@entity_type_id{i}", evt.EntityTypeId); + _ = cmd.Parameters.AddWithValue($"@entity_type_name{i}", evt.EntityTypeName); + _ = cmd.Parameters.Add(new NpgsqlParameter($"@payload{i}", NpgsqlDbType.Jsonb) { Value = evt.Payload }); + _ = cmd.Parameters.AddWithValue($"@subscriber_name{i}", subscriber.SubscriberName.ToString()); + } + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + + cmd.CommandText = $""" + INSERT INTO {_schemaName}.outbox_subscriber_queue + (message_id, event_id, timestamp, event_name, subject_id, entity_type_id, entity_type_name, pool_id, payload, subscriber_name) + VALUES + {string.Join(",\n ", valueRows)} + """; + + Log.ExecutingSql(logger, cmd.CommandText); + _ = await cmd.ExecuteNonQueryAsync(ct); + } + + /// + /// Executes multiple operations atomically in a single transaction. + /// + async Task IStore.ExecuteBatchAsync( + IReadOnlyList operations, + IReadOnlyList outboxEvents, + Ct ct) + { + if (operations.Count == 0) + { + return new BatchResult(true, []); + } + + await using var connection = await dataSource.OpenConnectionAsync(ct); + await using var transaction = await connection.BeginTransactionAsync(ct); + + var results = new List(); + + try + { + for (var i = 0; i < operations.Count; i++) + { + var outcome = operations[i] switch + { + CreateOperation create => await ExecuteCreateCoreAsync(connection, transaction, create, ct), + UpdateOperation update => await ExecuteUpdateCoreAsync(connection, transaction, update, ct), + DeleteOperation delete => (await ExecuteDeleteCoreAsync(connection, transaction, delete, ct)).Outcome, + LinkOperation link => await ExecuteLinkCoreAsync(connection, transaction, link, ct), + UnlinkOperation unlink => await ExecuteUnlinkCoreAsync(connection, transaction, unlink, ct), + _ => throw new InvalidOperationException($"Unknown operation type: {operations[i].GetType().Name}") + }; + + results.Add(new OperationResult(i, outcome)); + + if (outcome is not OperationOutcome.Success and not OperationOutcome.AlreadyLinked) + { + // Fail-fast: stop processing on first failure + // Transaction is rolled back automatically on dispose + return new BatchResult(false, results); + } + } + + // All operations succeeded — INSERT outbox events before committing + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(connection, transaction, outboxEvents, ct); + } + + await transaction.CommitAsync(ct); + return new BatchResult(true, results); + } + catch + { + await transaction.RollbackAsync(ct); + throw; + } + } + + async Task IStore.GetOutboxEventsForSubscriberAsync(SubscriberName subscriberName, int count, Ct ct) + { + await using var cnn = await dataSource.OpenConnectionAsync(ct); + await using var cmd = cnn.CreateCommand(); + cmd.CommandText = $""" + SELECT sequence_number, message_id, event_id, timestamp, event_name, subject_id, entity_type_id, entity_type_name, pool_id, payload, subscriber_name + FROM {_schemaName}.outbox_subscriber_queue + WHERE subscriber_name = @subscriber_name + ORDER BY sequence_number ASC + LIMIT @limit + """; + _ = cmd.Parameters.AddWithValue("@subscriber_name", subscriberName.ToString()); + _ = cmd.Parameters.AddWithValue("@limit", count + 1); + + Log.ExecutingSql(logger, cmd.CommandText); + await using var reader = await cmd.ExecuteReaderAsync(ct); + + var events = new List(); + while (await reader.ReadAsync(ct)) + { + var timestamp = await reader.GetFieldValueAsync(3, ct); + events.Add(new PersistedOutboxEvent + { + SequenceNumber = reader.GetInt64(0), + MessageId = reader.GetGuid(1), + EventId = reader.GetGuid(2), + Timestamp = timestamp, + EventName = OutboxEventName.Create(reader.GetString(4)), + SubjectId = Storage.UuidV7.From(reader.GetGuid(5)), + EntityTypeId = reader.GetInt32(6), + EntityTypeName = reader.GetString(7), + PoolId = PoolId.Load(reader.GetInt32(8)), + Payload = reader.GetString(9), + SubscriberName = SubscriberName.Create(reader.GetString(10)), + }); + } + + var hasMore = events.Count > count; + if (hasMore) + { + events.RemoveAt(events.Count - 1); + } + + return new OutboxEventsPage(events, hasMore); + } + + async Task IStore.DeleteOutboxEventsAsync(IReadOnlyList ids, Ct ct) + { + if (ids.Count == 0) + { + return; + } + + await using var cnn = await dataSource.OpenConnectionAsync(ct); + + const int MaxBatchSize = 1000; + for (var offset = 0; offset < ids.Count; offset += MaxBatchSize) + { + var chunk = ids.Skip(offset).Take(MaxBatchSize).ToArray(); + + await using var cmd = cnn.CreateCommand(); + cmd.CommandText = $""" + DELETE FROM {_schemaName}.outbox_subscriber_queue + WHERE message_id = ANY(@ids) + """; + var guidIds = chunk.Select(id => id.Value).ToArray(); + _ = cmd.Parameters.Add(new NpgsqlParameter("@ids", NpgsqlDbType.Array | NpgsqlDbType.Uuid) { Value = guidIds }); + + Log.ExecutingSql(logger, cmd.CommandText); + _ = await cmd.ExecuteNonQueryAsync(ct); + } + } + + private async Task ExecuteCreateCoreAsync( + NpgsqlConnection cnn, + NpgsqlTransaction tx, + CreateOperation op, + Ct ct) + { + var dsoVersion = op.DsoVersion; + var dsoTypeId = (int)dsoVersion.EntityType.Id; + var entityType = dsoVersion.EntityType; + var jsonDso = JsonSerializer.Serialize(op.Value); + + // Resolve expiration + var expiresAt = op.Expiration.Resolve(timeProvider); + if (expiresAt.HasValue && expiresAt.Value <= timeProvider.GetUtcNow()) + { + // Already expired — noop, return success without storing + return OperationOutcome.Success; + } + + Log.CreatingDso(logger, entityType, op.Id.Value, dsoVersion.SchemaVersion); + + var builder = new StringBuilder(); + + // Insert the record + _ = builder.AppendLine( + CultureInfo.InvariantCulture, + $""" + INSERT INTO {_schemaName}.entities (entity_type_id, entity_type_name, entity_id, value, dso_type_schema_version, value_version, pool_id, expires_at, created_at, last_updated_at) + VALUES (@entity_type_id, @entity_type_name, @entity_id, @value, @dso_type_schema_version, 1, @pool_id, @expires_at, @now, @now); + """); + + await using var createCmd = cnn.CreateCommand(); + createCmd.Transaction = tx; + _ = createCmd.Parameters.AddWithValue("@entity_type_id", dsoTypeId); + _ = createCmd.Parameters.AddWithValue("@entity_type_name", entityType.Name); + _ = createCmd.Parameters.AddWithValue("@entity_id", op.Id.Value); + _ = createCmd.Parameters.AddWithValue("@value", NpgsqlDbType.Jsonb, jsonDso); + _ = createCmd.Parameters.AddWithValue("@dso_type_schema_version", (int)dsoVersion.SchemaVersion); + _ = createCmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = createCmd.Parameters.AddWithValue("@now", NpgsqlDbType.TimestampTz, timeProvider.GetUtcNow().UtcDateTime); + if (expiresAt.HasValue) + { + _ = createCmd.Parameters.AddWithValue("@expires_at", NpgsqlDbType.TimestampTz, expiresAt.Value.UtcDateTime); + } + else + { + _ = createCmd.Parameters.AddWithValue("@expires_at", DBNull.Value); + } + + // Add an insert statement for each key + _ = AddInserts(builder, createCmd, op.Keys); + + // Add insert statements for search fields + _ = AddSearchFieldInserts(builder, createCmd, op.SearchFieldCollection); + + createCmd.CommandText = builder.ToString(); + + // Create a savepoint before the operation so we can recover from unique violations + // PostgreSQL aborts the transaction on errors, requiring rollback to a savepoint + var savepointName = $"create_{Guid.NewGuid()}"; + await tx.SaveAsync(savepointName, ct); + + Log.ExecutingSql(logger, createCmd.CommandText); + + try + { + _ = await createCmd.ExecuteNonQueryAsync(ct); + } + catch (PostgresException ex) when (string.Equals(ex.SqlState, PostgresErrorCodes.UniqueViolation, + StringComparison.OrdinalIgnoreCase)) + { + // Roll back to the savepoint to clear the aborted transaction state + // This allows subsequent queries to execute within this transaction + await tx.RollbackAsync(savepointName, ct); + + if (await AlreadyExistsInTransactionAsync(cnn, tx, entityType, op.Id.Value, ct)) + { + return OperationOutcome.AlreadyExists; + } + + // One of the keys already exists + return OperationOutcome.KeyConflict; + } + + return OperationOutcome.Success; + } + + private async Task ExecuteUpdateCoreAsync( + NpgsqlConnection cnn, + NpgsqlTransaction tx, + UpdateOperation op, + Ct ct) + { + var dsoVersion = op.DsoVersion; + var entityType = dsoVersion.EntityType; + var jsonDso = JsonSerializer.Serialize(op.Value); + + // Resolve expiration + DateTimeOffset? expiresAt = null; + var hasExpirationChange = op.Expiration is not null; + if (hasExpirationChange) + { + expiresAt = op.Expiration!.Resolve(timeProvider); + } + + Log.UpdatingDso(logger, entityType, op.Id.Value, dsoVersion.SchemaVersion, op.ExpectedEntityVersion); + + await using var readVersionCmd = cnn.CreateCommand(); + readVersionCmd.Transaction = tx; + + // Read the current version of the entity, locking the row + readVersionCmd.CommandText = + $"SELECT value_version FROM {_schemaName}.entities WHERE entity_type_id = @entity_type_id AND entity_id = @entity_id AND pool_id = @pool_id FOR UPDATE"; + + _ = readVersionCmd.Parameters.AddWithValue("@entity_type_id", (int)entityType.Id); + _ = readVersionCmd.Parameters.AddWithValue("@entity_id", op.Id.Value); + _ = readVersionCmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + + Log.ExecutingSql(logger, readVersionCmd.CommandText); + + var actualEntityVersion = (int?)await readVersionCmd.ExecuteScalarAsync(ct); + + if (actualEntityVersion == null) + { + return OperationOutcome.DoesNotExist; + } + + if (actualEntityVersion != op.ExpectedEntityVersion) + { + return OperationOutcome.UnexpectedVersion; + } + + var builder = new StringBuilder(); + + var expiresAtSql = hasExpirationChange + ? "expires_at = @expires_at," + : ""; // Don't change existing expires_at when expiration is null + + // Update the main DSO row, incrementing its version + _ = builder.AppendLine( + CultureInfo.InvariantCulture, + $""" + UPDATE {_schemaName}.entities + SET + entity_type_name = @entity_type_name, + value = @value, + dso_type_schema_version = @dso_type_schema_version, + value_version = value_version + 1, + {expiresAtSql} + last_updated_at = @now + WHERE + entity_type_id = @entity_type_id AND entity_id = @entity_id AND pool_id = @pool_id; + """); + + // Create the command + await using var updateCmd = cnn.CreateCommand(); + updateCmd.Transaction = tx; + _ = updateCmd.Parameters.AddWithValue("@entity_type_id", (int)entityType.Id); + _ = updateCmd.Parameters.AddWithValue("@entity_type_name", entityType.Name); + _ = updateCmd.Parameters.AddWithValue("@entity_id", op.Id.Value); + _ = updateCmd.Parameters.AddWithValue("@value", NpgsqlDbType.Jsonb, jsonDso); + _ = updateCmd.Parameters.AddWithValue("@dso_type_schema_version", (int)dsoVersion.SchemaVersion); + _ = updateCmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = updateCmd.Parameters.AddWithValue("@now", NpgsqlDbType.TimestampTz, timeProvider.GetUtcNow().UtcDateTime); + if (hasExpirationChange) + { + if (expiresAt.HasValue) + { + _ = updateCmd.Parameters.AddWithValue("@expires_at", NpgsqlDbType.TimestampTz, expiresAt.Value.UtcDateTime); + } + else + { + _ = updateCmd.Parameters.AddWithValue("@expires_at", DBNull.Value); + } + } + + // Delete the existing keys + _ = builder.AppendLine( + CultureInfo.InvariantCulture, + $"DELETE FROM {_schemaName}.entity_keys WHERE entity_type_id = @entity_type_id AND entity_id = @entity_id AND pool_id = @pool_id;"); + + // Delete the existing search fields + _ = builder.AppendLine( + CultureInfo.InvariantCulture, + $"DELETE FROM {_schemaName}.search_values WHERE entity_type_id = @entity_type_id AND entity_id = @entity_id AND pool_id = @pool_id;"); + + // re-insert the new keys + _ = AddInserts(builder, updateCmd, op.Keys); + + // re-insert the new search fields + _ = AddSearchFieldInserts(builder, updateCmd, op.SearchFieldCollection); + + updateCmd.CommandText = builder.ToString(); + + // Create a savepoint before the operation so we can recover from unique violations + // PostgreSQL aborts the transaction on errors, requiring rollback to a savepoint + var savepointName = $"update_{Guid.NewGuid()}"; + await tx.SaveAsync(savepointName, ct); + + Log.ExecutingSql(logger, updateCmd.CommandText); + + try + { + _ = await updateCmd.ExecuteNonQueryAsync(ct); + } + catch (PostgresException ex) when (string.Equals(ex.SqlState, PostgresErrorCodes.UniqueViolation, + StringComparison.OrdinalIgnoreCase)) + { + // Roll back to the savepoint to clear the aborted transaction state + await tx.RollbackAsync(savepointName, ct); + return OperationOutcome.KeyConflict; + } + + return OperationOutcome.Success; + } + + private async Task<(OperationOutcome Outcome, bool EntityDeleted)> ExecuteDeleteCoreAsync( + NpgsqlConnection cnn, + NpgsqlTransaction tx, + DeleteOperation op, + Ct ct) + { + var entityType = op.EntityType; + + await using var deleteCmd = cnn.CreateCommand(); + deleteCmd.Transaction = tx; + _ = deleteCmd.Parameters.AddWithValue("@entity_type_id", (int)entityType.Id); + _ = deleteCmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + + deleteCmd.CommandText = $"DELETE FROM {_schemaName}.entities WHERE entity_type_id = @entity_type_id AND pool_id = @pool_id"; + + // Both entity_keys and search_values have ON DELETE CASCADE, so we only need to delete from entities + if (op.Id is not null) + { + Log.DeletingDso(logger, entityType, op.Id.Value); + _ = deleteCmd.Parameters.AddWithValue("@entity_id", op.Id.Value); + + deleteCmd.CommandText += " AND entity_id = @entity_id"; + } + else if (op.Key is not null) + { + var key = op.Key; + _ = deleteCmd.Parameters.AddWithValue("@key_type_id", (int)key.DskVersion.KeyType.Id); + _ = deleteCmd.Parameters.AddWithValue("@key_type_version", (int)key.DskVersion.SchemaVersion); + _ = deleteCmd.Parameters.AddWithValue("@key_value", key.Value); + deleteCmd.CommandText += $""" + AND entity_id = ( + SELECT entity_id FROM {_schemaName}.entity_keys + WHERE entity_type_id = @entity_type_id + AND key_type_id = @key_type_id + AND key_type_version = @key_type_version + AND key_value = @key_value + AND pool_id = @pool_id + ) + """; + } + else + { + return (OperationOutcome.Success, false); + } + + Log.ExecutingSql(logger, deleteCmd.CommandText); + + // Resolve entity_id for link cleanup BEFORE deleting the entity, + // because entity_keys has ON DELETE CASCADE and will be gone after delete. + var entityId = op.Id?.Value ?? (op.Key is not null + ? await ResolveKeyToEntityIdAsync(cnn, tx, op.EntityType, op.Key, ct) + : null); + + var rowsAffected = await deleteCmd.ExecuteNonQueryAsync(ct); + + // Delete entity links (no FK to entities, must be done manually) + if (entityId.HasValue) + { + await using var linkDeleteCmd = cnn.CreateCommand(); + linkDeleteCmd.Transaction = tx; + linkDeleteCmd.CommandText = $""" + DELETE FROM {_schemaName}.entity_links + WHERE pool_id = @pool_id + AND (left_entity_id = @entity_id OR right_entity_id = @entity_id) + """; + _ = linkDeleteCmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = linkDeleteCmd.Parameters.AddWithValue("@entity_id", entityId.Value); + Log.ExecutingSql(logger, linkDeleteCmd.CommandText); + _ = await linkDeleteCmd.ExecuteNonQueryAsync(ct); + } + + return (OperationOutcome.Success, rowsAffected > 0); + } + + private async Task ResolveKeyToEntityIdAsync( + NpgsqlConnection cnn, + NpgsqlTransaction tx, + EntityType entityType, + DataStorageKey key, + Ct ct) + { + await using var cmd = cnn.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = $""" + SELECT entity_id FROM {_schemaName}.entity_keys + WHERE entity_type_id = @entity_type_id + AND key_type_id = @key_type_id + AND key_type_version = @key_type_version + AND key_value = @key_value + AND pool_id = @pool_id + """; + _ = cmd.Parameters.AddWithValue("@entity_type_id", (int)entityType.Id); + _ = cmd.Parameters.AddWithValue("@key_type_id", (int)key.DskVersion.KeyType.Id); + _ = cmd.Parameters.AddWithValue("@key_type_version", (int)key.DskVersion.SchemaVersion); + _ = cmd.Parameters.AddWithValue("@key_value", key.Value); + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + var result = await cmd.ExecuteScalarAsync(ct); + return result is Guid guid ? guid : null; + } + + private async Task AlreadyExistsInTransactionAsync( + NpgsqlConnection cnn, + NpgsqlTransaction tx, + EntityType entityType, + Guid entityId, + Ct ct) + { + await using var checkExistsCmd = cnn.CreateCommand(); + checkExistsCmd.Transaction = tx; + checkExistsCmd.CommandText = + $"SELECT value_version FROM {_schemaName}.entities WHERE entity_type_id = @entity_type_id AND entity_id = @entity_id AND pool_id = @pool_id"; + + _ = checkExistsCmd.Parameters.AddWithValue("@entity_type_id", (int)entityType.Id); + _ = checkExistsCmd.Parameters.AddWithValue("@entity_id", entityId); + _ = checkExistsCmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + + Log.ExecutingSql(logger, checkExistsCmd.CommandText); + + return (int?)await checkExistsCmd.ExecuteScalarAsync(ct) is not null; + } + + /// + async Task>> IStore.QueryLinksAsync( + LinkQueryDescriptor query, + DataRange dataRange, + Ct ct) + { + if (dataRange.TokenValue is not null) + { + throw new NotSupportedException("Cursor-based pagination is not supported for link queries."); + } + + var (skip, take) = ResolveOffsetAndSize(dataRange); + var dsoVersion = TDso.DsoVersion; + var sourceEntityTypeId = (int)query.SourceEntityType.Id; + + await using var cmd = dataSource.CreateCommand(); + + var joinSql = new StringBuilder(); + var whereLastJoin = ""; + + for (var i = 0; i < query.Joins.Count; i++) + { + var join = query.Joins[i]; + var linkTypeParam = $"@lt{i}"; + _ = cmd.Parameters.AddWithValue(linkTypeParam, (int)join.Definition.Link.Id); + + // Which side of this link corresponds to the source (entities table / previous join)? + string sourceSide; + string filterSide; + if (join.Direction == LinkJoinDirection.LeftToRight) + { + sourceSide = "left_entity_id"; + filterSide = "right_entity_id"; + } + else + { + sourceSide = "right_entity_id"; + filterSide = "left_entity_id"; + } + + if (i == 0) + { + // First join: links entity table to first link table + _ = joinSql.AppendLine(CultureInfo.InvariantCulture, + $"JOIN {_schemaName}.entity_links l0 ON l0.{sourceSide} = e.entity_id AND l0.link_type_id = {linkTypeParam} AND l0.pool_id = @pool_id"); + } + else + { + // Subsequent joins: link previous join's filter side to this join's source side + var prevJoin = query.Joins[i - 1]; + string prevFilterSide; + if (prevJoin.Direction == LinkJoinDirection.LeftToRight) + { + prevFilterSide = "right_entity_id"; + } + else + { + prevFilterSide = "left_entity_id"; + } + + _ = joinSql.AppendLine(CultureInfo.InvariantCulture, + $"JOIN {_schemaName}.entity_links l{i} ON l{i}.{sourceSide} = l{i - 1}.{prevFilterSide} AND l{i}.link_type_id = {linkTypeParam} AND l{i}.pool_id = @pool_id"); + } + + if (i == query.Joins.Count - 1) + { + whereLastJoin = $"l{i}.{filterSide}"; + } + } + + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@source_entity_type_id", sourceEntityTypeId); + _ = cmd.Parameters.AddWithValue("@offset", skip); + _ = cmd.Parameters.AddWithValue("@limit", take); + + string whereClause; + if (query.WhereEntityId is not null) + { + _ = cmd.Parameters.AddWithValue("@where_entity_id", query.WhereEntityId.Value); + whereClause = $"{whereLastJoin} = @where_entity_id"; + } + else + { + whereClause = "1=1"; + } + + var mainQuery = $""" + SELECT DISTINCT e.entity_id, e.value, e.dso_type_schema_version, e.value_version, e.created_at, e.last_updated_at + FROM {_schemaName}.entities e + {joinSql} + WHERE e.entity_type_id = @source_entity_type_id + AND e.pool_id = @pool_id + AND {whereClause} + ORDER BY e.entity_id + OFFSET @offset LIMIT @limit + """; + + cmd.CommandText = mainQuery; + Log.ExecutingQuery(logger, mainQuery); + + var items = new List>(); + var dsoType = dataStorageTypeRegistry.Get(dsoVersion); + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + { + var entityId = reader.GetGuid(0); + var jsonValue = reader.GetString(1); + var item = (TDso)JsonSerializer.Deserialize(jsonValue, dsoType)!; + var valueVersion = reader.GetInt32(3); + var created = await reader.GetFieldValueAsync(4, ct); + var lastUpdated = await reader.GetFieldValueAsync(5, ct); + items.Add(new MetadataEnvelope(item, entityId, valueVersion, created, lastUpdated)); + } + } + + // Count query + var countQuery = $""" + SELECT COUNT(DISTINCT e.entity_id) + FROM {_schemaName}.entities e + {joinSql} + WHERE e.entity_type_id = @source_entity_type_id + AND e.pool_id = @pool_id + AND {whereClause} + """; + + await using var countCmd = dataSource.CreateCommand(); + _ = countCmd.Parameters.AddWithValue("@source_entity_type_id", sourceEntityTypeId); + _ = countCmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + if (query.WhereEntityId is not null) + { + _ = countCmd.Parameters.AddWithValue("@where_entity_id", query.WhereEntityId.Value); + } + + for (var i = 0; i < query.Joins.Count; i++) + { + _ = countCmd.Parameters.AddWithValue($"@lt{i}", (int)query.Joins[i].Definition.Link.Id); + } + + countCmd.CommandText = countQuery; + Log.ExecutingSql(logger, countQuery); + var totalCount = Convert.ToInt32(await countCmd.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture); + + return new QueryResult> + { + Items = items, + TotalCount = totalCount, + TotalPages = (int)Math.Ceiling((double)totalCount / take), + HasMoreData = skip + take < totalCount + }; + } + + async Task IStore.CountAsync( + EntityType entityType, + IQueryExpression? filter, + Ct ct) + { + var entityTypeId = (int)entityType.Id; + + await using var cmd = dataSource.CreateCommand(); + + string whereClause; + if (filter is null or AllExpression) + { + whereClause = "1=1"; + } + else + { + var whereBuilder = new SqlWhereClauseBuilder(_schemaName, cmd, Dialect); + whereClause = whereBuilder.BuildWhereClause(filter); + } + + var query = $""" + SELECT COUNT(*) + FROM {_schemaName}.entities v + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({whereClause}) + """; + + _ = cmd.Parameters.AddWithValue("@entity_type_id", entityTypeId); + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + + cmd.CommandText = query; + + Log.ExecutingSql(logger, query); + + var result = await cmd.ExecuteScalarAsync(ct); + return Convert.ToInt64(result, CultureInfo.InvariantCulture); + } + + async Task IStore.PurgeExpiredAsync(int batchSize, Ct ct) + { + ArgumentOutOfRangeException.ThrowIfLessThan(batchSize, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(batchSize, StorageConstants.TtlCleanupMaxBatchSize); + + var now = timeProvider.GetUtcNow(); + + await using var cnn = await dataSource.OpenConnectionAsync(ct); + await using var tx = await cnn.BeginTransactionAsync(ct); + + try + { + await using var cmd = cnn.CreateCommand(); + cmd.Transaction = tx; + + var sql = new StringBuilder(); + + // Step 1: Lock expired rows into a temp table + _ = sql.AppendLine(CultureInfo.InvariantCulture, $""" + CREATE TEMP TABLE _expired ON COMMIT DROP AS + SELECT pool_id, entity_id, entity_type_id, entity_type_name, value, gen_random_uuid() AS event_id + FROM {_schemaName}.entities + WHERE expires_at IS NOT NULL AND expires_at <= @now + LIMIT @batchSize + FOR UPDATE SKIP LOCKED; + """); + _ = cmd.Parameters.Add(new NpgsqlParameter("@now", NpgsqlDbType.TimestampTz) { Value = now.UtcDateTime }); + _ = cmd.Parameters.AddWithValue("@batchSize", batchSize); + + // Step 2: Insert outbox events per matching subscriber + if (!outboxSubscribers.IsEmpty) + { + var eventName = OutboxEventName.EntityExpired; + _ = cmd.Parameters.AddWithValue("@eventName", eventName.ToString()); + + var subscriberIndex = 0; + foreach (var subscriber in outboxSubscribers.Subscribers) + { + if (subscriber.EventNames.Count > 0 && !subscriber.EventNames.Contains(eventName)) + { + continue; + } + + var subParam = $"@sub{subscriberIndex}"; + _ = cmd.Parameters.AddWithValue(subParam, subscriber.SubscriberName.ToString()); + + _ = sql.Append(CultureInfo.InvariantCulture, $""" + INSERT INTO {_schemaName}.outbox_subscriber_queue + (message_id, event_id, timestamp, event_name, subject_id, entity_type_id, entity_type_name, pool_id, payload, subscriber_name) + SELECT gen_random_uuid(), event_id, @now, @eventName, entity_id, entity_type_id, entity_type_name, pool_id, value, {subParam} + FROM _expired + """); + + if (subscriber.EntityTypeIds.Count > 0) + { + var typesParam = $"@subTypes{subscriberIndex}"; + _ = cmd.Parameters.Add(new NpgsqlParameter(typesParam, NpgsqlDbType.Array | NpgsqlDbType.Integer) + { + Value = subscriber.EntityTypeIds.ToArray() + }); + _ = sql.Append(CultureInfo.InvariantCulture, $" WHERE entity_type_id = ANY({typesParam})"); + } + + _ = sql.AppendLine(";"); + subscriberIndex++; + } + } + + // Step 3: Delete entity links + _ = sql.AppendLine(CultureInfo.InvariantCulture, $""" + DELETE FROM {_schemaName}.entity_links el + USING _expired e + WHERE el.pool_id = e.pool_id + AND ( + (el.left_entity_id = e.entity_id AND el.left_entity_type_id = e.entity_type_id) + OR (el.right_entity_id = e.entity_id AND el.right_entity_type_id = e.entity_type_id) + ); + """); + + // Step 4: Delete entities — last statement so ExecuteNonQueryAsync returns this count + _ = sql.Append(CultureInfo.InvariantCulture, $""" + DELETE FROM {_schemaName}.entities e + USING _expired + WHERE e.pool_id = _expired.pool_id AND e.entity_type_id = _expired.entity_type_id AND e.entity_id = _expired.entity_id + AND e.expires_at <= @now; + """); + + cmd.CommandText = sql.ToString(); + Log.ExecutingSql(logger, cmd.CommandText); + var deleted = await cmd.ExecuteNonQueryAsync(ct); + + await tx.CommitAsync(ct); + return deleted; + } + catch + { + await tx.RollbackAsync(ct); + throw; + } + } + +} diff --git a/storage/src/Storage.PostgreSql/Internal/PostgreSqlStoreOptionsValidator.cs b/storage/src/Storage.PostgreSql/Internal/PostgreSqlStoreOptionsValidator.cs new file mode 100644 index 000000000..874588142 --- /dev/null +++ b/storage/src/Storage.PostgreSql/Internal/PostgreSqlStoreOptionsValidator.cs @@ -0,0 +1,9 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Microsoft.Extensions.Options; + +namespace Duende.Storage.PostgreSql.Internal; + +#pragma warning disable CA1812 // Avoid uninstantiated internal classes +public sealed class PostgreSqlStoreOptionsValidator(string? name) : DataAnnotationValidateOptions(name); diff --git a/storage/src/Storage.PostgreSql/Migrations/V001_InitialCreate.sql b/storage/src/Storage.PostgreSql/Migrations/V001_InitialCreate.sql new file mode 100644 index 000000000..4b362024e --- /dev/null +++ b/storage/src/Storage.PostgreSql/Migrations/V001_InitialCreate.sql @@ -0,0 +1,175 @@ + +-- This statement has to be outside of the DO block to ensure the schema exists before we +-- attempt to read its comment for versioning +CREATE SCHEMA IF NOT EXISTS [[schemaname]]; + +-- Migration V0 → V1: initial schema creation +DO $$ +DECLARE + current_version INT; +BEGIN + -- Read current version from schema comment + SELECT COALESCE( + (SELECT (obj_description('[[schemaname]]'::regnamespace)::jsonb->>'Version')::int + WHERE obj_description('[[schemaname]]'::regnamespace) IS NOT NULL + AND obj_description('[[schemaname]]'::regnamespace) NOT IN ('standard public schema', '')), + 0) + INTO current_version; + + current_version := COALESCE(current_version, 0); + + IF current_version < 1 THEN + + CREATE SCHEMA IF NOT EXISTS [[schemaname]]; + + CREATE TABLE [[schemaname]].entities + ( + pool_id INTEGER NOT NULL, + entity_type_id INT NOT NULL, + entity_id UUID NOT NULL, + entity_type_name TEXT NOT NULL, + value JSONB NOT NULL, + dso_type_schema_version INT NOT NULL, + value_version INT NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + last_updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + expires_at TIMESTAMP WITH TIME ZONE NULL, + PRIMARY KEY (pool_id, entity_type_id, entity_id) + ); + + CREATE INDEX entities_expires_at_index + ON [[schemaname]].entities (expires_at) + WHERE expires_at IS NOT NULL; + + CREATE INDEX entities_created_at_index + ON [[schemaname]].entities (pool_id, entity_type_id, created_at); + + CREATE INDEX entities_last_updated_at_index + ON [[schemaname]].entities (pool_id, entity_type_id, last_updated_at); + + CREATE TABLE [[schemaname]].entity_keys + ( + pool_id INTEGER NOT NULL, + entity_type_id INT NOT NULL, + key_type_id INT NOT NULL, + key_type_version INT NOT NULL, + key_type_name TEXT NOT NULL, + key_value UUID NOT NULL, + key_json JSONB NULL, + entity_id UUID NOT NULL, + timestamp TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (pool_id, entity_type_id, key_type_id, key_type_version, key_value), + FOREIGN KEY (pool_id, entity_type_id, entity_id) + REFERENCES [[schemaname]].entities (pool_id, entity_type_id, entity_id) ON DELETE CASCADE + ); + + CREATE INDEX entity_keys_entity_type_id_entity_id_index + ON [[schemaname]].entity_keys (entity_type_id, entity_id); + + CREATE TABLE [[schemaname]].search_values + ( + entity_type_id INT NOT NULL, + entity_id UUID NOT NULL, + field_path UUID NOT NULL, + field_path_text TEXT NOT NULL, + item_index INT NOT NULL, + string_value TEXT NULL, + number_value NUMERIC NULL, + datetime_value TIMESTAMP WITH TIME ZONE NULL, + boolean_value BOOLEAN NULL, + guid_value UUID NULL, + pool_id INTEGER NOT NULL, + PRIMARY KEY (pool_id, entity_type_id, entity_id, field_path, item_index), + FOREIGN KEY (pool_id, entity_type_id, entity_id) + REFERENCES [[schemaname]].entities (pool_id, entity_type_id, entity_id) ON DELETE CASCADE + ); + + CREATE INDEX search_values_string_value_index + ON [[schemaname]].search_values (pool_id, entity_type_id, field_path, string_value) + WHERE string_value IS NOT NULL AND item_index = -1; + + CREATE INDEX search_values_number_value_index + ON [[schemaname]].search_values (pool_id, entity_type_id, field_path, number_value) + WHERE number_value IS NOT NULL AND item_index = -1; + + CREATE INDEX search_values_datetime_value_index + ON [[schemaname]].search_values (pool_id, entity_type_id, field_path, datetime_value) + WHERE datetime_value IS NOT NULL AND item_index = -1; + + CREATE INDEX search_values_boolean_value_index + ON [[schemaname]].search_values (pool_id, entity_type_id, field_path, boolean_value) + WHERE boolean_value IS NOT NULL AND item_index = -1; + + CREATE INDEX search_values_array_string_value_index + ON [[schemaname]].search_values (pool_id, entity_type_id, entity_id, field_path, item_index, string_value) + WHERE string_value IS NOT NULL AND item_index >= 0; + + CREATE INDEX search_values_array_number_value_index + ON [[schemaname]].search_values (pool_id, entity_type_id, entity_id, field_path, item_index, number_value) + WHERE number_value IS NOT NULL AND item_index >= 0; + + CREATE INDEX search_values_array_datetime_value_index + ON [[schemaname]].search_values (pool_id, entity_type_id, entity_id, field_path, item_index, datetime_value) + WHERE datetime_value IS NOT NULL AND item_index >= 0; + + CREATE INDEX search_values_array_boolean_value_index + ON [[schemaname]].search_values (pool_id, entity_type_id, entity_id, field_path, item_index, boolean_value) + WHERE boolean_value IS NOT NULL AND item_index >= 0; + + CREATE INDEX search_values_guid_value_index + ON [[schemaname]].search_values (pool_id, entity_type_id, field_path, guid_value) + WHERE item_index = -1 AND guid_value IS NOT NULL; + + CREATE INDEX search_values_array_guid_value_index + ON [[schemaname]].search_values (pool_id, entity_type_id, entity_id, field_path, item_index, guid_value) + WHERE item_index >= 0 AND guid_value IS NOT NULL; + + CREATE TABLE [[schemaname]].entity_links + ( + pool_id INTEGER NOT NULL, + link_type_id INT NOT NULL, + left_entity_type_id INT NOT NULL, + left_entity_id UUID NOT NULL, + right_entity_type_id INT NOT NULL, + right_entity_id UUID NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (pool_id, link_type_id, left_entity_id, right_entity_id) + ); + + CREATE INDEX entity_links_left_entity_index + ON [[schemaname]].entity_links (pool_id, link_type_id, left_entity_id); + + CREATE INDEX entity_links_right_entity_index + ON [[schemaname]].entity_links (pool_id, link_type_id, right_entity_id); + + CREATE INDEX entity_links_left_cascade_index + ON [[schemaname]].entity_links (pool_id, left_entity_id); + + CREATE INDEX entity_links_right_cascade_index + ON [[schemaname]].entity_links (pool_id, right_entity_id); + + CREATE TABLE [[schemaname]].outbox_subscriber_queue + ( + sequence_number BIGINT GENERATED ALWAYS AS IDENTITY, + message_id UUID NOT NULL, + event_id UUID NOT NULL, + timestamp TIMESTAMP WITH TIME ZONE NOT NULL, + event_name TEXT NOT NULL, + subject_id UUID NOT NULL, + entity_type_id INT NOT NULL, + entity_type_name TEXT NOT NULL, + pool_id INTEGER NOT NULL, + payload JSONB NOT NULL, + subscriber_name TEXT NOT NULL, + PRIMARY KEY (sequence_number), + UNIQUE (message_id) + ); + + CREATE INDEX outbox_subscriber_queue_subscriber_index + ON [[schemaname]].outbox_subscriber_queue (subscriber_name, sequence_number); + + COMMENT ON SCHEMA [[schemaname]] IS '{"Version":1}'; + + END IF; +END +$$; diff --git a/storage/src/Storage.PostgreSql/PostgreSqlStoreOptions.cs b/storage/src/Storage.PostgreSql/PostgreSqlStoreOptions.cs new file mode 100644 index 000000000..1a1306d3f --- /dev/null +++ b/storage/src/Storage.PostgreSql/PostgreSqlStoreOptions.cs @@ -0,0 +1,16 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.ComponentModel.DataAnnotations; + +namespace Duende.Storage.PostgreSql; + +public sealed class PostgreSqlStoreOptions +{ + /// + /// The schema name to use for the store's tables. + /// + [Required] + [RegularExpression(@"^[a-zA-Z0-9_\-\#\@]+$", ErrorMessage = "Schema name must contain only alphanumeric characters, underscores, hyphens, hashes, and at signs.")] + public string SchemaName { get; set; } = "public"; +} diff --git a/storage/src/Storage.PostgreSql/PostgreSqlStoreServiceCollectionExtensions.cs b/storage/src/Storage.PostgreSql/PostgreSqlStoreServiceCollectionExtensions.cs new file mode 100644 index 000000000..48cf5bc10 --- /dev/null +++ b/storage/src/Storage.PostgreSql/PostgreSqlStoreServiceCollectionExtensions.cs @@ -0,0 +1,76 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.ComponentModel.DataAnnotations; +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.PostgreSql.Internal; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Npgsql; + +namespace Duende.Storage.PostgreSql; + +public static class PostgreSqlStoreServiceCollectionExtensions +{ + extension(IStorageBuilder builder) + { + /// + /// Adds a PostgreSQL store with the specified service key for multi-store scenarios. + /// The caller must register a keyed with the same service key. + /// + internal IStorageBuilder AddPostgreSqlStore(string serviceKey, Action configure) + { + var services = builder.Services; + _ = services.AddStore(serviceKey); + _ = services.AddKeyedTransient(serviceKey, (sp, _) => + { + var dataSource = sp.GetRequiredKeyedService(serviceKey); + var outboxSubscribers = sp.GetRequiredKeyedService(serviceKey); + return BuildStore(sp, dataSource, outboxSubscribers, configure); + }); + return builder; + } + + /// + /// Adds a PostgreSQL store without a service key for single-store scenarios. + /// The caller must register an unkeyed . + /// + public IStorageBuilder AddPostgreSqlStore() => builder.AddPostgreSqlStore(_ => { }); + + /// + /// Adds a PostgreSQL store without a service key for single-store scenarios. + /// The caller must register an unkeyed . + /// + public IStorageBuilder AddPostgreSqlStore(Action configure) + { + var services = builder.Services; + _ = services.AddStore(); + _ = services.AddTransient(sp => + { + var dataSource = sp.GetRequiredService(); + var outboxSubscribers = sp.GetRequiredService(); + return BuildStore(sp, dataSource, outboxSubscribers, configure); + }); + return builder; + } + } + + private static PostgreSqlStore BuildStore( + IServiceProvider sp, + NpgsqlDataSource dataSource, + OutboxSubscribers outboxSubscribers, + Action configure) + { + var options = new PostgreSqlStoreOptions(); + configure(options); + Validator.ValidateObject(options, new ValidationContext(options), validateAllProperties: true); + return new PostgreSqlStore( + dataSource, + options, + sp.GetRequiredService(), + sp.GetRequiredService(), + outboxSubscribers, + sp.GetRequiredService>()); + } +} diff --git a/storage/src/Storage.PostgreSql/Storage.PostgreSql.csproj b/storage/src/Storage.PostgreSql/Storage.PostgreSql.csproj new file mode 100644 index 000000000..3a06ff243 --- /dev/null +++ b/storage/src/Storage.PostgreSql/Storage.PostgreSql.csproj @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/storage/src/Storage.Sqlite/Internal/Log.cs b/storage/src/Storage.Sqlite/Internal/Log.cs new file mode 100644 index 000000000..660183ada --- /dev/null +++ b/storage/src/Storage.Sqlite/Internal/Log.cs @@ -0,0 +1,83 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Microsoft.Extensions.Logging; + +namespace Duende.Storage.Sqlite.Internal; + +internal static partial class Log +{ + [LoggerMessage(Level = LogLevel.Information, Message = $"Checking schema version")] + internal static partial void CheckingSchemaVersion(ILogger logger); + + [LoggerMessage(Level = LogLevel.Information, Message = $"Creating schema {{{Parameters.SchemaName}}}")] + internal static partial void CreatingSchema(ILogger logger, string schemaName); + + [LoggerMessage(Level = LogLevel.Information, Message = $"Migrating schema {{{Parameters.SchemaName}}}")] + internal static partial void MigratingSchema(ILogger logger, string schemaName); + + [LoggerMessage(Level = LogLevel.Information, Message = $"Verifying schema {{{Parameters.SchemaName}}}")] + internal static partial void VerifyingSchema(ILogger logger, string schemaName); + + [LoggerMessage(Level = LogLevel.Warning, Message = $"Error While creating schema")] + internal static partial void ErrorWhileCreatingSchema(ILogger logger, Exception e); + + [LoggerMessage(Level = LogLevel.Debug, Message = $"Executing sql {{{Parameters.Sql}}}")] + internal static partial void ExecutingSql(ILogger logger, string sql); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Creating DSO: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Id)}={{{Parameters.Id}}}, {nameof(Parameters.DsoSchemaVersion)}={{{Parameters.DsoSchemaVersion}}}")] + internal static partial void CreatingDso(ILogger logger, EntityType entityType, Guid id, uint dsoSchemaVersion); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Deleting DSO: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Id)}={{{Parameters.Id}}}")] + internal static partial void DeletingDso(ILogger logger, EntityType entityType, Guid id); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Reading DSO: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Id)}={{{Parameters.Id}}}")] + internal static partial void ReadingDso(ILogger logger, EntityType entityType, Guid id); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Reading DSOs: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Count)}={{{Parameters.Count}}}")] + internal static partial void ReadingDsos(ILogger logger, EntityType entityType, int count); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Updating DSO: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.Id)}={{{Parameters.Id}}}, {nameof(Parameters.DsoSchemaVersion)}={{{Parameters.DsoSchemaVersion}}}, {nameof(Parameters.ExpectedEntityVersion)}={{{Parameters.ExpectedEntityVersion}}}")] + internal static partial void UpdatingDso(ILogger logger, EntityType entityType, Guid id, uint dsoSchemaVersion, int expectedEntityVersion); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Querying DSOs: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.PageNumber)}={{{Parameters.PageNumber}}}, {nameof(Parameters.PageSize)}={{{Parameters.PageSize}}}")] + internal static partial void QueryingDsos(ILogger logger, EntityType entityType, int pageNumber, int pageSize); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Querying DSO fields: {nameof(Parameters.EntityType)}={{{Parameters.EntityType}}}, {nameof(Parameters.FieldCount)}={{{Parameters.FieldCount}}}, {nameof(Parameters.PageNumber)}={{{Parameters.PageNumber}}}, {nameof(Parameters.PageSize)}={{{Parameters.PageSize}}}")] + internal static partial void QueryingFieldsDsos(ILogger logger, EntityType entityType, int fieldCount, int pageNumber, int pageSize); + + [LoggerMessage( + Level = LogLevel.Information, + Message = $"Executing query: {nameof(Parameters.Query)}={{{Parameters.Query}}}")] + internal static partial void ExecutingQuery(ILogger logger, string query); + + private static class Parameters + { + internal const string Count = nameof(Count); + internal const string DsoSchemaVersion = nameof(DsoSchemaVersion); + internal const string EntityType = nameof(EntityType); + internal const string ExpectedEntityVersion = nameof(ExpectedEntityVersion); + internal const string FieldCount = nameof(FieldCount); + internal const string Id = nameof(Id); + internal const string PageNumber = nameof(PageNumber); + internal const string PageSize = nameof(PageSize); + internal const string Query = nameof(Query); + internal const string SchemaName = nameof(SchemaName); + internal const string Sql = nameof(Sql); + } +} diff --git a/storage/src/Storage.Sqlite/Internal/MigrationScriptLoader.cs b/storage/src/Storage.Sqlite/Internal/MigrationScriptLoader.cs new file mode 100644 index 000000000..1cfc7a676 --- /dev/null +++ b/storage/src/Storage.Sqlite/Internal/MigrationScriptLoader.cs @@ -0,0 +1,42 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Globalization; +using System.Reflection; +using System.Text.RegularExpressions; + +namespace Duende.Storage.Sqlite.Internal; + +internal static class MigrationScriptLoader +{ + private static readonly Regex VersionPattern = new(@"\.Migrations\.V(\d+)_", RegexOptions.Compiled); + + public static IEnumerable<(int TargetVersion, string Sql)> GetScripts( + Assembly assembly, + DatabaseSchemaVersion fromVersion) + { + var assemblyName = assembly.GetName().Name; + var prefix = $"{assemblyName}.Migrations.V"; + + return assembly.GetManifestResourceNames() + .Where(name => name.StartsWith(prefix, StringComparison.Ordinal) && name.EndsWith(".sql", StringComparison.Ordinal)) + .Select(name => (Name: name, Version: ParseVersion(name))) + .Where(x => x.Version > fromVersion.Value) + .OrderBy(x => x.Version) + .Select(x => (x.Version, ReadResource(assembly, x.Name))); + } + + private static int ParseVersion(string resourceName) + { + var match = VersionPattern.Match(resourceName); + return match.Success ? int.Parse(match.Groups[1].Value, CultureInfo.InvariantCulture) : 0; + } + + private static string ReadResource(Assembly assembly, string resourceName) + { + using var stream = assembly.GetManifestResourceStream(resourceName) + ?? throw new InvalidOperationException($"Embedded resource '{resourceName}' not found."); + using var reader = new StreamReader(stream); + return reader.ReadToEnd(); + } +} diff --git a/storage/src/Storage.Sqlite/Internal/SqliteDialect.cs b/storage/src/Storage.Sqlite/Internal/SqliteDialect.cs new file mode 100644 index 000000000..21986dfb2 --- /dev/null +++ b/storage/src/Storage.Sqlite/Internal/SqliteDialect.cs @@ -0,0 +1,53 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Data.Common; +using Duende.Storage.Internal.Querying; +using Microsoft.Data.Sqlite; + +namespace Duende.Storage.Sqlite.Internal; + +/// +/// SQLite-specific SQL dialect implementation. +/// +internal sealed class SqliteDialect : ISqlDialect +{ + public string CaseInsensitiveLikeOperator => "LIKE"; + + public string LikeEscapeClause => " ESCAPE '\\'"; + + public string TrueLiteral => "1"; + + public string FalseLiteral => "0"; + + public string EscapeLikeWildcards(string value) + { + if (string.IsNullOrEmpty(value)) + { + return value; + } + + // SQLite uses backslash as escape character (like PostgreSQL) + // Replace backslash first to avoid double-escaping + return value + .Replace("\\", "\\\\", StringComparison.OrdinalIgnoreCase) + .Replace("%", "\\%", StringComparison.OrdinalIgnoreCase) + .Replace("_", "\\_", StringComparison.OrdinalIgnoreCase); + } + + public object FieldPathToParameterValue(Guid fieldPathId) => fieldPathId.ToByteArray(); + + public void AddParameter(DbCommand command, string name, object value) + { + var sqliteCommand = (SqliteCommand)command; + + _ = value switch + { + DateTimeOffset dto => sqliteCommand.Parameters.AddWithValue(name, dto.UtcDateTime.ToString("O")), + DateTime dt => sqliteCommand.Parameters.AddWithValue(name, new DateTimeOffset(dt, TimeSpan.Zero).UtcDateTime.ToString("O")), + Guid guid => sqliteCommand.Parameters.AddWithValue(name, guid.ToString()), + bool b => sqliteCommand.Parameters.AddWithValue(name, b ? 1 : 0), + _ => sqliteCommand.Parameters.AddWithValue(name, value) + }; + } +} diff --git a/storage/src/Storage.Sqlite/Internal/SqliteStore.cs b/storage/src/Storage.Sqlite/Internal/SqliteStore.cs new file mode 100644 index 000000000..5c17932ad --- /dev/null +++ b/storage/src/Storage.Sqlite/Internal/SqliteStore.cs @@ -0,0 +1,2438 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Globalization; +using System.Text; +using System.Text.Json; +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Outbox; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Expressions; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Internal.Querying.SearchFields; +using Duende.Storage.Internal.Querying.Sorting; +using Duende.Storage.Pagination; +using Duende.Storage.Querying; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging; +using OutboxEventId = Duende.Storage.Internal.Outbox.OutboxEventId; +using OutboxEventName = Duende.Storage.Internal.Outbox.OutboxEventName; +using SubscriberName = Duende.Storage.Internal.Outbox.SubscriberName; + +namespace Duende.Storage.Sqlite.Internal; + +#pragma warning disable CA1812 // Avoid uninstantiated internal classes +#pragma warning disable CA2100 // Review SQL queries for security vulnerabilities +internal sealed class SqliteStore( + SqliteStoreOptions options, + DataStorageTypeRegistry dataStorageTypeRegistry, + TimeProvider timeProvider, + OutboxSubscribers outboxSubscribers, + ILogger logger) : StoreBase, IStore, IDatabaseSchema +{ + private const int RequiredSchemaVersion = 1; + private static readonly SqliteDialect Dialect = new(); + + // SQLite's default schema is "main", used for SqlWhereClauseBuilder schema-qualified table names + private const string SchemaName = "main"; + + /// + /// Creates, opens, and configures a new SQLite connection. + /// Enables foreign key enforcement which is OFF by default in SQLite. + /// + private async Task OpenConnectionAsync(Ct ct) + { + var cnn = new SqliteConnection(options.ConnectionString); + await cnn.OpenAsync(ct); + + await using var pragmaCmd = cnn.CreateCommand(); + pragmaCmd.CommandText = "PRAGMA foreign_keys = ON"; + _ = await pragmaCmd.ExecuteNonQueryAsync(ct); + + return cnn; + } + + async Task IDatabaseSchema.CheckVersionAsync(Ct ct) + { + Log.CheckingSchemaVersion(logger); + + await using var connection = await OpenConnectionAsync(ct); + + // Check if __schema_info table exists + await using var checkCmd = connection.CreateCommand(); + checkCmd.CommandText = "SELECT name FROM sqlite_master WHERE type='table' AND name='__schema_info'"; + Log.ExecutingSql(logger, checkCmd.CommandText); + var tableExists = await checkCmd.ExecuteScalarAsync(ct); + + if (tableExists is null) + { + return new CheckSchemaVersionResult(0, RequiredSchemaVersion); + } + + await using var cmd = connection.CreateCommand(); + cmd.CommandText = "SELECT value FROM __schema_info WHERE key = 'SchemaVersion'"; + Log.ExecutingSql(logger, cmd.CommandText); + var scalar = await cmd.ExecuteScalarAsync(ct); + + if (scalar is null or DBNull) + { + return new CheckSchemaVersionResult(0, RequiredSchemaVersion); + } + + try + { + var schemaComment = JsonSerializer.Deserialize((string)scalar)!; + return new CheckSchemaVersionResult((uint)schemaComment.Version, RequiredSchemaVersion); + } + catch (JsonException ex) + { + throw new InvalidOperationException("Invalid database schema version info", ex); + } + } + + async Task IDatabaseSchema.MigrateAsync(Ct ct) + { + Log.MigratingSchema(logger, "sqlite"); + + var versionResult = await ((IDatabaseSchema)this).CheckVersionAsync(ct); + var currentVersion = new DatabaseSchemaVersion((int)versionResult.CurrentVersion); + + await using var connection = await OpenConnectionAsync(ct); + + foreach (var (_, sql) in MigrationScriptLoader.GetScripts(typeof(SqliteStore).Assembly, currentVersion)) + { + await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(ct); + try + { + await using var cmd = connection.CreateCommand(); + cmd.Transaction = transaction; + cmd.CommandText = sql; + Log.ExecutingSql(logger, cmd.CommandText); + _ = await cmd.ExecuteNonQueryAsync(ct); + + await transaction.CommitAsync(ct); + } + catch + { + await transaction.RollbackAsync(ct); + throw; + } + } + + var verifyResult = await ((IDatabaseSchema)this).VerifySchemaAsync(ct); + if (!verifyResult.IsValid) + { + var errors = string.Join("; ", verifyResult.Errors.Select(e => e.ErrorMessage)); + throw new InvalidOperationException($"Schema verification failed after migration: {errors}"); + } + } + + async Task IDatabaseSchema.VerifySchemaAsync(Ct ct) + { + Log.VerifyingSchema(logger, "sqlite"); + + var errors = new List(); + + await using var connection = await OpenConnectionAsync(ct); + + // Expected tables and their required columns (name -> affinity/type hint) + var expectedTables = new Dictionary> + { + ["__schema_info"] = + [ + ("key", "TEXT"), + ("value", "TEXT") + ], + ["entities"] = + [ + ("pool_id", "INTEGER"), + ("entity_type_id", "INTEGER"), + ("entity_id", "TEXT"), + ("entity_type_name", "TEXT"), + ("value", "TEXT"), + ("dso_type_schema_version", "INTEGER"), + ("value_version", "INTEGER"), + ("created_at", "TEXT"), + ("last_updated_at", "TEXT"), + ("expires_at", "TEXT") + ], + ["entity_keys"] = + [ + ("pool_id", "INTEGER"), + ("entity_type_id", "INTEGER"), + ("key_type_id", "INTEGER"), + ("key_type_version", "INTEGER"), + ("key_type_name", "TEXT"), + ("key_value", "TEXT"), + ("key_json", "TEXT"), + ("entity_id", "TEXT"), + ("timestamp", "TEXT") + ], + ["search_values"] = + [ + ("entity_type_id", "INTEGER"), + ("entity_id", "TEXT"), + ("field_path", "BLOB"), + ("field_path_text", "TEXT"), + ("item_index", "INTEGER"), + ("string_value", "TEXT"), + ("number_value", "REAL"), + ("datetime_value", "TEXT"), + ("boolean_value", "INTEGER"), + ("guid_value", "TEXT"), + ("pool_id", "INTEGER") + ], + ["entity_links"] = + [ + ("pool_id", "INTEGER"), + ("link_type_id", "INTEGER"), + ("left_entity_type_id", "INTEGER"), + ("left_entity_id", "TEXT"), + ("right_entity_type_id", "INTEGER"), + ("right_entity_id", "TEXT"), + ("created_at", "TEXT") + ], + ["outbox_subscriber_queue"] = + [ + ("sequence_number", "INTEGER"), + ("message_id", "TEXT"), + ("event_id", "TEXT"), + ("timestamp", "TEXT"), + ("event_name", "TEXT"), + ("subject_id", "TEXT"), + ("entity_type_id", "INTEGER"), + ("entity_type_name", "TEXT"), + ("pool_id", "INTEGER"), + ("payload", "TEXT"), + ("subscriber_name", "TEXT") + ] + }; + + // Check each expected table + foreach (var (tableName, expectedColumns) in expectedTables) + { + await using var tableCmd = connection.CreateCommand(); + tableCmd.CommandText = "SELECT name FROM sqlite_master WHERE type='table' AND name=@name"; + _ = tableCmd.Parameters.AddWithValue("@name", tableName); + var tableExists = await tableCmd.ExecuteScalarAsync(ct); + + if (tableExists is null) + { + errors.Add(new SchemaVerificationError(tableName, null, $"Table '{tableName}' is missing.", SchemaVerificationErrorKind.MissingTable)); + continue; + } + + // Check columns via PRAGMA table_info + await using var colCmd = connection.CreateCommand(); + colCmd.CommandText = $"PRAGMA table_info({tableName})"; + await using var colReader = await colCmd.ExecuteReaderAsync(ct); + + var actualColumns = new Dictionary(StringComparer.OrdinalIgnoreCase); + while (await colReader.ReadAsync(ct)) + { + var colName = colReader.GetString(1); + var colType = colReader.GetString(2).ToUpperInvariant(); + actualColumns[colName] = colType; + } + + foreach (var (colName, typeHint) in expectedColumns) + { + if (!actualColumns.TryGetValue(colName, out var actualType)) + { + errors.Add(new SchemaVerificationError(tableName, colName, $"Column '{tableName}.{colName}' is missing.", SchemaVerificationErrorKind.MissingColumn)); + } + else if (!actualType.Contains(typeHint, StringComparison.OrdinalIgnoreCase)) + { + errors.Add(new SchemaVerificationError(tableName, colName, $"Column '{tableName}.{colName}' has type '{actualType}', expected '{typeHint}'.", SchemaVerificationErrorKind.WrongType)); + } + } + } + + // Verify required indexes + var expectedIndexes = new (string Table, string Index)[] + { + ("entities", "entities_expires_at_index"), + ("entities", "entities_created_at_index"), + ("entities", "entities_last_updated_at_index"), + ("entity_keys", "entity_keys_entity_type_id_entity_id_index"), + ("search_values", "search_values_string_value_index"), + ("search_values", "search_values_number_value_index"), + ("search_values", "search_values_datetime_value_index"), + ("search_values", "search_values_boolean_value_index"), + ("search_values", "search_values_array_string_value_index"), + ("search_values", "search_values_array_number_value_index"), + ("search_values", "search_values_array_datetime_value_index"), + ("search_values", "search_values_array_boolean_value_index"), + ("search_values", "search_values_guid_value_index"), + ("search_values", "search_values_array_guid_value_index"), + ("entity_links", "entity_links_left_entity_index"), + ("entity_links", "entity_links_right_entity_index"), + ("entity_links", "entity_links_left_cascade_index"), + ("entity_links", "entity_links_right_cascade_index"), + ("outbox_subscriber_queue", "outbox_subscriber_queue_subscriber_index"), + }; + + { + await using var idxCmd = connection.CreateCommand(); + idxCmd.CommandText = "SELECT tbl_name, name FROM sqlite_master WHERE type='index' AND name IS NOT NULL"; + await using var idxReader = await idxCmd.ExecuteReaderAsync(ct); + + var actualIndexes = new HashSet<(string Table, string Index)>( + EqualityComparer<(string, string)>.Create( + (a, b) => string.Equals(a.Item1, b.Item1, StringComparison.OrdinalIgnoreCase) && + string.Equals(a.Item2, b.Item2, StringComparison.OrdinalIgnoreCase), + x => StringComparer.OrdinalIgnoreCase.GetHashCode(x.Item1) ^ StringComparer.OrdinalIgnoreCase.GetHashCode(x.Item2))); + + while (await idxReader.ReadAsync(ct)) + { + _ = actualIndexes.Add((idxReader.GetString(0), idxReader.GetString(1))); + } + + foreach (var (tableName, indexName) in expectedIndexes) + { + if (!actualIndexes.Contains((tableName, indexName))) + { + errors.Add(new SchemaVerificationError( + tableName, null, + $"Index '{indexName}' is missing from table '{tableName}'.", + SchemaVerificationErrorKind.MissingIndex)); + } + } + } + + // Verify required foreign keys + var expectedForeignKeys = new (string Table, string ReferencedTable)[] + { + ("entity_keys", "entities"), + ("search_values", "entities"), + }; + + foreach (var (tableName, referencedTable) in expectedForeignKeys) + { + await using var fkCmd = connection.CreateCommand(); + fkCmd.CommandText = $"PRAGMA foreign_key_list({tableName})"; + await using var fkReader = await fkCmd.ExecuteReaderAsync(ct); + + var referencedTables = new HashSet(StringComparer.OrdinalIgnoreCase); + while (await fkReader.ReadAsync(ct)) + { + _ = referencedTables.Add(fkReader.GetString(2)); // column 2 is "table" (referenced table) + } + + if (!referencedTables.Contains(referencedTable)) + { + errors.Add(new SchemaVerificationError( + tableName, null, + $"Foreign key from '{tableName}' to '{referencedTable}' is missing.", + SchemaVerificationErrorKind.MissingForeignKey)); + } + } + + return new SchemaVerificationResult(errors); + } + + string IDatabaseSchema.BuildMigrationScript(DatabaseSchemaVersion fromVersion) + { + var sb = new StringBuilder(); + foreach (var (targetVersion, sql) in MigrationScriptLoader.GetScripts(typeof(SqliteStore).Assembly, fromVersion)) + { + _ = sb.AppendLine(CultureInfo.InvariantCulture, $"-- Migration step: V{targetVersion - 1} → V{targetVersion}"); + _ = sb.AppendLine(sql); + } + + return sb.ToString(); + } + + /// + /// Creates a new entity in the store. + /// + async Task IStore.CreateAsync( + Storage.UuidV7 id, + TDso dso, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration expiration, + IReadOnlyList outboxEvents, + Ct ct) + { + var createOp = CreateOperation.For(id, dso, keys, searchFieldCollection, expiration); + + await using var cnn = await OpenConnectionAsync(ct); + await using var tx = (SqliteTransaction)await cnn.BeginTransactionAsync(ct); + + var outcome = await ExecuteCreateCoreAsync(cnn, tx, createOp, ct); + + if (outcome == OperationOutcome.Success) + { + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(cnn, tx, outboxEvents, ct); + } + await tx.CommitAsync(ct); + } + + return outcome switch + { + OperationOutcome.Success => CreateResult.Success, + OperationOutcome.AlreadyExists => CreateResult.AlreadyExists, + OperationOutcome.KeyConflict => CreateResult.KeyConflict, + _ => throw new InvalidOperationException($"Unexpected outcome from create operation: {outcome}") + }; + } + + async Task IStore.TryReadAsync( + EntityType entityType, + Storage.UuidV7 id, + Ct ct) + { + Log.ReadingDso(logger, entityType, id.Value); + + + await using var cnn = await OpenConnectionAsync(ct); + + await using var cmd = cnn.CreateCommand(); + cmd.CommandText = """ + SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at + FROM main.entities v + WHERE v.entity_type_id = @entity_type_id AND v.entity_id = @entity_id AND v.pool_id = @pool_id + """; + + _ = cmd.Parameters.AddWithValue("@entity_type_id", (int)entityType.Id); + _ = cmd.Parameters.AddWithValue("@entity_id", id.Value.ToString()); + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + Log.ExecutingSql(logger, cmd.CommandText); + await using var reader = await cmd.ExecuteReaderAsync(ct); + + if (!await reader.ReadAsync(ct)) + { + return StoreGetResult.NotFound(); + } + + var entityId = Guid.Parse(reader.GetString(0)); + var jsonValue = reader.GetString(1); + var dsoTypeVersion = reader.GetInt32(2); + var valueVersion = reader.GetInt32(3); + var created = DateTimeOffset.Parse(reader.GetString(4), null, DateTimeStyles.RoundtripKind); + var lastUpdated = DateTimeOffset.Parse(reader.GetString(5), null, DateTimeStyles.RoundtripKind); + + var version = new DataStorageObjectVersion(entityType, (uint)dsoTypeVersion); + var dsoType = dataStorageTypeRegistry.Get(version); + var item = (IDataStorageObject)JsonSerializer.Deserialize(jsonValue, dsoType)!; + + return StoreGetResult.IsFound(item, entityId, valueVersion, created, lastUpdated); + } + + async Task IStore.TryReadAsync( + EntityType entityType, + DataStorageKey key, + Ct ct) + { + Log.ReadingDso(logger, entityType, key.Value); + + var keyGuid = key.Value; + var keyTypeId = (int)key.DskVersion.KeyType.Id; + var keyTypeVersion = (int)key.DskVersion.SchemaVersion; + + await using var cnn = await OpenConnectionAsync(ct); + + await using var cmd = cnn.CreateCommand(); + cmd.CommandText = """ + SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at + FROM main.entity_keys i + INNER JOIN main.entities v ON i.entity_type_id = v.entity_type_id AND i.entity_id = v.entity_id + WHERE i.entity_type_id = @entity_type_id + AND i.key_type_id = @key_type_id + AND i.key_type_version = @key_type_version + AND i.key_value = @key_value + AND i.pool_id = @pool_id + AND v.pool_id = @pool_id + """; + + _ = cmd.Parameters.AddWithValue("@entity_type_id", (int)entityType.Id); + _ = cmd.Parameters.AddWithValue("@key_type_id", keyTypeId); + _ = cmd.Parameters.AddWithValue("@key_type_version", keyTypeVersion); + _ = cmd.Parameters.AddWithValue("@key_value", keyGuid.ToString()); + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + + Log.ExecutingSql(logger, cmd.CommandText); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + + if (!await reader.ReadAsync(ct)) + { + return StoreGetResult.NotFound(); + } + + var entityId = Guid.Parse(reader.GetString(0)); + var jsonValue = reader.GetString(1); + var dsoTypeVersion = reader.GetInt32(2); + var valueVersion = reader.GetInt32(3); + var created = DateTimeOffset.Parse(reader.GetString(4), null, DateTimeStyles.RoundtripKind); + var lastUpdated = DateTimeOffset.Parse(reader.GetString(5), null, DateTimeStyles.RoundtripKind); + + var version = new DataStorageObjectVersion(entityType, (uint)dsoTypeVersion); + var dsoType = dataStorageTypeRegistry.Get(version); + var item = (IDataStorageObject)JsonSerializer.Deserialize(jsonValue, dsoType)!; + + return StoreGetResult.IsFound(item, entityId, valueVersion, created, lastUpdated); + } + + async Task> IStore.TryReadManyAsync( + EntityType entityType, + IReadOnlySet ids, + int maximum, + Ct ct) + { + if (ids.Count > maximum) + { + throw new InvalidOperationException( + $"The number of requested IDs ({ids.Count}) exceeds the maximum allowed ({maximum})."); + } + + Log.ReadingDsos(logger, entityType, ids.Count); + + await using var cnn = await OpenConnectionAsync(ct); + + await using var cmd = cnn.CreateCommand(); + + // Build IN clause with individual parameters (SQLite doesn't support array parameters) + var idParams = new List(); + var i = 0; + foreach (var id in ids) + { + var paramName = $"@id{i}"; + idParams.Add(paramName); + _ = cmd.Parameters.AddWithValue(paramName, id.Value.ToString()); + i++; + } + + var inClause = string.Join(", ", idParams); + cmd.CommandText = $""" + SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at + FROM main.entities v + WHERE v.entity_type_id = @entityTypeId AND v.entity_id IN ({inClause}) AND v.pool_id = @poolId + """; + + _ = cmd.Parameters.AddWithValue("@entityTypeId", (int)entityType.Id); + _ = cmd.Parameters.AddWithValue("@poolId", PoolId.Value); + + Log.ExecutingSql(logger, cmd.CommandText); + + await using var reader = await cmd.ExecuteReaderAsync(ct); + + var results = new List(); + while (await reader.ReadAsync(ct)) + { + var entityId = Guid.Parse(reader.GetString(0)); + var jsonValue = reader.GetString(1); + var dsoTypeVersion = reader.GetInt32(2); + var valueVersion = reader.GetInt32(3); + var created = DateTimeOffset.Parse(reader.GetString(4), null, DateTimeStyles.RoundtripKind); + var lastUpdated = DateTimeOffset.Parse(reader.GetString(5), null, DateTimeStyles.RoundtripKind); + + var version = new DataStorageObjectVersion(entityType, (uint)dsoTypeVersion); + var dsoType = dataStorageTypeRegistry.Get(version); + var item = (IDataStorageObject)JsonSerializer.Deserialize(jsonValue, dsoType)!; + + results.Add(StoreGetResult.IsFound(item, entityId, valueVersion, created, lastUpdated)); + } + + return results; + } + + /// + /// Updates an existing entity in the store. + /// + async Task IStore.UpdateAsync( + Storage.UuidV7 id, + TDso dso, + int expectedEntityVersion, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration? expiration, + IReadOnlyList outboxEvents, + Ct ct) + { + var updateOp = UpdateOperation.For(id, dso, expectedEntityVersion, keys, searchFieldCollection, expiration); + + await using var cnn = await OpenConnectionAsync(ct); + await using var tx = (SqliteTransaction)await cnn.BeginTransactionAsync(ct); + + var outcome = await ExecuteUpdateCoreAsync(cnn, tx, updateOp, ct); + + if (outcome == OperationOutcome.Success) + { + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(cnn, tx, outboxEvents, ct); + } + await tx.CommitAsync(ct); + } + + return outcome switch + { + OperationOutcome.Success => UpdateResult.Success, + OperationOutcome.DoesNotExist => UpdateResult.DoesNotExist, + OperationOutcome.UnexpectedVersion => UpdateResult.UnexpectedVersion, + OperationOutcome.KeyConflict => UpdateResult.KeyConflict, + _ => throw new InvalidOperationException($"Unexpected outcome from update operation: {outcome}") + }; + } + + async Task IStore.DeleteAsync(EntityType entityType, Storage.UuidV7 id, IReadOnlyList outboxEvents, Ct ct) + { + var deleteOp = DeleteOperation.ById(entityType, id); + + await using var cnn = await OpenConnectionAsync(ct); + await using var tx = (SqliteTransaction)await cnn.BeginTransactionAsync(ct); + + var (_, entityDeleted) = await ExecuteDeleteCoreAsync(cnn, tx, deleteOp, ct); + + if (entityDeleted && outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(cnn, tx, outboxEvents, ct); + } + + await tx.CommitAsync(ct); + return DeleteResult.Success; + } + + async Task IStore.DeleteAsync(EntityType entityType, DataStorageKey key, IReadOnlyList outboxEvents, Ct ct) + { + var deleteOp = DeleteOperation.ByKey(entityType, key); + + await using var cnn = await OpenConnectionAsync(ct); + await using var tx = (SqliteTransaction)await cnn.BeginTransactionAsync(ct); + + var (_, entityDeleted) = await ExecuteDeleteCoreAsync(cnn, tx, deleteOp, ct); + + if (entityDeleted && outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(cnn, tx, outboxEvents, ct); + } + + await tx.CommitAsync(ct); + return DeleteResult.Success; + } + + private static int AddInserts(StringBuilder builder, + SqliteCommand cmd, + IReadOnlyCollection keys) + { + var idNumber = 0; + + foreach (var key in keys) + { + var keyTypeIdParam = $"@key_type_id{idNumber}"; + var keyTypeNameParam = $"@key_type_name{idNumber}"; + var keyTypeVersionParam = $"@key_type_version{idNumber}"; + var keyValueParam = $"@key{idNumber}"; + var keyStringParam = $"@key_json{idNumber}"; + + _ = builder.AppendLine( + CultureInfo.InvariantCulture, + $""" + INSERT INTO main.entity_keys (entity_type_id, key_type_id, key_type_name, key_value, key_json, key_type_version, entity_id, pool_id) + VALUES (@entity_type_id, {keyTypeIdParam}, {keyTypeNameParam}, {keyValueParam}, {keyStringParam}, {keyTypeVersionParam}, @entity_id, @pool_id); + """); + + _ = cmd.Parameters.AddWithValue(keyTypeIdParam, (int)key.DskVersion.KeyType.Id); + _ = cmd.Parameters.AddWithValue(keyTypeNameParam, key.DskVersion.KeyType.Name); + _ = cmd.Parameters.AddWithValue(keyValueParam, key.Value.ToString()); + _ = cmd.Parameters.AddWithValue(keyStringParam, (object?)key.KeyJsonValue ?? DBNull.Value); + _ = cmd.Parameters.AddWithValue(keyTypeVersionParam, (int)key.DskVersion.SchemaVersion); + + ++idNumber; + } + + return idNumber; + } + + private static int AddSearchFieldInserts( + StringBuilder builder, + SqliteCommand cmd, + SearchFieldCollection? searchFields) + { + if (searchFields is null || searchFields.Count == 0) + { + return 0; + } + + var fieldNumber = 0; + foreach (var field in searchFields) + { + var fieldPathParam = $"@field_path{fieldNumber}"; + var fieldPathTextParam = $"@field_path_text{fieldNumber}"; + var itemIndexParam = $"@item_index{fieldNumber}"; + var stringValueParam = $"@string_value{fieldNumber}"; + var numberValueParam = $"@number_value{fieldNumber}"; + var datetimeValueParam = $"@datetime_value{fieldNumber}"; + var booleanValueParam = $"@boolean_value{fieldNumber}"; + var guidValueParam = $"@guid_value{fieldNumber}"; + + _ = builder.AppendLine( + CultureInfo.InvariantCulture, + $""" + INSERT INTO main.search_values (entity_type_id, entity_id, field_path, field_path_text, item_index, string_value, number_value, datetime_value, boolean_value, guid_value, pool_id) + VALUES (@entity_type_id, @entity_id, {fieldPathParam}, {fieldPathTextParam}, {itemIndexParam}, {stringValueParam}, {numberValueParam}, {datetimeValueParam}, {booleanValueParam}, {guidValueParam}, @pool_id); + """); + + _ = cmd.Parameters.AddWithValue(fieldPathParam, field.FieldPathId.ToByteArray()); + _ = cmd.Parameters.AddWithValue(fieldPathTextParam, field.FieldPath); + _ = cmd.Parameters.AddWithValue(itemIndexParam, field.ItemIndex ?? -1); + _ = cmd.Parameters.AddWithValue(stringValueParam, (object?)field.StringValue ?? DBNull.Value); + _ = cmd.Parameters.AddWithValue(numberValueParam, + field.NumberValue.HasValue ? field.NumberValue.Value : DBNull.Value); + + // Convert DateTimeOffset to ISO 8601 text for SQLite + if (field.DateTimeValue.HasValue) + { + _ = cmd.Parameters.AddWithValue(datetimeValueParam, + field.DateTimeValue.Value.UtcDateTime.ToString("O")); + } + else + { + _ = cmd.Parameters.AddWithValue(datetimeValueParam, DBNull.Value); + } + + _ = cmd.Parameters.AddWithValue(booleanValueParam, + field.BooleanValue.HasValue ? field.BooleanValue.Value ? 1 : 0 : DBNull.Value); + + _ = cmd.Parameters.AddWithValue(guidValueParam, + field.GuidValue.HasValue ? field.GuidValue.Value.ToString() : DBNull.Value); + + ++fieldNumber; + } + + return fieldNumber; + } + + /// + /// Queries entities with the specified pagination strategy. + /// + async Task>> IStore.QueryAsync( + EntityType entityType, + IQueryExpression filter, + SortParameter sort, + DataRange dataRange, + Ct ct) + { + if (dataRange.TokenValue is not null) + { + return await QueryCursorCoreAsync(entityType, filter, sort, dataRange.TokenValue, ct); + } + + var (skip, take) = ResolveOffsetAndSize(dataRange); + var dsoVersion = TDso.DsoVersion; + var entityTypeId = (int)entityType.Id; + + Log.QueryingDsos(logger, entityType, skip, take); + + await using var cnn = await OpenConnectionAsync(ct); + + await using var cmd = cnn.CreateCommand(); + var queryClauses = BuildQueryClauses(cmd, filter, sort); + + // Build main query using CTEs + string allMatchesSelect; + string pagedOrderBy; + string outerOrderBy; + if (!sort.IsEmpty) + { + var sortColumn = GetSortColumnName(sort.Field!); + var sortDirection = sort.Direction == SortDirection.Ascending ? "ASC" : "DESC"; + allMatchesSelect = $"SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at, {sortColumn} AS sort_value"; + pagedOrderBy = $"ORDER BY CASE WHEN sort_value IS NULL THEN 1 ELSE 0 END, sort_value {sortDirection}, entity_id ASC"; + outerOrderBy = $"ORDER BY CASE WHEN p.sort_value IS NULL THEN 1 ELSE 0 END, p.sort_value {sortDirection}, p.entity_id ASC"; + } + else + { + allMatchesSelect = "SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at"; + pagedOrderBy = "ORDER BY entity_id ASC"; + outerOrderBy = "ORDER BY p.entity_id ASC"; + } + + var query = $""" + WITH all_matches AS ( + {allMatchesSelect} + FROM main.entities v + {queryClauses.JoinClause} + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({queryClauses.WhereClause}) + ), + total AS ( + SELECT COUNT(*) AS total_count FROM all_matches + ), + paged AS ( + SELECT * FROM all_matches + {pagedOrderBy} + LIMIT @limit OFFSET @offset + ) + SELECT p.entity_id, p.value, p.dso_type_schema_version, p.value_version, p.created_at, p.last_updated_at, t.total_count + FROM total t + LEFT JOIN paged p ON 1 + {outerOrderBy} + """; + + Dialect.AddParameter(cmd, "@entity_type_id", entityTypeId); + Dialect.AddParameter(cmd, "@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@offset", skip); + _ = cmd.Parameters.AddWithValue("@limit", take); + + cmd.CommandText = query; + + Log.ExecutingQuery(logger, query); + + var items = new List>(); + var totalCount = 0; + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + var dsoType = dataStorageTypeRegistry.Get(dsoVersion); + while (await reader.ReadAsync(ct)) + { + if (totalCount == 0) + { + totalCount = Convert.ToInt32(reader.GetInt64(6)); + } + + if (await reader.IsDBNullAsync(0, ct)) + { + continue; + } + + var entityId = Guid.Parse(reader.GetString(0)); + var jsonValue = reader.GetString(1); + var item = (TDso)JsonSerializer.Deserialize(jsonValue, dsoType)!; + var valueVersion = reader.GetInt32(3); + var created = DateTimeOffset.Parse(reader.GetString(4), null, DateTimeStyles.RoundtripKind); + var lastUpdated = DateTimeOffset.Parse(reader.GetString(5), null, DateTimeStyles.RoundtripKind); + items.Add(new MetadataEnvelope(item, entityId, valueVersion, created, lastUpdated)); + } + } + + return new QueryResult> + { + Items = items, + TotalCount = totalCount, + TotalPages = (int)Math.Ceiling((double)totalCount / take), + HasMoreData = skip + take < totalCount + }; + } + + /// + /// Queries for specific field values with the specified pagination strategy. + /// + async Task> IStore.QueryFieldsAsync( + EntityType entityType, + IReadOnlyCollection fields, + IQueryExpression filter, + SortParameter sort, + DataRange dataRange, + Ct ct) + { + if (dataRange.TokenValue is not null) + { + return await QueryFieldsCursorCoreAsync(entityType, fields, filter, sort, dataRange.TokenValue, ct); + } + + var (skip, take) = ResolveOffsetAndSize(dataRange); + var entityTypeId = (int)entityType.Id; + + Log.QueryingFieldsDsos(logger, entityType, fields.Count, skip, take); + + await using var cnn = await OpenConnectionAsync(ct); + + await using var cmd = cnn.CreateCommand(); + var queryClauses = BuildQueryClauses(cmd, filter, sort); + + var fieldPaths = fields.Select(f => f.Path).ToList(); + var fieldConditions = new List(); + var paramIndex = 0; + for (var i = 0; i < fieldPaths.Count; i++) + { + if (SystemFields.IsSystemField(fieldPaths[i])) + { + continue; + } + + _ = cmd.Parameters.AddWithValue($"@select_field_{paramIndex}", DeterministicGuidGenerator.Create(fieldPaths[i].ToUpperInvariant()).ToByteArray()); + fieldConditions.Add($"field_sv.field_path = @select_field_{paramIndex}"); + paramIndex++; + } + var fieldConditionsClause = fieldConditions.Count > 0 + ? string.Join(" OR ", fieldConditions) + : "1=0"; + + string cteSelect; + string cteJoin; + + if (!sort.IsEmpty) + { + cteJoin = queryClauses.JoinClause; + var sortColumn = GetSortColumnName(sort.Field!); + cteSelect = $"SELECT v.entity_id, v.created_at, v.last_updated_at, v.value_version, {sortColumn} AS sort_value, ROW_NUMBER() OVER ({queryClauses.OrderByClause}) AS row_num"; + } + else + { + cteSelect = "SELECT v.entity_id, v.created_at, v.last_updated_at, v.value_version, ROW_NUMBER() OVER (ORDER BY v.entity_id ASC) AS row_num"; + cteJoin = ""; + } + + var query = $""" + WITH all_matches AS ( + {cteSelect} + FROM main.entities v + {cteJoin} + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({queryClauses.WhereClause}) + ), + total AS ( + SELECT COUNT(*) AS total_count FROM all_matches + ), + filtered_ids AS ( + SELECT * FROM all_matches + ORDER BY row_num ASC + LIMIT @limit OFFSET @offset + ) + SELECT + fi.entity_id, + field_sv.field_path_text, + field_sv.string_value, + field_sv.number_value, + field_sv.datetime_value, + field_sv.boolean_value, + field_sv.guid_value, + t.total_count, + fi.created_at, + fi.last_updated_at, + fi.value_version + FROM total t + LEFT JOIN filtered_ids fi ON 1 + LEFT JOIN main.search_values field_sv + ON fi.entity_id = field_sv.entity_id + AND field_sv.entity_type_id = @entity_type_id + AND field_sv.pool_id = @pool_id + AND field_sv.item_index = -1 + AND ({fieldConditionsClause}) + ORDER BY fi.row_num, fi.entity_id + """; + + Dialect.AddParameter(cmd, "@entity_type_id", entityTypeId); + Dialect.AddParameter(cmd, "@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@offset", skip); + _ = cmd.Parameters.AddWithValue("@limit", take); + + cmd.CommandText = query; + + Log.ExecutingQuery(logger, query); + + var resultsById = new Dictionary FieldValues, DateTimeOffset Created, DateTimeOffset LastUpdated, int Version)>(); + var orderedIds = new List(); + var totalCount = 0; + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + { + if (totalCount == 0) + { + totalCount = Convert.ToInt32(reader.GetInt64(7)); + } + + if (await reader.IsDBNullAsync(0, ct)) + { + continue; + } + + var entityId = Guid.Parse(reader.GetString(0)); + var fieldPath = await reader.IsDBNullAsync(1, ct) ? null : reader.GetString(1); + + if (!resultsById.TryGetValue(entityId, out var entry)) + { + var fieldValues = new Dictionary(); + orderedIds.Add(entityId); + + foreach (var field in fields) + { + fieldValues[field.Path] = null; + } + + var created = DateTimeOffset.Parse(reader.GetString(8), null, DateTimeStyles.RoundtripKind); + var lastUpdated = DateTimeOffset.Parse(reader.GetString(9), null, DateTimeStyles.RoundtripKind); + var version = reader.GetInt32(10); + + // Populate system fields from entity columns + foreach (var field in fields) + { + if (string.Equals(field.Path, SystemFields.Created, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.CreatedAttributeName, StringComparison.OrdinalIgnoreCase)) + { + fieldValues[field.Path] = created; + } + else if (string.Equals(field.Path, SystemFields.LastUpdated, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.LastUpdatedAttributeName, StringComparison.OrdinalIgnoreCase)) + { + fieldValues[field.Path] = lastUpdated; + } + } + + entry = (fieldValues, created, lastUpdated, version); + resultsById[entityId] = entry; + } + + if (fieldPath != null && entry.FieldValues.ContainsKey(fieldPath)) + { + var field = fields.First(f => f.Path == fieldPath); + var value = await ReadFieldValueAsync(reader, field.Type, 2, ct); + entry.FieldValues[fieldPath] = value; + } + } + } + + var items = orderedIds + .Select(id => new ProjectedResult(id, resultsById[id].FieldValues)) + .ToList(); + + return new QueryResult + { + Items = items, + TotalCount = totalCount, + TotalPages = (int)Math.Ceiling((double)totalCount / take), + HasMoreData = skip + take < totalCount + }; + } + + /// + /// Cursor-based query core for full entities. + /// + private async Task>> QueryCursorCoreAsync( + EntityType entityType, + IQueryExpression filter, + SortParameter sort, + ContinuationTokenDataRange tokenRange, + Ct ct) where TDso : IDataStorageObject + { + ArgumentNullException.ThrowIfNull(sort); + if (sort.IsEmpty) + { + throw new ArgumentException("Sort parameter is required for cursor-based pagination.", nameof(sort)); + } + + var pageSize = tokenRange.Size.Value; + var dsoVersion = TDso.DsoVersion; + var entityTypeId = (int)entityType.Id; + + Log.QueryingDsos(logger, entityType, 0, pageSize); + + await using var cnn = await OpenConnectionAsync(ct); + + await using var cmd = cnn.CreateCommand(); + var queryClauses = BuildCursorQueryClauses(cmd, entityTypeId, filter, sort, tokenRange); + + var query = $""" + SELECT v.entity_id, v.value, v.dso_type_schema_version, v.value_version, v.created_at, v.last_updated_at, {queryClauses.SortColumnName} + FROM main.entities v + {queryClauses.JoinClause} + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({queryClauses.WhereClause}) + {queryClauses.SeekClause} + {queryClauses.OrderByClause} + LIMIT @limit + """; + + Dialect.AddParameter(cmd, "@entity_type_id", entityTypeId); + Dialect.AddParameter(cmd, "@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@limit", pageSize + 1); + + cmd.CommandText = query; + + Log.ExecutingQuery(logger, query); + + var items = new List<(Guid Id, TDso Item, int Version, DateTimeOffset Created, DateTimeOffset LastUpdated, object? SortValue)>(); + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + var dsoType = dataStorageTypeRegistry.Get(dsoVersion); + while (await reader.ReadAsync(ct)) + { + var entityId = Guid.Parse(reader.GetString(0)); + var jsonValue = reader.GetString(1); + var item = (TDso)JsonSerializer.Deserialize(jsonValue, dsoType)!; + var valueVersion = reader.GetInt32(3); + var created = DateTimeOffset.Parse(reader.GetString(4), null, DateTimeStyles.RoundtripKind); + var lastUpdated = DateTimeOffset.Parse(reader.GetString(5), null, DateTimeStyles.RoundtripKind); + var sortValue = await ReadSortValueAsync(reader, sort.Field!, 6, ct); + items.Add((entityId, item, valueVersion, created, lastUpdated, sortValue)); + } + } + + var hasMore = items.Count > pageSize; + var pageItems = items.Take(pageSize).ToList(); + + ContinuationToken? nextToken = null; + if (pageItems.Count > 0) + { + var lastItem = pageItems[^1]; + var token = CreateCursorToken(lastItem.Id, lastItem.SortValue); + nextToken = (ContinuationToken)token.Encode(); + } + + var resultItems = pageItems.Select(x => new MetadataEnvelope(x.Item, x.Id, x.Version, x.Created, x.LastUpdated)).ToList(); + return new QueryResult> + { + Items = resultItems, + NextToken = nextToken, + HasMoreData = hasMore + }; + } + + /// + /// Cursor-based query core for projected field values. + /// + private async Task> QueryFieldsCursorCoreAsync( + EntityType entityType, + IReadOnlyCollection fields, + IQueryExpression filter, + SortParameter sort, + ContinuationTokenDataRange tokenRange, + Ct ct) + { + ArgumentNullException.ThrowIfNull(sort); + if (sort.IsEmpty) + { + throw new ArgumentException("Sort parameter is required for cursor-based pagination.", nameof(sort)); + } + + var pageSize = tokenRange.Size.Value; + var entityTypeId = (int)entityType.Id; + + Log.QueryingFieldsDsos(logger, entityType, fields.Count, 0, pageSize); + + await using var cnn = await OpenConnectionAsync(ct); + + await using var cmd = cnn.CreateCommand(); + var queryClauses = BuildCursorQueryClauses(cmd, entityTypeId, filter, sort, tokenRange); + + var fieldPaths = fields.Select(f => f.Path).ToList(); + var fieldConditions = new List(); + var paramIndex = 0; + for (var i = 0; i < fieldPaths.Count; i++) + { + if (SystemFields.IsSystemField(fieldPaths[i])) + { + continue; + } + + _ = cmd.Parameters.AddWithValue($"@select_field_{paramIndex}", DeterministicGuidGenerator.Create(fieldPaths[i].ToUpperInvariant()).ToByteArray()); + fieldConditions.Add($"field_sv.field_path = @select_field_{paramIndex}"); + paramIndex++; + } + var fieldConditionsClause = fieldConditions.Count > 0 + ? string.Join(" OR ", fieldConditions) + : "1=0"; + + var query = $""" + WITH filtered_ids AS ( + SELECT v.entity_id, v.created_at, v.last_updated_at, v.value_version, {queryClauses.SortColumnName} AS sort_value, ROW_NUMBER() OVER ({queryClauses.OrderByClause}) AS row_num + FROM main.entities v + {queryClauses.JoinClause} + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({queryClauses.WhereClause}) + {queryClauses.SeekClause} + {queryClauses.OrderByClause} + LIMIT @limit + ) + SELECT + fi.entity_id, + field_sv.field_path_text, + field_sv.string_value, + field_sv.number_value, + field_sv.datetime_value, + field_sv.boolean_value, + field_sv.guid_value, + fi.sort_value, + fi.created_at, + fi.last_updated_at, + fi.value_version + FROM filtered_ids fi + LEFT JOIN main.search_values field_sv + ON fi.entity_id = field_sv.entity_id + AND field_sv.entity_type_id = @entity_type_id + AND field_sv.pool_id = @pool_id + AND field_sv.item_index = -1 + AND ({fieldConditionsClause}) + ORDER BY fi.row_num, fi.entity_id + """; + + Dialect.AddParameter(cmd, "@entity_type_id", entityTypeId); + Dialect.AddParameter(cmd, "@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@limit", pageSize + 1); + + cmd.CommandText = query; + + Log.ExecutingQuery(logger, query); + + var resultsByid = new Dictionary FieldValues, DateTimeOffset Created, DateTimeOffset LastUpdated, object? SortValue, int Version)>(); + var orderedIds = new List(); + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + { + var entityId = Guid.Parse(reader.GetString(0)); + var fieldPath = await reader.IsDBNullAsync(1, ct) ? null : reader.GetString(1); + + if (!resultsByid.TryGetValue(entityId, out var entry)) + { + var fieldValues = new Dictionary(); + + foreach (var field in fields) + { + fieldValues[field.Path] = null; + } + + var sortValue = await ReadSortValueAsync(reader, sort.Field!, 7, ct); + var created = DateTimeOffset.Parse(reader.GetString(8), null, DateTimeStyles.RoundtripKind); + var lastUpdated = DateTimeOffset.Parse(reader.GetString(9), null, DateTimeStyles.RoundtripKind); + var version = reader.GetInt32(10); + + // Populate system fields from entity columns + foreach (var field in fields) + { + if (string.Equals(field.Path, SystemFields.Created, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.CreatedAttributeName, StringComparison.OrdinalIgnoreCase)) + { + fieldValues[field.Path] = created; + } + else if (string.Equals(field.Path, SystemFields.LastUpdated, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.LastUpdatedAttributeName, StringComparison.OrdinalIgnoreCase)) + { + fieldValues[field.Path] = lastUpdated; + } + } + + entry = (fieldValues, created, lastUpdated, sortValue, version); + resultsByid[entityId] = entry; + orderedIds.Add(entityId); + } + + if (fieldPath != null && entry.FieldValues.ContainsKey(fieldPath)) + { + var field = fields.First(f => f.Path == fieldPath); + var value = await ReadFieldValueAsync(reader, field.Type, 2, ct); + entry.FieldValues[fieldPath] = value; + } + } + } + + var itemsList = orderedIds.Select(id => (Id: id, resultsByid[id])).ToList(); + var hasMore = itemsList.Count > pageSize; + var pageItems = itemsList.Take(pageSize).ToList(); + + ContinuationToken? nextToken = null; + if (pageItems.Count > 0) + { + var lastItem = pageItems[^1]; + var token = CreateCursorToken(lastItem.Id, lastItem.Item2.SortValue); + nextToken = (ContinuationToken)token.Encode(); + } + + var items = pageItems + .Select(item => new ProjectedResult(item.Id, item.Item2.FieldValues)) + .ToList(); + + return new QueryResult + { + Items = items, + NextToken = nextToken, + HasMoreData = hasMore + }; + } + + /// + /// Builds the WHERE clause, ORDER BY clause, and JOIN clause for a query. + /// + private static QueryClauses BuildQueryClauses( + SqliteCommand cmd, + IQueryExpression filter, + SortParameter sort) + { + var whereBuilder = new SqlWhereClauseBuilder(SchemaName, cmd, Dialect); + var whereClause = whereBuilder.BuildWhereClause(filter); + + string joinClause; + string orderByClause; + if (!sort.IsEmpty) + { + var sortFieldPath = sort.Field!.Path; + var sortDirection = sort.Direction == SortDirection.Ascending ? "ASC" : "DESC"; + var sortColumn = GetSortColumnName(sort.Field!); + + if (SystemFields.IsSystemField(sortFieldPath)) + { + joinClause = ""; + } + else + { + joinClause = $""" + LEFT JOIN main.search_values sort_sv + ON v.entity_type_id = sort_sv.entity_type_id + AND v.entity_id = sort_sv.entity_id + AND v.pool_id = sort_sv.pool_id + AND sort_sv.field_path = @sort_field_path + AND sort_sv.item_index = -1 + """; + + _ = cmd.Parameters.AddWithValue("@sort_field_path", DeterministicGuidGenerator.Create(sortFieldPath.ToUpperInvariant()).ToByteArray()); + } + + orderByClause = $""" + ORDER BY + CASE WHEN {sortColumn} IS NULL THEN 1 ELSE 0 END, + {sortColumn} {sortDirection}, + v.entity_id ASC + """; + } + else + { + joinClause = ""; + orderByClause = "ORDER BY v.entity_id ASC"; + } + + return new QueryClauses(whereClause, joinClause, orderByClause); + } + + /// + /// Builds the WHERE clause, JOIN clause, ORDER BY clause, and seek clause for cursor-based pagination. + /// + private static CursorQueryClauses BuildCursorQueryClauses( + SqliteCommand cmd, + int entityTypeId, + IQueryExpression filter, + SortParameter sort, + ContinuationTokenDataRange tokenRange) + { + var whereBuilder = new SqlWhereClauseBuilder(SchemaName, cmd, Dialect); + var whereClause = whereBuilder.BuildWhereClause(filter); + + var sortFieldPath = sort.Field!.Path; + var sortDirection = sort.Direction == SortDirection.Ascending ? "ASC" : "DESC"; + var sortColumn = GetSortColumnName(sort.Field!); + + string joinClause; + if (SystemFields.IsSystemField(sortFieldPath)) + { + joinClause = ""; + } + else + { + joinClause = $""" + LEFT JOIN main.search_values sort_sv + ON v.entity_type_id = sort_sv.entity_type_id + AND v.entity_id = sort_sv.entity_id + AND v.pool_id = sort_sv.pool_id + AND sort_sv.field_path = @sort_field_path + AND sort_sv.item_index = -1 + """; + + _ = cmd.Parameters.AddWithValue("@sort_field_path", DeterministicGuidGenerator.Create(sortFieldPath.ToUpperInvariant()).ToByteArray()); + } + + var orderByClause = $""" + ORDER BY + CASE WHEN {sortColumn} IS NULL THEN 1 ELSE 0 END, + {sortColumn} {sortDirection}, + v.entity_id ASC + """; + + // Build seek clause for cursor position + var seekClause = ""; + var tokenValue = tokenRange.Start.Value; + if (tokenValue != ContinuationToken.Beginning) + { + var decodedToken = CursorToken.Decode(tokenValue); + if (decodedToken != null) + { + var lastSortParam = "@last_sort_value"; + var lastIdParam = "@last_id"; + + if (decodedToken.GuidValue.HasValue) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, decodedToken.GuidValue.Value.ToString()); + } + else if (decodedToken.StringValue != null) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, decodedToken.StringValue); + } + else if (decodedToken.NumberValue.HasValue) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, (double)decodedToken.NumberValue.Value); + } + else if (decodedToken.DateTimeValue.HasValue) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, decodedToken.DateTimeValue.Value.UtcDateTime.ToString("O")); + } + else if (decodedToken.BooleanValue.HasValue) + { + _ = cmd.Parameters.AddWithValue(lastSortParam, decodedToken.BooleanValue.Value ? 1 : 0); + } + else + { + _ = cmd.Parameters.AddWithValue(lastSortParam, DBNull.Value); + } + + _ = cmd.Parameters.AddWithValue(lastIdParam, decodedToken.Id.ToString()); + + if (sort.Direction == SortDirection.Ascending) + { + // With NULLS LAST in ascending: non-NULL values first, then NULLs + // If last value was non-NULL, include rows with greater value OR NULL values + // If last value was NULL, only include NULLs with greater ID + seekClause = $""" + AND ( + ({sortColumn} > {lastSortParam} OR ({sortColumn} = {lastSortParam} AND v.entity_id > {lastIdParam})) + OR ({sortColumn} IS NULL AND {lastSortParam} IS NOT NULL) + OR ({sortColumn} IS NULL AND {lastSortParam} IS NULL AND v.entity_id > {lastIdParam}) + ) + """; + } + else + { + // With NULLS LAST in descending: non-NULL values (descending), then NULLs + // If last value was non-NULL, include rows with lesser value OR NULL values + // If last value was NULL, only include NULLs with greater ID + seekClause = $""" + AND ( + ({sortColumn} < {lastSortParam} OR ({sortColumn} = {lastSortParam} AND v.entity_id > {lastIdParam})) + OR ({sortColumn} IS NULL AND {lastSortParam} IS NOT NULL) + OR ({sortColumn} IS NULL AND {lastSortParam} IS NULL AND v.entity_id > {lastIdParam}) + ) + """; + } + } + } + + return new CursorQueryClauses(whereClause, joinClause, orderByClause, seekClause, sortColumn); + } + + /// + /// Creates a cursor token from a sort value and entity ID. + /// + private static CursorToken CreateCursorToken(Guid id, object? sortValue) => + sortValue switch + { + string s => CursorToken.Create(id, s, null, null, null, null), + decimal d => CursorToken.Create(id, null, d, null, null, null), + DateTime dt => CursorToken.Create(id, null, null, new DateTimeOffset(dt, TimeSpan.Zero), null, null), + DateTimeOffset dto => CursorToken.Create(id, null, null, dto, null, null), + bool b => CursorToken.Create(id, null, null, null, b, null), + Guid g => CursorToken.Create(id, null, null, null, null, g), + null => CursorToken.Create(id, null, null, null, null, null), + _ => throw new InvalidOperationException($"Unsupported sort value type: {sortValue.GetType().Name}") + }; + + /// + /// Resolves the skip and take values from a DataRange (offset or page-based). + /// + private static (int Skip, int Take) ResolveOffsetAndSize(DataRange dataRange) + { + if (dataRange.OffsetValue is not null) + { + return ((int)dataRange.OffsetValue.Skip.Value, dataRange.OffsetValue.Take.Value); + } + + if (dataRange.PageValue is not null) + { + var page = dataRange.PageValue.Page.Value; + var size = dataRange.PageValue.PageSize.Value; + return ((page - 1) * size, size); + } + + // Fallback — should not happen since TokenValue is checked before calling this + return (0, DataRangeSize.Default.Value); + } + + /// + /// Gets the SQL column name for sorting based on field type. + /// + private static string GetSortColumnName(Field field) + { + if (SystemFields.IsSystemField(field.Path)) + { + return field is DateTimeField + ? string.Equals(field.Path, SystemFields.Created, StringComparison.OrdinalIgnoreCase) || + string.Equals(field.Path, SystemFields.CreatedAttributeName, StringComparison.OrdinalIgnoreCase) + ? "v.created_at" + : "v.last_updated_at" + : throw new InvalidOperationException($"System field '{field.Path}' must use DateTimeField, not {field.GetType().Name}."); + } + + return field switch + { + StringField => "sort_sv.string_value", + NumberField => "sort_sv.number_value", + DateTimeField => "sort_sv.datetime_value", + BooleanField => "sort_sv.boolean_value", + GuidField or ExactMatchField => "sort_sv.guid_value", + _ => throw new InvalidOperationException($"Unsupported field type for sorting: {field.GetType().Name}") + }; + } + + /// + /// Reads a field value from the database reader. + /// + private static async Task ReadFieldValueAsync(SqliteDataReader reader, FieldType fieldType, int stringValueColumnIndex, Ct ct) + { + var columnIndex = fieldType switch + { + FieldType.String => stringValueColumnIndex, + FieldType.Number => stringValueColumnIndex + 1, + FieldType.DateTime => stringValueColumnIndex + 2, + FieldType.Boolean => stringValueColumnIndex + 3, + FieldType.Guid => stringValueColumnIndex + 4, + _ => throw new InvalidOperationException($"Unsupported field type: {fieldType}") + }; + + if (await reader.IsDBNullAsync(columnIndex, ct)) + { + return null; + } + + return fieldType switch + { + FieldType.String => reader.GetString(columnIndex), + FieldType.Number => Convert.ToDecimal(reader.GetValue(columnIndex), CultureInfo.InvariantCulture), + FieldType.DateTime => DateTimeOffset.Parse(reader.GetString(columnIndex), CultureInfo.InvariantCulture), + FieldType.Boolean => reader.GetInt64(columnIndex) != 0, + FieldType.Guid => Guid.Parse(reader.GetString(columnIndex)), + _ => throw new InvalidOperationException($"Unsupported field type: {fieldType}") + }; + } + + /// + /// Reads a sort value from a database reader for the specified field type. + /// + private static async Task ReadSortValueAsync(SqliteDataReader reader, Field sortField, int columnIndex, Ct ct) + { + if (await reader.IsDBNullAsync(columnIndex, ct)) + { + return null; + } + + return sortField switch + { + StringField => reader.GetString(columnIndex), + NumberField => Convert.ToDecimal(reader.GetValue(columnIndex), CultureInfo.InvariantCulture), + DateTimeField => DateTimeOffset.Parse(reader.GetString(columnIndex), CultureInfo.InvariantCulture), + BooleanField => reader.GetInt64(columnIndex) != 0, + GuidField or ExactMatchField => Guid.Parse(reader.GetString(columnIndex)), + _ => throw new InvalidOperationException($"Unsupported field type for sorting: {sortField.GetType().Name}") + }; + } + + /// + async Task IStore.LinkAsync(LinkDefinition definition, Storage.UuidV7 leftEntityId, Storage.UuidV7 rightEntityId, IReadOnlyList outboxEvents, Ct ct) + { + await using var cnn = await OpenConnectionAsync(ct); + await using var tx = (SqliteTransaction)await cnn.BeginTransactionAsync(ct); + var outcome = await ExecuteLinkCoreAsync(cnn, tx, LinkOperation.For(definition, leftEntityId, rightEntityId), ct); + if (outcome == OperationOutcome.Success) + { + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(cnn, tx, outboxEvents, ct); + } + await tx.CommitAsync(ct); + } + return outcome == OperationOutcome.AlreadyLinked ? LinkResult.AlreadyLinked : LinkResult.Success; + } + + /// + async Task IStore.UnlinkAsync(LinkDefinition definition, Storage.UuidV7 leftEntityId, Storage.UuidV7 rightEntityId, IReadOnlyList outboxEvents, Ct ct) + { + await using var cnn = await OpenConnectionAsync(ct); + await using var tx = (SqliteTransaction)await cnn.BeginTransactionAsync(ct); + _ = await ExecuteUnlinkCoreAsync(cnn, tx, UnlinkOperation.For(definition, leftEntityId, rightEntityId), ct); + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(cnn, tx, outboxEvents, ct); + } + await tx.CommitAsync(ct); + return UnlinkResult.Success; + } + + private async Task ExecuteLinkCoreAsync( + SqliteConnection cnn, + SqliteTransaction tx, + LinkOperation op, + Ct ct) + { + + await using var cmd = cnn.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = """ + INSERT OR IGNORE INTO main.entity_links (pool_id, link_type_id, left_entity_type_id, left_entity_id, right_entity_type_id, right_entity_id) + VALUES (@pool_id, @link_type_id, @left_entity_type_id, @left_entity_id, @right_entity_type_id, @right_entity_id) + """; + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@link_type_id", (int)op.Definition.Link.Id); + _ = cmd.Parameters.AddWithValue("@left_entity_type_id", (int)op.Definition.Left.Id); + _ = cmd.Parameters.AddWithValue("@left_entity_id", op.LeftEntityId.Value.ToString()); + _ = cmd.Parameters.AddWithValue("@right_entity_type_id", (int)op.Definition.Right.Id); + _ = cmd.Parameters.AddWithValue("@right_entity_id", op.RightEntityId.Value.ToString()); + + Log.ExecutingSql(logger, cmd.CommandText); + + var rowsAffected = await cmd.ExecuteNonQueryAsync(ct); + return rowsAffected == 0 ? OperationOutcome.AlreadyLinked : OperationOutcome.Success; + } + + private async Task ExecuteUnlinkCoreAsync( + SqliteConnection cnn, + SqliteTransaction tx, + UnlinkOperation op, + Ct ct) + { + + await using var cmd = cnn.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = """ + DELETE FROM main.entity_links + WHERE pool_id = @pool_id + AND link_type_id = @link_type_id + AND left_entity_id = @left_entity_id + AND right_entity_id = @right_entity_id + """; + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@link_type_id", (int)op.Definition.Link.Id); + _ = cmd.Parameters.AddWithValue("@left_entity_id", op.LeftEntityId.Value.ToString()); + _ = cmd.Parameters.AddWithValue("@right_entity_id", op.RightEntityId.Value.ToString()); + + Log.ExecutingSql(logger, cmd.CommandText); + + _ = await cmd.ExecuteNonQueryAsync(ct); + return OperationOutcome.Success; + } + + private async Task ExecuteOutboxInsertBatchCoreAsync( + SqliteConnection cnn, + SqliteTransaction tx, + IReadOnlyList outboxEvents, + Ct ct) + { + var rows = new List<(OutboxEvent Evt, IOutboxSubscriber Subscriber)>(); + foreach (var evt in outboxEvents) + { + foreach (var subscriber in outboxSubscribers.GetMatchingSubscribers(evt.EventName, evt.EntityTypeId)) + { + rows.Add((evt, subscriber)); + } + } + + if (rows.Count == 0) + { + return; + } + + await using var cmd = cnn.CreateCommand(); + cmd.Transaction = tx; + + var valueRows = new List(rows.Count); + for (var i = 0; i < rows.Count; i++) + { + var (evt, subscriber) = rows[i]; + valueRows.Add($"(@message_id{i}, @event_id{i}, @timestamp{i}, @event_name{i}, @subject_id{i}, @entity_type_id{i}, @entity_type_name{i}, @pool_id, @payload{i}, @subscriber_name{i})"); + _ = cmd.Parameters.AddWithValue($"@message_id{i}", Guid.CreateVersion7().ToString()); + _ = cmd.Parameters.AddWithValue($"@event_id{i}", evt.Id.Value.ToString()); + _ = cmd.Parameters.AddWithValue($"@timestamp{i}", evt.Timestamp.UtcDateTime.ToString("O")); + _ = cmd.Parameters.AddWithValue($"@event_name{i}", evt.EventName.ToString()); + _ = cmd.Parameters.AddWithValue($"@subject_id{i}", evt.SubjectId.Value.ToString()); + _ = cmd.Parameters.AddWithValue($"@entity_type_id{i}", evt.EntityTypeId); + _ = cmd.Parameters.AddWithValue($"@entity_type_name{i}", evt.EntityTypeName); + _ = cmd.Parameters.AddWithValue($"@payload{i}", evt.Payload); + _ = cmd.Parameters.AddWithValue($"@subscriber_name{i}", subscriber.SubscriberName.ToString()); + } + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + + cmd.CommandText = $""" + INSERT INTO main.outbox_subscriber_queue + (message_id, event_id, timestamp, event_name, subject_id, entity_type_id, entity_type_name, pool_id, payload, subscriber_name) + VALUES + {string.Join(",\n ", valueRows)} + """; + + Log.ExecutingSql(logger, cmd.CommandText); + _ = await cmd.ExecuteNonQueryAsync(ct); + } + + /// + /// Executes multiple operations atomically in a single transaction. + /// + async Task IStore.ExecuteBatchAsync( + IReadOnlyList operations, + IReadOnlyList outboxEvents, + Ct ct) + { + if (operations.Count == 0) + { + return new BatchResult(true, []); + } + + await using var connection = await OpenConnectionAsync(ct); + await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(ct); + + var results = new List(); + + try + { + for (var i = 0; i < operations.Count; i++) + { + var outcome = operations[i] switch + { + CreateOperation create => await ExecuteCreateCoreAsync(connection, transaction, create, ct), + UpdateOperation update => await ExecuteUpdateCoreAsync(connection, transaction, update, ct), + DeleteOperation delete => (await ExecuteDeleteCoreAsync(connection, transaction, delete, ct)).Outcome, + LinkOperation link => await ExecuteLinkCoreAsync(connection, transaction, link, ct), + UnlinkOperation unlink => await ExecuteUnlinkCoreAsync(connection, transaction, unlink, ct), + _ => throw new InvalidOperationException($"Unknown operation type: {operations[i].GetType().Name}") + }; + + results.Add(new OperationResult(i, outcome)); + + if (outcome is not OperationOutcome.Success and not OperationOutcome.AlreadyLinked) + { + return new BatchResult(false, results); + } + } + + if (outboxEvents is { Count: > 0 }) + { + await ExecuteOutboxInsertBatchCoreAsync(connection, transaction, outboxEvents, ct); + } + + await transaction.CommitAsync(ct); + return new BatchResult(true, results); + } + catch + { + await transaction.RollbackAsync(ct); + throw; + } + } + + async Task IStore.GetOutboxEventsForSubscriberAsync(SubscriberName subscriberName, int count, Ct ct) + { + await using var cnn = await OpenConnectionAsync(ct); + + await using var cmd = cnn.CreateCommand(); + cmd.CommandText = """ + SELECT sequence_number, message_id, event_id, timestamp, event_name, subject_id, entity_type_id, entity_type_name, pool_id, payload, subscriber_name + FROM main.outbox_subscriber_queue + WHERE subscriber_name = @subscriber_name + ORDER BY sequence_number ASC + LIMIT @limit + """; + _ = cmd.Parameters.AddWithValue("@limit", count + 1); + _ = cmd.Parameters.AddWithValue("@subscriber_name", subscriberName.ToString()); + + Log.ExecutingSql(logger, cmd.CommandText); + await using var reader = await cmd.ExecuteReaderAsync(ct); + + var events = new List(); + while (await reader.ReadAsync(ct)) + { + var timestamp = DateTimeOffset.Parse(reader.GetString(3), CultureInfo.InvariantCulture); + events.Add(new PersistedOutboxEvent + { + SequenceNumber = reader.GetInt64(0), + MessageId = Guid.Parse(reader.GetString(1)), + EventId = Guid.Parse(reader.GetString(2)), + Timestamp = timestamp, + EventName = OutboxEventName.Create(reader.GetString(4)), + SubjectId = Storage.UuidV7.From(Guid.Parse(reader.GetString(5))), + EntityTypeId = reader.GetInt32(6), + EntityTypeName = reader.GetString(7), + PoolId = PoolId.Load(reader.GetInt32(8)), + Payload = reader.GetString(9), + SubscriberName = SubscriberName.Create(reader.GetString(10)), + }); + } + + var hasMore = events.Count > count; + if (hasMore) + { + events.RemoveAt(events.Count - 1); + } + + return new OutboxEventsPage(events, hasMore); + } + + async Task IStore.DeleteOutboxEventsAsync(IReadOnlyList ids, Ct ct) + { + if (ids.Count == 0) + { + return; + } + + await using var cnn = await OpenConnectionAsync(ct); + + const int MaxBatchSize = 1000; + for (var offset = 0; offset < ids.Count; offset += MaxBatchSize) + { + var chunk = ids.Skip(offset).Take(MaxBatchSize).ToArray(); + + await using var cmd = cnn.CreateCommand(); + + var idParams = new List(); + for (var i = 0; i < chunk.Length; i++) + { + var paramName = $"@id{i}"; + idParams.Add(paramName); + _ = cmd.Parameters.AddWithValue(paramName, chunk[i].Value.ToString()); + } + + cmd.CommandText = $""" + DELETE FROM main.outbox_subscriber_queue + WHERE message_id IN ({string.Join(", ", idParams)}) + """; + + Log.ExecutingSql(logger, cmd.CommandText); + _ = await cmd.ExecuteNonQueryAsync(ct); + } + } + + private async Task ExecuteCreateCoreAsync( + SqliteConnection cnn, + SqliteTransaction tx, + CreateOperation op, + Ct ct) + { + var dsoVersion = op.DsoVersion; + var dsoTypeId = (int)dsoVersion.EntityType.Id; + var entityType = dsoVersion.EntityType; + var jsonDso = JsonSerializer.Serialize(op.Value); + + // Resolve expiration + var expiresAt = op.Expiration.Resolve(timeProvider); + if (expiresAt.HasValue && expiresAt.Value <= timeProvider.GetUtcNow()) + { + // Already expired — noop, return success without storing + return OperationOutcome.Success; + } + + Log.CreatingDso(logger, entityType, op.Id.Value, dsoVersion.SchemaVersion); + + var builder = new StringBuilder(); + + _ = builder.AppendLine( + CultureInfo.InvariantCulture, + $""" + INSERT INTO main.entities (entity_type_id, entity_type_name, entity_id, value, dso_type_schema_version, value_version, pool_id, expires_at, created_at, last_updated_at) + VALUES (@entity_type_id, @entity_type_name, @entity_id, @value, @dso_type_schema_version, 1, @pool_id, @expires_at, @now, @now); + """); + + await using var createCmd = cnn.CreateCommand(); + createCmd.Transaction = tx; + _ = createCmd.Parameters.AddWithValue("@entity_type_id", dsoTypeId); + _ = createCmd.Parameters.AddWithValue("@entity_type_name", entityType.Name); + _ = createCmd.Parameters.AddWithValue("@entity_id", op.Id.Value.ToString()); + _ = createCmd.Parameters.AddWithValue("@value", jsonDso); + _ = createCmd.Parameters.AddWithValue("@dso_type_schema_version", (int)dsoVersion.SchemaVersion); + _ = createCmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = createCmd.Parameters.AddWithValue("@now", timeProvider.GetUtcNow().UtcDateTime.ToString("O")); + if (expiresAt.HasValue) + { + _ = createCmd.Parameters.AddWithValue("@expires_at", expiresAt.Value.UtcDateTime.ToString("O")); + } + else + { + _ = createCmd.Parameters.AddWithValue("@expires_at", DBNull.Value); + } + + // Add insert statements for keys + _ = AddInserts(builder, createCmd, op.Keys); + + // Add insert statements for search fields + _ = AddSearchFieldInserts(builder, createCmd, op.SearchFieldCollection); + + createCmd.CommandText = builder.ToString(); + + Log.ExecutingSql(logger, createCmd.CommandText); + + try + { + _ = await createCmd.ExecuteNonQueryAsync(ct); + } + catch (SqliteException ex) when (ex.SqliteErrorCode == 19) // SQLITE_CONSTRAINT + { + // Distinguish between entity already exists (constraint on entities table) + // and key conflict (constraint on entity_keys table) by checking the error message. + // SQLite constraint messages include the table name, e.g.: + // "UNIQUE constraint failed: entities.pool_id, entities.entity_type_id, entities.entity_id" + // "UNIQUE constraint failed: entity_keys.pool_id, entity_keys.entity_type_id, ..." + if (ex.Message.Contains("entities.", StringComparison.OrdinalIgnoreCase)) + { + return OperationOutcome.AlreadyExists; + } + + return OperationOutcome.KeyConflict; + } + + return OperationOutcome.Success; + } + + private async Task ExecuteUpdateCoreAsync( + SqliteConnection cnn, + SqliteTransaction tx, + UpdateOperation op, + Ct ct) + { + var dsoVersion = op.DsoVersion; + var entityType = dsoVersion.EntityType; + var jsonDso = JsonSerializer.Serialize(op.Value); + + // Resolve expiration + DateTimeOffset? expiresAt = null; + var hasExpirationChange = op.Expiration is not null; + if (hasExpirationChange) + { + expiresAt = op.Expiration!.Resolve(timeProvider); + } + + Log.UpdatingDso(logger, entityType, op.Id.Value, dsoVersion.SchemaVersion, op.ExpectedEntityVersion); + + // Read the current version of the entity + await using var readVersionCmd = cnn.CreateCommand(); + readVersionCmd.Transaction = tx; + readVersionCmd.CommandText = + "SELECT value_version FROM main.entities WHERE entity_type_id = @entity_type_id AND entity_id = @entity_id AND pool_id = @pool_id"; + + _ = readVersionCmd.Parameters.AddWithValue("@entity_type_id", (int)entityType.Id); + _ = readVersionCmd.Parameters.AddWithValue("@entity_id", op.Id.Value.ToString()); + _ = readVersionCmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + + Log.ExecutingSql(logger, readVersionCmd.CommandText); + + var actualEntityVersion = (long?)await readVersionCmd.ExecuteScalarAsync(ct); + + if (actualEntityVersion == null) + { + return OperationOutcome.DoesNotExist; + } + + if (actualEntityVersion != op.ExpectedEntityVersion) + { + return OperationOutcome.UnexpectedVersion; + } + + var builder = new StringBuilder(); + + var expiresAtSql = hasExpirationChange + ? "expires_at = @expires_at," + : ""; + + _ = builder.AppendLine( + CultureInfo.InvariantCulture, + $""" + UPDATE main.entities + SET + entity_type_name = @entity_type_name, + value = @value, + dso_type_schema_version = @dso_type_schema_version, + value_version = value_version + 1, + {expiresAtSql} + last_updated_at = @now + WHERE + entity_type_id = @entity_type_id AND entity_id = @entity_id AND pool_id = @pool_id; + """); + + await using var updateCmd = cnn.CreateCommand(); + updateCmd.Transaction = tx; + _ = updateCmd.Parameters.AddWithValue("@entity_type_id", (int)entityType.Id); + _ = updateCmd.Parameters.AddWithValue("@entity_type_name", entityType.Name); + _ = updateCmd.Parameters.AddWithValue("@entity_id", op.Id.Value.ToString()); + _ = updateCmd.Parameters.AddWithValue("@value", jsonDso); + _ = updateCmd.Parameters.AddWithValue("@dso_type_schema_version", (int)dsoVersion.SchemaVersion); + _ = updateCmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = updateCmd.Parameters.AddWithValue("@now", timeProvider.GetUtcNow().UtcDateTime.ToString("O")); + if (hasExpirationChange) + { + if (expiresAt.HasValue) + { + _ = updateCmd.Parameters.AddWithValue("@expires_at", expiresAt.Value.UtcDateTime.ToString("O")); + } + else + { + _ = updateCmd.Parameters.AddWithValue("@expires_at", DBNull.Value); + } + } + + // Delete existing keys + _ = builder.AppendLine( + CultureInfo.InvariantCulture, + $"DELETE FROM main.entity_keys WHERE entity_type_id = @entity_type_id AND entity_id = @entity_id AND pool_id = @pool_id;"); + + // Delete existing search fields + _ = builder.AppendLine( + CultureInfo.InvariantCulture, + $"DELETE FROM main.search_values WHERE entity_type_id = @entity_type_id AND entity_id = @entity_id AND pool_id = @pool_id;"); + + // Re-insert new keys + _ = AddInserts(builder, updateCmd, op.Keys); + + // Re-insert new search fields + _ = AddSearchFieldInserts(builder, updateCmd, op.SearchFieldCollection); + + updateCmd.CommandText = builder.ToString(); + + Log.ExecutingSql(logger, updateCmd.CommandText); + + try + { + _ = await updateCmd.ExecuteNonQueryAsync(ct); + } + catch (SqliteException ex) when (ex.SqliteErrorCode == 19) // SQLITE_CONSTRAINT + { + return OperationOutcome.KeyConflict; + } + + return OperationOutcome.Success; + } + + private async Task<(OperationOutcome Outcome, bool EntityDeleted)> ExecuteDeleteCoreAsync( + SqliteConnection cnn, + SqliteTransaction tx, + DeleteOperation op, + Ct ct) + { + var entityType = op.EntityType; + + await using var deleteCmd = cnn.CreateCommand(); + deleteCmd.Transaction = tx; + _ = deleteCmd.Parameters.AddWithValue("@entity_type_id", (int)entityType.Id); + _ = deleteCmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + + deleteCmd.CommandText = "DELETE FROM main.entities WHERE entity_type_id = @entity_type_id AND pool_id = @pool_id"; + + if (op.Id is not null) + { + Log.DeletingDso(logger, entityType, op.Id.Value); + _ = deleteCmd.Parameters.AddWithValue("@entity_id", op.Id.Value.ToString()); + + deleteCmd.CommandText += " AND entity_id = @entity_id"; + } + else if (op.Key is not null) + { + var key = op.Key; + _ = deleteCmd.Parameters.AddWithValue("@key_type_id", (int)key.DskVersion.KeyType.Id); + _ = deleteCmd.Parameters.AddWithValue("@key_type_version", (int)key.DskVersion.SchemaVersion); + _ = deleteCmd.Parameters.AddWithValue("@key_value", key.Value.ToString()); + deleteCmd.CommandText += """ + AND entity_id = ( + SELECT entity_id FROM main.entity_keys + WHERE entity_type_id = @entity_type_id + AND key_type_id = @key_type_id + AND key_type_version = @key_type_version + AND key_value = @key_value + AND pool_id = @pool_id + ) + """; + } + else + { + return (OperationOutcome.Success, false); + } + + Log.ExecutingSql(logger, deleteCmd.CommandText); + + // Resolve entity_id for link cleanup BEFORE deleting the entity + var entityId = op.Id?.Value ?? (op.Key is not null + ? await ResolveKeyToEntityIdAsync(cnn, tx, op.EntityType, op.Key, PoolId, ct) + : null); + + var rowsAffected = await deleteCmd.ExecuteNonQueryAsync(ct); + + // Delete entity links (no FK to entities, must be done manually) + if (entityId.HasValue) + { + await using var linkDeleteCmd = cnn.CreateCommand(); + linkDeleteCmd.Transaction = tx; + linkDeleteCmd.CommandText = """ + DELETE FROM main.entity_links + WHERE pool_id = @pool_id + AND (left_entity_id = @entity_id OR right_entity_id = @entity_id) + """; + _ = linkDeleteCmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = linkDeleteCmd.Parameters.AddWithValue("@entity_id", entityId.Value.ToString()); + Log.ExecutingSql(logger, linkDeleteCmd.CommandText); + _ = await linkDeleteCmd.ExecuteNonQueryAsync(ct); + } + + return (OperationOutcome.Success, rowsAffected > 0); + } + + private static async Task ResolveKeyToEntityIdAsync( + SqliteConnection cnn, + SqliteTransaction tx, + EntityType entityType, + DataStorageKey key, + PoolId poolId, + Ct ct) + { + await using var cmd = cnn.CreateCommand(); + cmd.Transaction = tx; + cmd.CommandText = """ + SELECT entity_id FROM main.entity_keys + WHERE entity_type_id = @entity_type_id + AND key_type_id = @key_type_id + AND key_type_version = @key_type_version + AND key_value = @key_value + AND pool_id = @pool_id + """; + _ = cmd.Parameters.AddWithValue("@entity_type_id", (int)entityType.Id); + _ = cmd.Parameters.AddWithValue("@key_type_id", (int)key.DskVersion.KeyType.Id); + _ = cmd.Parameters.AddWithValue("@key_type_version", (int)key.DskVersion.SchemaVersion); + _ = cmd.Parameters.AddWithValue("@key_value", key.Value.ToString()); + _ = cmd.Parameters.AddWithValue("@pool_id", poolId.Value); + var result = await cmd.ExecuteScalarAsync(ct); + return result is string s ? Guid.Parse(s) : null; + } + + /// + async Task>> IStore.QueryLinksAsync( + LinkQueryDescriptor query, + DataRange dataRange, + Ct ct) + { + if (dataRange.TokenValue is not null) + { + throw new NotSupportedException("Cursor-based pagination is not supported for link queries."); + } + + var (skip, take) = ResolveOffsetAndSize(dataRange); + var dsoVersion = TDso.DsoVersion; + var sourceEntityTypeId = (int)query.SourceEntityType.Id; + + await using var cnn = await OpenConnectionAsync(ct); + + await using var cmd = cnn.CreateCommand(); + + var joinSql = new StringBuilder(); + var whereLastJoin = ""; + + for (var i = 0; i < query.Joins.Count; i++) + { + var join = query.Joins[i]; + var linkTypeParam = $"@lt{i}"; + _ = cmd.Parameters.AddWithValue(linkTypeParam, (int)join.Definition.Link.Id); + + string sourceSide; + string filterSide; + if (join.Direction == LinkJoinDirection.LeftToRight) + { + sourceSide = "left_entity_id"; + filterSide = "right_entity_id"; + } + else + { + sourceSide = "right_entity_id"; + filterSide = "left_entity_id"; + } + + if (i == 0) + { + _ = joinSql.AppendLine(CultureInfo.InvariantCulture, + $"JOIN main.entity_links l0 ON l0.{sourceSide} = e.entity_id AND l0.link_type_id = {linkTypeParam} AND l0.pool_id = @pool_id"); + } + else + { + var prevJoin = query.Joins[i - 1]; + string prevFilterSide; + if (prevJoin.Direction == LinkJoinDirection.LeftToRight) + { + prevFilterSide = "right_entity_id"; + } + else + { + prevFilterSide = "left_entity_id"; + } + + _ = joinSql.AppendLine(CultureInfo.InvariantCulture, + $"JOIN main.entity_links l{i} ON l{i}.{sourceSide} = l{i - 1}.{prevFilterSide} AND l{i}.link_type_id = {linkTypeParam} AND l{i}.pool_id = @pool_id"); + } + + if (i == query.Joins.Count - 1) + { + whereLastJoin = $"l{i}.{filterSide}"; + } + } + + _ = cmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + _ = cmd.Parameters.AddWithValue("@source_entity_type_id", sourceEntityTypeId); + _ = cmd.Parameters.AddWithValue("@offset", skip); + _ = cmd.Parameters.AddWithValue("@limit", take); + + string whereClause; + if (query.WhereEntityId is not null) + { + _ = cmd.Parameters.AddWithValue("@where_entity_id", query.WhereEntityId.Value.ToString()); + whereClause = $"{whereLastJoin} = @where_entity_id"; + } + else + { + whereClause = "1"; + } + + var mainQuery = $""" + SELECT DISTINCT e.entity_id, e.value, e.dso_type_schema_version, e.value_version, e.created_at, e.last_updated_at + FROM main.entities e + {joinSql} + WHERE e.entity_type_id = @source_entity_type_id + AND e.pool_id = @pool_id + AND {whereClause} + ORDER BY e.entity_id + LIMIT @limit OFFSET @offset + """; + + cmd.CommandText = mainQuery; + Log.ExecutingQuery(logger, mainQuery); + + var items = new List>(); + var dsoType = dataStorageTypeRegistry.Get(dsoVersion); + await using (var reader = await cmd.ExecuteReaderAsync(ct)) + { + while (await reader.ReadAsync(ct)) + { + var entityId = Guid.Parse(reader.GetString(0)); + var jsonValue = reader.GetString(1); + var valueVersion = reader.GetInt32(3); + var created = DateTimeOffset.Parse(reader.GetString(4), null, DateTimeStyles.RoundtripKind); + var lastUpdated = DateTimeOffset.Parse(reader.GetString(5), null, DateTimeStyles.RoundtripKind); + var item = (TDso)JsonSerializer.Deserialize(jsonValue, dsoType)!; + items.Add(new MetadataEnvelope(item, entityId, valueVersion, created, lastUpdated)); + } + } + + // Count query + var countQuery = $""" + SELECT COUNT(DISTINCT e.entity_id) + FROM main.entities e + {joinSql} + WHERE e.entity_type_id = @source_entity_type_id + AND e.pool_id = @pool_id + AND {whereClause} + """; + + await using var countCmd = cnn.CreateCommand(); + _ = countCmd.Parameters.AddWithValue("@source_entity_type_id", sourceEntityTypeId); + _ = countCmd.Parameters.AddWithValue("@pool_id", PoolId.Value); + if (query.WhereEntityId is not null) + { + _ = countCmd.Parameters.AddWithValue("@where_entity_id", query.WhereEntityId.Value.ToString()); + } + + for (var i = 0; i < query.Joins.Count; i++) + { + _ = countCmd.Parameters.AddWithValue($"@lt{i}", (int)query.Joins[i].Definition.Link.Id); + } + + countCmd.CommandText = countQuery; + Log.ExecutingSql(logger, countQuery); + var totalCount = Convert.ToInt32(await countCmd.ExecuteScalarAsync(ct), CultureInfo.InvariantCulture); + + return new QueryResult> + { + Items = items, + TotalCount = totalCount, + TotalPages = (int)Math.Ceiling((double)totalCount / take), + HasMoreData = skip + take < totalCount + }; + } + + async Task IStore.CountAsync( + EntityType entityType, + IQueryExpression? filter, + Ct ct) + { + await using var cnn = await OpenConnectionAsync(ct); + await using var cmd = cnn.CreateCommand(); + + string whereClause; + if (filter is null or AllExpression) + { + whereClause = "1=1"; + } + else + { + var whereBuilder = new SqlWhereClauseBuilder(SchemaName, cmd, Dialect); + whereClause = whereBuilder.BuildWhereClause(filter); + } + + var query = $""" + SELECT COUNT(*) + FROM {SchemaName}.entities v + WHERE v.entity_type_id = @entity_type_id + AND v.pool_id = @pool_id + AND ({whereClause}) + """; + var entityTypeId = (int)entityType.Id; + _ = cmd.Parameters.AddWithValue("@entity_type_id", entityTypeId); + Dialect.AddParameter(cmd, "@pool_id", PoolId.Value); + + cmd.CommandText = query; + + Log.ExecutingSql(logger, query); + + var result = await cmd.ExecuteScalarAsync(ct); + return Convert.ToInt64(result, CultureInfo.InvariantCulture); + } + + async Task IStore.PurgeExpiredAsync(int batchSize, Ct ct) + { + ArgumentOutOfRangeException.ThrowIfLessThan(batchSize, 1); + ArgumentOutOfRangeException.ThrowIfGreaterThan(batchSize, StorageConstants.TtlCleanupMaxBatchSize); + + var now = timeProvider.GetUtcNow().UtcDateTime.ToString("O"); + + await using var cnn = await OpenConnectionAsync(ct); + await using var tx = (SqliteTransaction)await cnn.BeginTransactionAsync(ct); + + try + { + // Step 1: Select expired rows (SQLite doesn't have FOR UPDATE SKIP LOCKED, but + // the transaction provides database-level locking) + var expired = new List<(string PoolId, int EntityTypeId, string EntityId, string EntityTypeName, string Value, string EventId)>(); + await using (var selectCmd = cnn.CreateCommand()) + { + selectCmd.Transaction = tx; + selectCmd.CommandText = """ + SELECT pool_id, entity_type_id, entity_id, entity_type_name, value + FROM main.entities + WHERE expires_at IS NOT NULL AND expires_at <= @now + LIMIT @batchSize + """; + _ = selectCmd.Parameters.AddWithValue("@now", now); + _ = selectCmd.Parameters.AddWithValue("@batchSize", batchSize); + Log.ExecutingSql(logger, selectCmd.CommandText); + await using var reader = await selectCmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + expired.Add(( + reader.GetString(0), + reader.GetInt32(1), + reader.GetString(2), + reader.GetString(3), + reader.GetString(4), + Guid.CreateVersion7().ToString())); + } + } + + if (expired.Count == 0) + { + await tx.RollbackAsync(ct); + return 0; + } + + // Step 2: Insert outbox events per matching subscriber + if (!outboxSubscribers.IsEmpty) + { + var eventName = OutboxEventName.EntityExpired; + foreach (var subscriber in outboxSubscribers.Subscribers) + { + if (subscriber.EventNames.Count > 0 && !subscriber.EventNames.Contains(eventName)) + { + continue; + } + + var matchingExpired = subscriber.EntityTypeIds.Count > 0 + ? expired.Where(e => subscriber.EntityTypeIds.Contains(e.EntityTypeId)) + : expired; + + foreach (var (poolId, entityTypeId, entityId, entityTypeName, value, eventId) in matchingExpired) + { + await using var outboxCmd = cnn.CreateCommand(); + outboxCmd.Transaction = tx; + outboxCmd.CommandText = """ + INSERT INTO main.outbox_subscriber_queue + (message_id, event_id, timestamp, event_name, subject_id, entity_type_id, entity_type_name, pool_id, payload, subscriber_name) + VALUES (@message_id, @event_id, @timestamp, @event_name, @subject_id, @entity_type_id, @entity_type_name, @pool_id, @payload, @subscriber_name) + """; + _ = outboxCmd.Parameters.AddWithValue("@message_id", Guid.CreateVersion7().ToString()); + _ = outboxCmd.Parameters.AddWithValue("@event_id", eventId); + _ = outboxCmd.Parameters.AddWithValue("@timestamp", now); + _ = outboxCmd.Parameters.AddWithValue("@event_name", eventName.ToString()); + _ = outboxCmd.Parameters.AddWithValue("@subject_id", entityId); + _ = outboxCmd.Parameters.AddWithValue("@entity_type_id", entityTypeId); + _ = outboxCmd.Parameters.AddWithValue("@entity_type_name", entityTypeName); + _ = outboxCmd.Parameters.AddWithValue("@pool_id", poolId); + _ = outboxCmd.Parameters.AddWithValue("@payload", value); + _ = outboxCmd.Parameters.AddWithValue("@subscriber_name", subscriber.SubscriberName.ToString()); + Log.ExecutingSql(logger, outboxCmd.CommandText); + _ = await outboxCmd.ExecuteNonQueryAsync(ct); + } + } + } + + // Step 3: Delete entity links for expired entities + foreach (var (poolId, entityTypeId, entityId, _, _, _) in expired) + { + await using var linkCmd = cnn.CreateCommand(); + linkCmd.Transaction = tx; + linkCmd.CommandText = """ + DELETE FROM main.entity_links + WHERE pool_id = @pool_id + AND ( + (left_entity_id = @entity_id AND left_entity_type_id = @entity_type_id) + OR (right_entity_id = @entity_id AND right_entity_type_id = @entity_type_id) + ) + """; + _ = linkCmd.Parameters.AddWithValue("@pool_id", poolId); + _ = linkCmd.Parameters.AddWithValue("@entity_id", entityId); + _ = linkCmd.Parameters.AddWithValue("@entity_type_id", entityTypeId); + Log.ExecutingSql(logger, linkCmd.CommandText); + _ = await linkCmd.ExecuteNonQueryAsync(ct); + } + + // Step 4: Delete entities (entity_keys and search_values cascade) + var deleted = 0; + foreach (var (poolId, entityTypeId, entityId, _, _, _) in expired) + { + await using var deleteCmd = cnn.CreateCommand(); + deleteCmd.Transaction = tx; + deleteCmd.CommandText = """ + DELETE FROM main.entities + WHERE pool_id = @pool_id AND entity_type_id = @entity_type_id AND entity_id = @entity_id + AND expires_at <= @now + """; + _ = deleteCmd.Parameters.AddWithValue("@pool_id", poolId); + _ = deleteCmd.Parameters.AddWithValue("@entity_type_id", entityTypeId); + _ = deleteCmd.Parameters.AddWithValue("@entity_id", entityId); + _ = deleteCmd.Parameters.AddWithValue("@now", now); + Log.ExecutingSql(logger, deleteCmd.CommandText); + deleted += await deleteCmd.ExecuteNonQueryAsync(ct); + } + + await tx.CommitAsync(ct); + return deleted; + } + catch + { + await tx.RollbackAsync(ct); + throw; + } + } + + private sealed record QueryClauses(string WhereClause, string JoinClause, string OrderByClause); + + private sealed record CursorQueryClauses( + string WhereClause, + string JoinClause, + string OrderByClause, + string SeekClause, + string SortColumnName); + + private sealed record SchemaComment(int Version); +} diff --git a/storage/src/Storage.Sqlite/Internal/SqliteStoreOptionsValidator.cs b/storage/src/Storage.Sqlite/Internal/SqliteStoreOptionsValidator.cs new file mode 100644 index 000000000..cb1302519 --- /dev/null +++ b/storage/src/Storage.Sqlite/Internal/SqliteStoreOptionsValidator.cs @@ -0,0 +1,9 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Microsoft.Extensions.Options; + +namespace Duende.Storage.Sqlite.Internal; + +#pragma warning disable CA1812 // Avoid uninstantiated internal classes +public sealed class SqliteStoreOptionsValidator(string? name) : DataAnnotationValidateOptions(name); diff --git a/storage/src/Storage.Sqlite/Migrations/V001_InitialCreate.sql b/storage/src/Storage.Sqlite/Migrations/V001_InitialCreate.sql new file mode 100644 index 000000000..605cc01a1 --- /dev/null +++ b/storage/src/Storage.Sqlite/Migrations/V001_InitialCreate.sql @@ -0,0 +1,152 @@ +-- V0 → V1: initial schema creation + +CREATE TABLE IF NOT EXISTS __schema_info ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +-- Only run if current version < 1 +-- SQLite lacks procedural IF, so we use IF NOT EXISTS on all objects +-- and the version bump at the end is guarded by a WHERE clause. + +CREATE TABLE IF NOT EXISTS entities ( + pool_id INTEGER NOT NULL, + entity_type_id INTEGER NOT NULL, + entity_id TEXT NOT NULL, + entity_type_name TEXT NOT NULL, + value TEXT NOT NULL, + dso_type_schema_version INTEGER NOT NULL, + value_version INTEGER NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + last_updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + expires_at TEXT NULL, + PRIMARY KEY (pool_id, entity_type_id, entity_id) +); + +CREATE INDEX IF NOT EXISTS entities_expires_at_index +ON entities (expires_at) +WHERE expires_at IS NOT NULL; + +CREATE INDEX IF NOT EXISTS entities_created_at_index +ON entities (pool_id, entity_type_id, created_at); + +CREATE INDEX IF NOT EXISTS entities_last_updated_at_index +ON entities (pool_id, entity_type_id, last_updated_at); + +CREATE TABLE IF NOT EXISTS entity_keys ( + pool_id INTEGER NOT NULL, + entity_type_id INTEGER NOT NULL, + key_type_id INTEGER NOT NULL, + key_type_version INTEGER NOT NULL, + key_type_name TEXT NOT NULL, + key_value TEXT NOT NULL, + key_json TEXT NULL, + entity_id TEXT NOT NULL, + timestamp TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + PRIMARY KEY (pool_id, entity_type_id, key_type_id, key_type_version, key_value), + FOREIGN KEY (pool_id, entity_type_id, entity_id) REFERENCES entities (pool_id, entity_type_id, entity_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS entity_keys_entity_type_id_entity_id_index +ON entity_keys (entity_type_id, entity_id); + +CREATE TABLE IF NOT EXISTS search_values ( + entity_type_id INTEGER NOT NULL, + entity_id TEXT NOT NULL, + field_path BLOB NOT NULL, + field_path_text TEXT NOT NULL, + item_index INTEGER NOT NULL, + string_value TEXT NULL, + number_value REAL NULL, + datetime_value TEXT NULL, + boolean_value INTEGER NULL, + guid_value TEXT NULL, + pool_id INTEGER NOT NULL, + PRIMARY KEY (pool_id, entity_type_id, entity_id, field_path, item_index), + FOREIGN KEY (pool_id, entity_type_id, entity_id) REFERENCES entities (pool_id, entity_type_id, entity_id) ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS search_values_string_value_index +ON search_values (pool_id, entity_type_id, field_path, string_value) +WHERE string_value IS NOT NULL AND item_index = -1; + +CREATE INDEX IF NOT EXISTS search_values_number_value_index +ON search_values (pool_id, entity_type_id, field_path, number_value) +WHERE number_value IS NOT NULL AND item_index = -1; + +CREATE INDEX IF NOT EXISTS search_values_datetime_value_index +ON search_values (pool_id, entity_type_id, field_path, datetime_value) +WHERE datetime_value IS NOT NULL AND item_index = -1; + +CREATE INDEX IF NOT EXISTS search_values_boolean_value_index +ON search_values (pool_id, entity_type_id, field_path, boolean_value) +WHERE boolean_value IS NOT NULL AND item_index = -1; + +CREATE INDEX IF NOT EXISTS search_values_array_string_value_index +ON search_values (pool_id, entity_type_id, entity_id, field_path, item_index, string_value) +WHERE string_value IS NOT NULL AND item_index >= 0; + +CREATE INDEX IF NOT EXISTS search_values_array_number_value_index +ON search_values (pool_id, entity_type_id, entity_id, field_path, item_index, number_value) +WHERE number_value IS NOT NULL AND item_index >= 0; + +CREATE INDEX IF NOT EXISTS search_values_array_datetime_value_index +ON search_values (pool_id, entity_type_id, entity_id, field_path, item_index, datetime_value) +WHERE datetime_value IS NOT NULL AND item_index >= 0; + +CREATE INDEX IF NOT EXISTS search_values_array_boolean_value_index +ON search_values (pool_id, entity_type_id, entity_id, field_path, item_index, boolean_value) +WHERE boolean_value IS NOT NULL AND item_index >= 0; + +CREATE INDEX IF NOT EXISTS search_values_guid_value_index +ON search_values (pool_id, entity_type_id, field_path, guid_value) +WHERE item_index = -1 AND guid_value IS NOT NULL; + +CREATE INDEX IF NOT EXISTS search_values_array_guid_value_index +ON search_values (pool_id, entity_type_id, entity_id, field_path, item_index, guid_value) +WHERE item_index >= 0 AND guid_value IS NOT NULL; + +CREATE TABLE IF NOT EXISTS entity_links ( + pool_id INTEGER NOT NULL, + link_type_id INTEGER NOT NULL, + left_entity_type_id INTEGER NOT NULL, + left_entity_id TEXT NOT NULL, + right_entity_type_id INTEGER NOT NULL, + right_entity_id TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + PRIMARY KEY (pool_id, link_type_id, left_entity_id, right_entity_id) +); + +CREATE INDEX IF NOT EXISTS entity_links_left_entity_index +ON entity_links (pool_id, link_type_id, left_entity_id); + +CREATE INDEX IF NOT EXISTS entity_links_right_entity_index +ON entity_links (pool_id, link_type_id, right_entity_id); + +CREATE INDEX IF NOT EXISTS entity_links_left_cascade_index +ON entity_links (pool_id, left_entity_id); + +CREATE INDEX IF NOT EXISTS entity_links_right_cascade_index +ON entity_links (pool_id, right_entity_id); + +CREATE TABLE IF NOT EXISTS outbox_subscriber_queue ( + sequence_number INTEGER PRIMARY KEY AUTOINCREMENT, + message_id TEXT NOT NULL, + event_id TEXT NOT NULL, + timestamp TEXT NOT NULL, + event_name TEXT NOT NULL, + subject_id TEXT NOT NULL, + entity_type_id INTEGER NOT NULL, + entity_type_name TEXT NOT NULL, + pool_id INTEGER NOT NULL, + payload TEXT NOT NULL, + subscriber_name TEXT NOT NULL, + UNIQUE (message_id) +); + +CREATE INDEX IF NOT EXISTS outbox_subscriber_queue_subscriber_index +ON outbox_subscriber_queue (subscriber_name, sequence_number); + +-- Version bump: only update if not already at version 1 +INSERT OR IGNORE INTO __schema_info (key, value) VALUES ('SchemaVersion', '{"Version":1}'); +UPDATE __schema_info SET value = '{"Version":1}' WHERE key = 'SchemaVersion' AND json_extract(value, '$.Version') < 1; diff --git a/storage/src/Storage.Sqlite/SqliteStoreOptions.cs b/storage/src/Storage.Sqlite/SqliteStoreOptions.cs new file mode 100644 index 000000000..a95001852 --- /dev/null +++ b/storage/src/Storage.Sqlite/SqliteStoreOptions.cs @@ -0,0 +1,18 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.ComponentModel.DataAnnotations; + +namespace Duende.Storage.Sqlite; + +/// +/// Configuration options for the SQLite store. +/// +public sealed class SqliteStoreOptions +{ + /// + /// The connection string for the SQLite database. + /// + [Required] + public string? ConnectionString { get; set; } +} diff --git a/storage/src/Storage.Sqlite/SqliteStoreServiceCollectionExtensions.cs b/storage/src/Storage.Sqlite/SqliteStoreServiceCollectionExtensions.cs new file mode 100644 index 000000000..31a98ae73 --- /dev/null +++ b/storage/src/Storage.Sqlite/SqliteStoreServiceCollectionExtensions.cs @@ -0,0 +1,107 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.ComponentModel.DataAnnotations; +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Sqlite.Internal; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; + +namespace Duende.Storage.Sqlite; + +public static class SqliteStoreServiceCollectionExtensions +{ + extension(IStorageBuilder builder) + { + /// + /// Adds a SQLite store with the specified service key for multi-store scenarios. + /// + internal IStorageBuilder AddSqliteStore(object serviceKey, Action configure) + { + var services = builder.Services; + var options = BuildOptions(configure); + _ = services.AddStore(serviceKey); + _ = services.AddKeyedSingleton(serviceKey, (_, _) => + { + var connection = new SqliteConnection(options.ConnectionString); + connection.Open(); + return connection; + }); + _ = services.AddKeyedTransient(serviceKey, (sp, _) => + { + // Ensure the keep-alive connection exists (required for in-memory databases) + _ = sp.GetRequiredKeyedService(serviceKey); + var outboxSubscribers = sp.GetRequiredKeyedService(serviceKey); + return BuildStore(sp, outboxSubscribers, options); + }); + return builder; + } + + /// + /// Adds a SQLite store without a service key for single-store scenarios. + /// + public IStorageBuilder AddSqliteStore(Action configure) + { + var services = builder.Services; + var options = BuildOptions(configure); + _ = services.AddStore(); + _ = services.AddSingleton(_ => + { + var connection = new SqliteConnection(options.ConnectionString); + connection.Open(); + return connection; + }); + _ = services.AddTransient(sp => + { + // Ensure the keep-alive connection exists (required for in-memory databases) + _ = sp.GetRequiredService(); + var outboxSubscribers = sp.GetRequiredService(); + return BuildStore(sp, outboxSubscribers, options); + }); + return builder; + } + + /// + /// Adds a SQLite in-memory store intended for testing only. + /// Uses a shared in-memory database with a generated unique name. + /// This method is NOT intended for production use. + /// + public IStorageBuilder AddSqliteInMemoryStore() => + builder.AddSqliteInMemoryStore($"InMemoryDb_{Guid.NewGuid():N}"); + + /// + /// Adds a SQLite in-memory store intended for testing only. + /// Uses a shared in-memory database with the specified data source name, + /// allowing multiple connections (and tests) to share the same database. + /// This method is NOT intended for production use. + /// + /// + /// The data source name for the shared in-memory database. + /// Use the same name across tests to share state. + /// + public IStorageBuilder AddSqliteInMemoryStore(string dataSourceName) => + builder.AddSqliteStore(opt => opt.ConnectionString = $"Data Source={dataSourceName};Mode=Memory;Cache=Shared"); + } + + private static SqliteStoreOptions BuildOptions(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + var options = new SqliteStoreOptions(); + configure(options); + Validator.ValidateObject(options, new ValidationContext(options), validateAllProperties: true); + return options; + } + + private static SqliteStore BuildStore( + IServiceProvider sp, + OutboxSubscribers outboxSubscribers, + SqliteStoreOptions options) => + new( + options, + sp.GetRequiredService(), + sp.GetRequiredService(), + outboxSubscribers, + sp.GetRequiredService>()); +} diff --git a/storage/src/Storage.Sqlite/Storage.Sqlite.csproj b/storage/src/Storage.Sqlite/Storage.Sqlite.csproj new file mode 100644 index 000000000..4d2ede218 --- /dev/null +++ b/storage/src/Storage.Sqlite/Storage.Sqlite.csproj @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/storage/src/Storage/DatabaseSchemaVersion.cs b/storage/src/Storage/DatabaseSchemaVersion.cs new file mode 100644 index 000000000..42776f7fc --- /dev/null +++ b/storage/src/Storage/DatabaseSchemaVersion.cs @@ -0,0 +1,17 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage; + +public sealed record DatabaseSchemaVersion +{ + public int Value { get; } + + public DatabaseSchemaVersion(int value) + { + ArgumentOutOfRangeException.ThrowIfNegative(value); + Value = value; + } + + public static readonly DatabaseSchemaVersion Zero = new(0); +} diff --git a/storage/src/Storage/EntityAttributeValue/AttributeCode.cs b/storage/src/Storage/EntityAttributeValue/AttributeCode.cs new file mode 100644 index 000000000..a097a071d --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/AttributeCode.cs @@ -0,0 +1,58 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +/// +/// Represents a schema attribute code (programmatic identifier). Preserves the original +/// casing but compares case-insensitively (using ordinal ignore-case) for equality and hashing. +/// +[StringValue] +public partial record AttributeCode +{ + private const int MaxLength = 100; + + private static readonly StringComparer Comparer = StringComparer.OrdinalIgnoreCase; + + static string Normalize(string value) => value.Trim(); + + private static bool TryValidate(string? input, out IReadOnlyList? errors) + { + errors = null; + + if (input is null or { Length: 0 }) + { + errors = ["A value is required."]; + return false; + } + + var validationErrors = new List(); + + if (!char.IsAsciiLetter(input[0])) + { + validationErrors.Add("Must start with an ASCII letter."); + } + + if (input[^1] == '_') + { + validationErrors.Add("Must not end with an underscore."); + } + + foreach (var c in input.AsSpan()) + { + if (!char.IsAsciiLetter(c) && !char.IsAsciiDigit(c) && c != '_') + { + validationErrors.Add("Must only contain ASCII letters, digits, or underscores."); + break; + } + } + + if (validationErrors.Count > 0) + { + errors = validationErrors; + return false; + } + + return true; + } +} diff --git a/storage/src/Storage/EntityAttributeValue/AttributeCode.g.cs b/storage/src/Storage/EntityAttributeValue/AttributeCode.g.cs new file mode 100644 index 000000000..171a5e09e --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/AttributeCode.g.cs @@ -0,0 +1,95 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Collections.Generic; +using Duende.Storage; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.EntityAttributeValue; + +[System.ComponentModel.TypeConverter(typeof(ValueOfTypeConverter))] +partial record AttributeCode : IStringValue +{ + // Constructor for controlled creation + private AttributeCode(string value) => Value = value; + + public string Value { get; } + + public static AttributeCode Create(string s) + { + if (!TryCreate(s, out var result, out var errors)) + { + throw new FormatException($"The value '{s}' is not a valid AttributeCode. {string.Join(" ", errors)}"); + } + return result; + } + + public static bool TryCreate(string? s, [NotNullWhen(true)] out AttributeCode? result) + => TryCreate(s, out result, out _); + + public static bool TryCreate(string? s, [NotNullWhen(true)] out AttributeCode? result, [NotNullWhen(false)] out IReadOnlyList? errors) + { + result = null; + errors = null; + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["A value is required."]; + return false; + } + + s = Normalize(s); + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["Value is empty after normalization."]; + return false; + } + var validationErrors = new List(); + if (s.Length > MaxLength) + { + validationErrors.Add($"Must not exceed {MaxLength} characters."); + } + if (!TryValidate(s, out var tryValidateErrors)) + { + if (tryValidateErrors is { Count: > 0 }) + { + validationErrors.AddRange(tryValidateErrors); + } + else + { + validationErrors.Add($"The value '{s}' is not valid."); + } + } + if (validationErrors.Count > 0) + { + errors = validationErrors; + return false; + } + result = new AttributeCode(s); + return true; + } + + public static implicit operator AttributeCode(string value) => Create(value); + + public override string ToString() => Value; + + public static AttributeCode? CreateOrDefault(string? input) + { + if (string.IsNullOrEmpty(input)) + { + return null; + } + + return Create(input); + } + + internal static AttributeCode Load(string value) => new AttributeCode(value); + + public virtual bool Equals(AttributeCode? other) => + other is not null && Comparer.Equals(Value, other.Value); + + public override int GetHashCode() => + Value is null ? 0 : Comparer.GetHashCode(Value); + +} diff --git a/storage/src/Storage/EntityAttributeValue/AttributeDefinition.cs b/storage/src/Storage/EntityAttributeValue/AttributeDefinition.cs new file mode 100644 index 000000000..7a9e6df19 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/AttributeDefinition.cs @@ -0,0 +1,187 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +public sealed record AttributeDefinition +{ + /// + /// Creates a definition with a scalar data type. + /// + public AttributeDefinition( + AttributeCode Code, + ScalarDataType DataType, + AttributeDescription? Description, + bool IsUnique, + IReadOnlyCollection? Tags) + : this(Code, new ScalarAttributeType(DataType), Description, IsUnique, Tags, null, 0) + { + } + + /// + /// Creates a definition with a scalar data type and IsUnique flag. + /// + public AttributeDefinition( + AttributeCode Code, + ScalarDataType DataType, + AttributeDescription? Description, + bool IsUnique) + : this(Code, new ScalarAttributeType(DataType), Description, IsUnique, null, null, 0) + { + } + + /// + /// Creates a definition with a scalar data type (convenience overload without IsUnique/Tags). + /// + public AttributeDefinition( + AttributeCode Code, + ScalarDataType DataType, + AttributeDescription? Description) + : this(Code, new ScalarAttributeType(DataType), Description, false, null, null, 0) + { + } + + /// + /// Creates a definition with any . + /// + public AttributeDefinition( + AttributeCode Code, + AttributeType AttributeType, + AttributeDescription? Description, + bool IsUnique, + IReadOnlyCollection? Tags) + : this(Code, AttributeType, Description, IsUnique, Tags, null, 0) + { + } + + /// + /// Creates a definition with any (convenience overload without IsUnique/Tags). + /// + public AttributeDefinition( + AttributeCode Code, + AttributeType AttributeType, + AttributeDescription? Description) + : this(Code, AttributeType, Description, false, null, null, 0) + { + } + + /// + /// Creates a definition with a scalar data type, group, and order. + /// + public AttributeDefinition( + AttributeCode Code, + ScalarDataType DataType, + AttributeDescription? Description, + bool IsUnique, + IReadOnlyCollection? Tags, + AttributeGroupCode? GroupCode, + int Order) + : this(Code, new ScalarAttributeType(DataType), Description, IsUnique, Tags, GroupCode, Order) + { + } + + /// + /// Creates a definition with any , group, and order. + /// + public AttributeDefinition( + AttributeCode Code, + AttributeType AttributeType, + AttributeDescription? Description, + bool IsUnique, + IReadOnlyCollection? Tags, + AttributeGroupCode? GroupCode, + int Order) + { + if (IsUnique && AttributeType is ComplexAttributeType or ListAttributeType) + { + throw new ArgumentException( + "IsUnique is not supported for complex or list attribute types.", + nameof(IsUnique)); + } + + this.Code = Code; + this.AttributeType = AttributeType; + this.Description = Description; + this.IsUnique = IsUnique; + this.Tags = Tags ?? []; + this.GroupCode = GroupCode; + this.Order = Order; + } + + public static AttributeDefinition Load( + AttributeCode code, + AttributeType attributeType, + AttributeDescription? description, + bool isUnique, + IReadOnlyCollection tags, + AttributeGroupCode? groupCode, + int order) => + Load(code, attributeType, description, isUnique, tags, groupCode, order, null); + + public static AttributeDefinition Load( + AttributeCode code, + AttributeType attributeType, + AttributeDescription? description, + bool isUnique, + IReadOnlyCollection tags, + AttributeGroupCode? groupCode, + int order, + AttributeDisplayName? displayName) => + new(code, attributeType, description, isUnique, tags, groupCode, order) + { + DisplayName = displayName + }; + + public AttributeCode Code { get; } + + /// + /// The full type descriptor for this attribute. + /// + public AttributeType AttributeType { get; } + + /// + /// Convenience accessor for scalar attribute types. + /// Throws for non-scalar types. + /// + public ScalarDataType DataType => + AttributeType is ScalarAttributeType scalar + ? scalar.DataType + : throw new InvalidOperationException( + $"Attribute '{Code}' has type '{AttributeType.GetType().Name}', not a scalar type. Use AttributeType instead."); + + public AttributeDescription? Description { get; } + public AttributeDisplayName? DisplayName { get; init; } + public bool IsUnique { get; } + public IReadOnlyCollection Tags { get; } + + /// + /// The group this attribute belongs to, or null if ungrouped. + /// + public AttributeGroupCode? GroupCode { get; } + + /// + /// Sort weight controlling display order within the group (or among ungrouped attributes). + /// Not required to be unique; ties are resolved by a stable secondary sort (e.g. name). + /// + public int Order { get; } + +#pragma warning disable CA2225 // Operator overloads have named alternates + public static implicit operator AttributeCode(AttributeDefinition definition) + { + ArgumentNullException.ThrowIfNull(definition); + return definition.Code; + } +#pragma warning restore CA2225 + + /// + /// Replaces the compiler-generated PrintMembers (private for sealed records) + /// to avoid calling , which throws for non-scalar attribute types. + /// + private bool PrintMembers(System.Text.StringBuilder builder) + { + _ = builder.Append( + System.FormattableString.Invariant( + $"Code = {Code}, AttributeType = {AttributeType}, Description = {Description?.Value ?? "(none)"}, DisplayName = {DisplayName?.Value ?? "(none)"}, IsUnique = {IsUnique}, Tags = [{string.Join(", ", Tags)}], GroupCode = {GroupCode?.Value ?? "(none)"}, Order = {Order}")); + return true; + } +} diff --git a/storage/src/Storage/EntityAttributeValue/AttributeDescription.cs b/storage/src/Storage/EntityAttributeValue/AttributeDescription.cs new file mode 100644 index 000000000..ecce095f6 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/AttributeDescription.cs @@ -0,0 +1,15 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +/// +/// Represents a schema attribute description. +/// +[StringValue] +public partial record AttributeDescription +{ + internal const int MaxLength = 200; + + static string Normalize(string value) => value.Trim(); +} diff --git a/storage/src/Storage/EntityAttributeValue/AttributeDescription.g.cs b/storage/src/Storage/EntityAttributeValue/AttributeDescription.g.cs new file mode 100644 index 000000000..72e7a7ccc --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/AttributeDescription.g.cs @@ -0,0 +1,78 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Collections.Generic; +using Duende.Storage; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.EntityAttributeValue; + +[System.ComponentModel.TypeConverter(typeof(ValueOfTypeConverter))] +partial record AttributeDescription : IStringValue +{ + // Constructor for controlled creation + private AttributeDescription(string value) => Value = value; + + public string Value { get; } + + public static AttributeDescription Create(string s) + { + if (!TryCreate(s, out var result, out var errors)) + { + throw new FormatException($"The value '{s}' is not a valid AttributeDescription. {string.Join(" ", errors)}"); + } + return result; + } + + public static bool TryCreate(string? s, [NotNullWhen(true)] out AttributeDescription? result) + => TryCreate(s, out result, out _); + + public static bool TryCreate(string? s, [NotNullWhen(true)] out AttributeDescription? result, [NotNullWhen(false)] out IReadOnlyList? errors) + { + result = null; + errors = null; + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["A value is required."]; + return false; + } + + s = Normalize(s); + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["Value is empty after normalization."]; + return false; + } + var validationErrors = new List(); + if (s.Length > MaxLength) + { + validationErrors.Add($"Must not exceed {MaxLength} characters."); + } + if (validationErrors.Count > 0) + { + errors = validationErrors; + return false; + } + result = new AttributeDescription(s); + return true; + } + + public static implicit operator AttributeDescription(string value) => Create(value); + + public override string ToString() => Value; + + public static AttributeDescription? CreateOrDefault(string? input) + { + if (string.IsNullOrEmpty(input)) + { + return null; + } + + return Create(input); + } + + internal static AttributeDescription Load(string value) => new AttributeDescription(value); + +} diff --git a/storage/src/Storage/EntityAttributeValue/AttributeDisplayName.cs b/storage/src/Storage/EntityAttributeValue/AttributeDisplayName.cs new file mode 100644 index 000000000..fe2d9622c --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/AttributeDisplayName.cs @@ -0,0 +1,15 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +/// +/// Represents a human-readable display name for an attribute or attribute group. +/// +[StringValue] +public partial record AttributeDisplayName +{ + internal const int MaxLength = 200; + + static string Normalize(string value) => value.Trim(); +} diff --git a/storage/src/Storage/EntityAttributeValue/AttributeDisplayName.g.cs b/storage/src/Storage/EntityAttributeValue/AttributeDisplayName.g.cs new file mode 100644 index 000000000..7bbf93de8 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/AttributeDisplayName.g.cs @@ -0,0 +1,78 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Collections.Generic; +using Duende.Storage; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.EntityAttributeValue; + +[System.ComponentModel.TypeConverter(typeof(ValueOfTypeConverter))] +partial record AttributeDisplayName : IStringValue +{ + // Constructor for controlled creation + private AttributeDisplayName(string value) => Value = value; + + public string Value { get; } + + public static AttributeDisplayName Create(string s) + { + if (!TryCreate(s, out var result, out var errors)) + { + throw new FormatException($"The value '{s}' is not a valid AttributeDisplayName. {string.Join(" ", errors)}"); + } + return result; + } + + public static bool TryCreate(string? s, [NotNullWhen(true)] out AttributeDisplayName? result) + => TryCreate(s, out result, out _); + + public static bool TryCreate(string? s, [NotNullWhen(true)] out AttributeDisplayName? result, [NotNullWhen(false)] out IReadOnlyList? errors) + { + result = null; + errors = null; + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["A value is required."]; + return false; + } + + s = Normalize(s); + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["Value is empty after normalization."]; + return false; + } + var validationErrors = new List(); + if (s.Length > MaxLength) + { + validationErrors.Add($"Must not exceed {MaxLength} characters."); + } + if (validationErrors.Count > 0) + { + errors = validationErrors; + return false; + } + result = new AttributeDisplayName(s); + return true; + } + + public static implicit operator AttributeDisplayName(string value) => Create(value); + + public override string ToString() => Value; + + public static AttributeDisplayName? CreateOrDefault(string? input) + { + if (string.IsNullOrEmpty(input)) + { + return null; + } + + return Create(input); + } + + internal static AttributeDisplayName Load(string value) => new AttributeDisplayName(value); + +} diff --git a/storage/src/Storage/EntityAttributeValue/AttributeGroup.cs b/storage/src/Storage/EntityAttributeValue/AttributeGroup.cs new file mode 100644 index 000000000..e9ced8260 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/AttributeGroup.cs @@ -0,0 +1,27 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +/// +/// Represents a named group of attributes with display metadata and sort ordering. +/// +public sealed record AttributeGroup +{ + public AttributeGroup( + AttributeGroupCode Code, + AttributeDisplayName? DisplayName, + AttributeDescription? Description, + int Order) + { + this.Code = Code; + this.DisplayName = DisplayName; + this.Description = Description; + this.Order = Order; + } + + public AttributeGroupCode Code { get; init; } + public AttributeDisplayName? DisplayName { get; init; } + public AttributeDescription? Description { get; init; } + public int Order { get; init; } +} diff --git a/storage/src/Storage/EntityAttributeValue/AttributeGroupCode.cs b/storage/src/Storage/EntityAttributeValue/AttributeGroupCode.cs new file mode 100644 index 000000000..b6c0e5bf5 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/AttributeGroupCode.cs @@ -0,0 +1,23 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Text.RegularExpressions; + +namespace Duende.Storage.EntityAttributeValue; + +/// +/// Represents an attribute group code. Preserves the original casing of the name +/// but compares case-insensitively (using ordinal ignore-case) for equality and hashing. +/// +[StringValue] +public partial record AttributeGroupCode +{ + internal const int MaxLength = 100; + + internal static readonly StringComparer Comparer = StringComparer.OrdinalIgnoreCase; + + [GeneratedRegex(@"^[a-zA-Z0-9_-]+$")] + internal static partial Regex Regex(); + + static string Normalize(string value) => value.Trim(); +} diff --git a/storage/src/Storage/EntityAttributeValue/AttributeGroupCode.g.cs b/storage/src/Storage/EntityAttributeValue/AttributeGroupCode.g.cs new file mode 100644 index 000000000..b2a9dbae3 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/AttributeGroupCode.g.cs @@ -0,0 +1,88 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Collections.Generic; +using Duende.Storage; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.EntityAttributeValue; + +[System.ComponentModel.TypeConverter(typeof(ValueOfTypeConverter))] +partial record AttributeGroupCode : IStringValue +{ + // Constructor for controlled creation + private AttributeGroupCode(string value) => Value = value; + + public string Value { get; } + + public static AttributeGroupCode Create(string s) + { + if (!TryCreate(s, out var result, out var errors)) + { + throw new FormatException($"The value '{s}' is not a valid AttributeGroupCode. {string.Join(" ", errors)}"); + } + return result; + } + + public static bool TryCreate(string? s, [NotNullWhen(true)] out AttributeGroupCode? result) + => TryCreate(s, out result, out _); + + public static bool TryCreate(string? s, [NotNullWhen(true)] out AttributeGroupCode? result, [NotNullWhen(false)] out IReadOnlyList? errors) + { + result = null; + errors = null; + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["A value is required."]; + return false; + } + + s = Normalize(s); + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["Value is empty after normalization."]; + return false; + } + var validationErrors = new List(); + if (s.Length > MaxLength) + { + validationErrors.Add($"Must not exceed {MaxLength} characters."); + } + if (!Regex().IsMatch(s)) + { + validationErrors.Add("Must match the required pattern."); + } + if (validationErrors.Count > 0) + { + errors = validationErrors; + return false; + } + result = new AttributeGroupCode(s); + return true; + } + + public static implicit operator AttributeGroupCode(string value) => Create(value); + + public override string ToString() => Value; + + public static AttributeGroupCode? CreateOrDefault(string? input) + { + if (string.IsNullOrEmpty(input)) + { + return null; + } + + return Create(input); + } + + internal static AttributeGroupCode Load(string value) => new AttributeGroupCode(value); + + public virtual bool Equals(AttributeGroupCode? other) => + other is not null && Comparer.Equals(Value, other.Value); + + public override int GetHashCode() => + Value is null ? 0 : Comparer.GetHashCode(Value); + +} diff --git a/storage/src/Storage/EntityAttributeValue/AttributeType.cs b/storage/src/Storage/EntityAttributeValue/AttributeType.cs new file mode 100644 index 000000000..cfdf8c78b --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/AttributeType.cs @@ -0,0 +1,46 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +/// +/// Base type for schema attribute type descriptors. +/// +public abstract record AttributeType +{ + internal AttributeType() { } + + /// + /// Validates that no list is nested inside another list at any depth. + /// + internal void ValidateNesting() => ValidateNesting(insideList: false); + + private void ValidateNesting(bool insideList) + { + switch (this) + { + case ScalarAttributeType: + // leaf types — always valid + break; + + case ComplexAttributeType complex: + foreach (var (_, prop) in complex.Properties) + { + prop.Type.ValidateNesting(insideList); + } + break; + + case ListAttributeType list: + if (insideList) + { + throw new ArgumentException("List types cannot be nested inside another list type."); + } + list.ElementType.ValidateNesting(insideList: true); + break; + + default: + throw new InvalidOperationException( + $"Unsupported {nameof(AttributeType)} subtype encountered in {nameof(ValidateNesting)}: {GetType().FullName}"); + } + } +} diff --git a/storage/src/Storage/EntityAttributeValue/AttributeValue.cs b/storage/src/Storage/EntityAttributeValue/AttributeValue.cs new file mode 100644 index 000000000..6559f8c95 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/AttributeValue.cs @@ -0,0 +1,51 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Collections.ObjectModel; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.EntityAttributeValue; + +#pragma warning disable CA1711 // Identifiers should not have incorrect suffix +public abstract record AttributeValue +{ + private protected AttributeValue(AttributeCode code) => Code = code; + + public AttributeCode Code { get; } + + public abstract object UntypedValue { get; } + + public bool TryGetValue([MaybeNullWhen(false)] out T value) + { + if (this is AttributeValue typed) + { + value = typed.TypedValue; + return true; + } + + value = default; + return false; + } + + public override string ToString() => UntypedValue.ToString()!; + + public static AttributeValue Load(AttributeCode code, T value) => new(code, value); +} + +public sealed record AttributeValue : AttributeValue +{ + internal AttributeValue(AttributeCode code, T value) : base(code) => + TypedValue = value switch + { + IReadOnlyDictionary dict => (T)(object)new ReadOnlyDictionary( + new Dictionary(dict)), + IReadOnlyList list => (T)(object)list.ToList().AsReadOnly(), + _ => value + }; + + public T TypedValue { get; } + + public override object UntypedValue => TypedValue!; + + internal static AttributeValue Load(AttributeCode code, T value) => new(code, value); +} diff --git a/storage/src/Storage/EntityAttributeValue/AttributeValueCollection.cs b/storage/src/Storage/EntityAttributeValue/AttributeValueCollection.cs new file mode 100644 index 000000000..1184f5f3e --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/AttributeValueCollection.cs @@ -0,0 +1,49 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Collections; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.EntityAttributeValue; + +public sealed class AttributeValueCollection : IEnumerable +{ + private readonly Dictionary _dict = []; + + public AttributeValueCollection() { } + + public AttributeValueCollection(IEnumerable attributes) + { + foreach (var attribute in attributes) + { + if (!_dict.TryAdd(attribute.Code, attribute)) + { + throw new ArgumentException( + $"The attributes contain more than one attribute named '{attribute.Code}'", nameof(attributes)); + } + } + } + + public void Set(AttributeValue attribute) + { + ArgumentNullException.ThrowIfNull(attribute); + _dict[attribute.Code] = attribute; + } + + public int Count => _dict.Count; + + public bool Remove(AttributeCode code) => _dict.Remove(code); + + public bool Contains(AttributeCode code) => _dict.ContainsKey(code); + + public bool TryGet(AttributeCode code, [MaybeNullWhen(false)] out AttributeValue attribute) => + _dict.TryGetValue(code, out attribute); + +#pragma warning disable CA1043 // Use integral or string argument for indexers + public AttributeValue this[AttributeCode code] => _dict[code]; +#pragma warning restore CA1043 + + public IEnumerator GetEnumerator() => _dict.Values.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); +} diff --git a/storage/src/Storage/EntityAttributeValue/ComplexAttributeProperty.cs b/storage/src/Storage/EntityAttributeValue/ComplexAttributeProperty.cs new file mode 100644 index 000000000..4e0e68f7b --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/ComplexAttributeProperty.cs @@ -0,0 +1,43 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +/// +/// Represents a sub-property within a , +/// bundling the property's type with optional display metadata. +/// +public sealed record ComplexAttributeProperty +{ + private ComplexAttributeProperty(AttributeType type, AttributeDisplayName? displayName, AttributeDescription? description) + { + Type = type; + DisplayName = displayName; + Description = description; + } + + /// The type of this sub-property. + public AttributeType Type { get; } + + /// Optional human-readable display name for this sub-property. + public AttributeDisplayName? DisplayName { get; } + + /// Optional description for this sub-property. + public AttributeDescription? Description { get; } + + /// Creates a property with a scalar type and no metadata. + public static ComplexAttributeProperty Of(ScalarDataType dataType) => + new(new ScalarAttributeType(dataType), null, null); + + /// Creates a property with the given type and no metadata. + public static ComplexAttributeProperty Of(AttributeType type) => + new(type, null, null); + + /// Creates a property with the given type and display name. + public static ComplexAttributeProperty Of(AttributeType type, AttributeDisplayName? displayName) => + new(type, displayName, null); + + /// Creates a property with the given type and optional metadata. + public static ComplexAttributeProperty Of(AttributeType type, AttributeDisplayName? displayName, AttributeDescription? description) => + new(type, displayName, description); +} diff --git a/storage/src/Storage/EntityAttributeValue/ComplexAttributeType.cs b/storage/src/Storage/EntityAttributeValue/ComplexAttributeType.cs new file mode 100644 index 000000000..e30b7f0ac --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/ComplexAttributeType.cs @@ -0,0 +1,108 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.EntityAttributeValue; + +/// +/// Represents a complex (nested object) attribute type with named properties. +/// +public sealed record ComplexAttributeType : AttributeType +{ + public ComplexAttributeType(IReadOnlyDictionary Properties) + { + ArgumentNullException.ThrowIfNull(Properties, nameof(Properties)); + + if (Properties.Count == 0) + { + throw new ArgumentException("Properties must contain at least one entry.", nameof(Properties)); + } + + foreach (var (_, value) in Properties) + { + ArgumentNullException.ThrowIfNull(value, nameof(Properties)); + } + + // AttributeCode already implements case-insensitive Equals/GetHashCode via the generated code. + var dict = new Dictionary(); + foreach (var (k, v) in Properties) + { + dict[k] = v; + } + this.Properties = dict; + } + + /// + /// The named sub-properties and their types. All properties are optional — complex + /// values may contain any subset of the defined properties. Unknown properties + /// (not listed here) are rejected during validation. + /// + public IReadOnlyDictionary Properties { get; } + + /// + /// Tries to get a property by name (case-insensitive) and returns the schema-canonical + /// key alongside the property. This ensures callers can normalize to the schema-defined casing. + /// + public bool TryGetProperty(string name, [NotNullWhen(true)] out AttributeCode? canonicalKey, out ComplexAttributeProperty property) + { + if (!AttributeCode.TryCreate(name, out var code)) + { + canonicalKey = null; + property = default!; + return false; + } + + // The underlying dictionary uses AttributeCode.Comparer (OrdinalIgnoreCase), so TryGetValue matches case-insensitively. + // To recover the canonical key we walk Keys — the dictionary is small (schema-defined properties). + if (Properties.TryGetValue(code, out property!)) + { + foreach (var key in Properties.Keys) + { + if (key == code) + { + canonicalKey = key; + return true; + } + } + } + + canonicalKey = null; + property = default!; + return false; + } + + public bool Equals(ComplexAttributeType? other) + { + if (other is null) + { + return false; + } + + if (Properties.Count != other.Properties.Count) + { + return false; + } + + foreach (var (key, value) in Properties) + { + if (!other.Properties.TryGetValue(key, out var otherValue) || !value.Equals(otherValue)) + { + return false; + } + } + + return true; + } + + public override int GetHashCode() + { + var hash = new HashCode(); + foreach (var (key, value) in Properties.OrderBy(p => p.Key.Value, StringComparer.OrdinalIgnoreCase)) + { + hash.Add(key); + hash.Add(value); + } + return hash.ToHashCode(); + } +} diff --git a/storage/src/Storage/EntityAttributeValue/IReadOnlyAttributeSchema.cs b/storage/src/Storage/EntityAttributeValue/IReadOnlyAttributeSchema.cs new file mode 100644 index 000000000..d0b9fb5fb --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/IReadOnlyAttributeSchema.cs @@ -0,0 +1,66 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.EntityAttributeValue; + +public interface IReadOnlyAttributeSchema +{ + IReadOnlyDictionary AttributeDefinitions { get; } + + /// + /// The named groups defined in this schema, keyed by group name. + /// + IReadOnlyDictionary Groups { get; } + + AttributeValue CreateAttribute(AttributeCode code, bool value); + + AttributeValue CreateAttribute(AttributeCode code, DateOnly value); + + AttributeValue CreateAttribute(AttributeCode code, DateTimeOffset value); + + AttributeValue CreateAttribute(AttributeCode code, decimal value); + + AttributeValue CreateAttribute(AttributeCode code, int value); + + AttributeValue CreateAttribute(AttributeCode code, string value); + + AttributeValue> CreateAttribute(AttributeCode code, IReadOnlyDictionary complexValue); + + AttributeValue> CreateAttribute(AttributeCode code, IReadOnlyList listValue); + + bool TryCreateAttribute(AttributeCode code, bool value, [NotNullWhen(true)] out AttributeValue? attribute); + + bool TryCreateAttribute(AttributeCode code, DateOnly value, [NotNullWhen(true)] out AttributeValue? attribute); + + bool TryCreateAttribute(AttributeCode code, DateTimeOffset value, [NotNullWhen(true)] out AttributeValue? attribute); + + bool TryCreateAttribute(AttributeCode code, decimal value, [NotNullWhen(true)] out AttributeValue? attribute); + + bool TryCreateAttribute(AttributeCode code, int value, [NotNullWhen(true)] out AttributeValue? attribute); + + bool TryCreateAttribute(AttributeCode code, string value, [NotNullWhen(true)] out AttributeValue? attribute); + + bool TryCreateAttribute(AttributeCode code, IReadOnlyDictionary complexValue, [NotNullWhen(true)] out AttributeValue>? attribute); + + bool TryCreateAttribute(AttributeCode code, IReadOnlyList listValue, [NotNullWhen(true)] out AttributeValue>? attribute); + + bool TryCreateAttribute(AttributeCode code, bool value, [NotNullWhen(true)] out AttributeValue? attribute, [NotNullWhen(false)] out IReadOnlyList? errors); + + bool TryCreateAttribute(AttributeCode code, DateOnly value, [NotNullWhen(true)] out AttributeValue? attribute, [NotNullWhen(false)] out IReadOnlyList? errors); + + bool TryCreateAttribute(AttributeCode code, DateTimeOffset value, [NotNullWhen(true)] out AttributeValue? attribute, [NotNullWhen(false)] out IReadOnlyList? errors); + + bool TryCreateAttribute(AttributeCode code, decimal value, [NotNullWhen(true)] out AttributeValue? attribute, [NotNullWhen(false)] out IReadOnlyList? errors); + + bool TryCreateAttribute(AttributeCode code, int value, [NotNullWhen(true)] out AttributeValue? attribute, [NotNullWhen(false)] out IReadOnlyList? errors); + + bool TryCreateAttribute(AttributeCode code, string value, [NotNullWhen(true)] out AttributeValue? attribute, [NotNullWhen(false)] out IReadOnlyList? errors); + + bool TryCreateAttribute(AttributeCode code, IReadOnlyDictionary complexValue, [NotNullWhen(true)] out AttributeValue>? attribute, [NotNullWhen(false)] out IReadOnlyList? errors); + + bool TryCreateAttribute(AttributeCode code, IReadOnlyList listValue, [NotNullWhen(true)] out AttributeValue>? attribute, [NotNullWhen(false)] out IReadOnlyList? errors); + + AttributeValueCollection CreateAttributes(IEnumerable attributes); +} diff --git a/storage/src/Storage/EntityAttributeValue/Internal/AttributeSchema.cs b/storage/src/Storage/EntityAttributeValue/Internal/AttributeSchema.cs new file mode 100644 index 000000000..75a2638f8 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/Internal/AttributeSchema.cs @@ -0,0 +1,405 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Diagnostics.CodeAnalysis; +using Duende.Storage.Internal.Querying; + +namespace Duende.Storage.EntityAttributeValue.Internal; + +/// +/// Represents a dynamic collection of attributes. +/// +public sealed class AttributeSchema : IReadOnlyAttributeSchema +{ + private readonly Dictionary _attributesDefinitions; + private readonly Dictionary _groups; + + private AttributeSchema(IEnumerable attributeDefinitions, IEnumerable groups) + { + _attributesDefinitions = new Dictionary(); + foreach (var d in attributeDefinitions) + { + _attributesDefinitions[d.Code] = d; // Last-write-wins for duplicates from storage + } + + _groups = new Dictionary(); + foreach (var g in groups) + { + _groups[g.Code] = g; // Last-write-wins for duplicates from storage + } + } + + public AttributeSchema() : this([], []) + { + } + + public IReadOnlyDictionary AttributeDefinitions => _attributesDefinitions; + + public IReadOnlyDictionary Groups => _groups; + + public bool AddGroup(AttributeGroup group) => _groups.TryAdd(group.Code, group); + + public bool RemoveGroup(AttributeGroupCode name) + { + if (!_groups.Remove(name)) + { + return false; + } + + // Ungroup all attributes that referenced this group + var toUngroup = _attributesDefinitions.Values + .Where(d => d.GroupCode != null && d.GroupCode.Equals(name)) + .ToList(); + + foreach (var definition in toUngroup) + { + _attributesDefinitions[definition.Code] = AttributeDefinition.Load( + definition.Code, + definition.AttributeType, + definition.Description, + definition.IsUnique, + definition.Tags, + null, + definition.Order); + } + + return true; + } + + public bool UpdateGroup(AttributeGroup group) + { + if (!_groups.ContainsKey(group.Code)) + { + return false; + } + + _groups[group.Code] = group; + return true; + } + + public bool AddAttributeDefinition(AttributeDefinition definition) + { + if (SystemFields.IsReservedAttributeName(definition.Code.Value)) + { + return false; + } + + if (definition.GroupCode != null && !_groups.ContainsKey(definition.GroupCode)) + { + return false; + } + + return _attributesDefinitions.TryAdd(definition.Code, definition); + } + + public void RemoveAttributeDefinition(AttributeCode code) => _ = _attributesDefinitions.Remove(code); + + public AttributeValue CreateAttribute(AttributeCode code, bool value) => + TryCreateAttribute(code, value, out var attribute, out var errors) + ? attribute + : throw new ArgumentException(string.Join("; ", errors)); + + public AttributeValue CreateAttribute(AttributeCode code, DateOnly value) => + TryCreateAttribute(code, value, out var attribute, out var errors) + ? attribute + : throw new ArgumentException(string.Join("; ", errors)); + + public AttributeValue CreateAttribute(AttributeCode code, DateTimeOffset value) => + TryCreateAttribute(code, value, out var attribute, out var errors) + ? attribute + : throw new ArgumentException(string.Join("; ", errors)); + + public AttributeValue CreateAttribute(AttributeCode code, decimal value) => + TryCreateAttribute(code, value, out var attribute, out var errors) + ? attribute + : throw new ArgumentException(string.Join("; ", errors)); + + public AttributeValue CreateAttribute(AttributeCode code, int value) => + TryCreateAttribute(code, value, out var attribute, out var errors) + ? attribute + : throw new ArgumentException(string.Join("; ", errors)); + + public AttributeValue CreateAttribute(AttributeCode code, string value) => + TryCreateAttribute(code, value, out var attribute, out var errors) + ? attribute + : throw new ArgumentException(string.Join("; ", errors)); + + public AttributeValue> CreateAttribute(AttributeCode code, IReadOnlyDictionary complexValue) => + TryCreateAttribute(code, complexValue, out var attribute, out var errors) + ? attribute + : throw new ArgumentException(string.Join("; ", errors)); + + public AttributeValue> CreateAttribute(AttributeCode code, IReadOnlyList listValue) => + TryCreateAttribute(code, listValue, out var attribute, out var errors) + ? attribute + : throw new ArgumentException(string.Join("; ", errors)); + + public bool TryCreateAttribute(AttributeCode code, bool value, [NotNullWhen(true)] out AttributeValue? attribute) => + TryCreateAttribute(code, value, out attribute, out _); + + public bool TryCreateAttribute(AttributeCode code, DateOnly value, [NotNullWhen(true)] out AttributeValue? attribute) => + TryCreateAttribute(code, value, out attribute, out _); + + public bool TryCreateAttribute(AttributeCode code, DateTimeOffset value, [NotNullWhen(true)] out AttributeValue? attribute) => + TryCreateAttribute(code, value, out attribute, out _); + + public bool TryCreateAttribute(AttributeCode code, decimal value, [NotNullWhen(true)] out AttributeValue? attribute) => + TryCreateAttribute(code, value, out attribute, out _); + + public bool TryCreateAttribute(AttributeCode code, int value, [NotNullWhen(true)] out AttributeValue? attribute) => + TryCreateAttribute(code, value, out attribute, out _); + + public bool TryCreateAttribute(AttributeCode code, string value, [NotNullWhen(true)] out AttributeValue? attribute) => + TryCreateAttribute(code, value, out attribute, out _); + + public bool TryCreateAttribute(AttributeCode code, IReadOnlyDictionary complexValue, [NotNullWhen(true)] out AttributeValue>? attribute) => + TryCreateAttribute(code, complexValue, out attribute, out _); + + public bool TryCreateAttribute(AttributeCode code, IReadOnlyList listValue, [NotNullWhen(true)] out AttributeValue>? attribute) => + TryCreateAttribute(code, listValue, out attribute, out _); + + public bool TryCreateAttribute(AttributeCode code, bool value, [NotNullWhen(true)] out AttributeValue? attribute, [NotNullWhen(false)] out IReadOnlyList? errors) => + TryCreateScalarAttribute(code, value, ScalarDataType.Boolean, out attribute, out errors); + + public bool TryCreateAttribute(AttributeCode code, DateOnly value, [NotNullWhen(true)] out AttributeValue? attribute, [NotNullWhen(false)] out IReadOnlyList? errors) => + TryCreateScalarAttribute(code, value, ScalarDataType.Date, out attribute, out errors); + + public bool TryCreateAttribute(AttributeCode code, DateTimeOffset value, [NotNullWhen(true)] out AttributeValue? attribute, [NotNullWhen(false)] out IReadOnlyList? errors) => + TryCreateScalarAttribute(code, value, ScalarDataType.DateTime, out attribute, out errors); + + public bool TryCreateAttribute(AttributeCode code, decimal value, [NotNullWhen(true)] out AttributeValue? attribute, [NotNullWhen(false)] out IReadOnlyList? errors) => + TryCreateScalarAttribute(code, value, ScalarDataType.Decimal, out attribute, out errors); + + public bool TryCreateAttribute(AttributeCode code, int value, [NotNullWhen(true)] out AttributeValue? attribute, [NotNullWhen(false)] out IReadOnlyList? errors) => + TryCreateScalarAttribute(code, value, ScalarDataType.Integer, out attribute, out errors); + + public bool TryCreateAttribute(AttributeCode code, string value, [NotNullWhen(true)] out AttributeValue? attribute, [NotNullWhen(false)] out IReadOnlyList? errors) + { + if (!_attributesDefinitions.TryGetValue(code, out var definition)) + { + attribute = null; + errors = [$"Attribute '{code}' is not defined in the schema."]; + return false; + } + + if (definition.AttributeType is not ScalarAttributeType scalar || scalar.DataType != ScalarDataType.String) + { + var providedType = typeof(string).Name; + var definedType = definition.AttributeType is ScalarAttributeType s ? s.DataType.ToString() : definition.AttributeType.GetType().Name; + attribute = null; + errors = [$"Attribute '{code}' is defined as '{definedType}' but a '{providedType}' value was provided."]; + return false; + } + + attribute = new AttributeValue(code, value); + errors = null; + return true; + } + + public bool TryCreateAttribute(AttributeCode code, IReadOnlyDictionary complexValue, [NotNullWhen(true)] out AttributeValue>? attribute, [NotNullWhen(false)] out IReadOnlyList? errors) + { + if (!_attributesDefinitions.TryGetValue(code, out var definition)) + { + attribute = null; + errors = [$"Attribute '{code}' is not defined in the schema."]; + return false; + } + + if (definition.AttributeType is not ComplexAttributeType complexType) + { + attribute = null; + errors = [$"Attribute '{code}' is not a complex type."]; + return false; + } + + var errorList = new List(); + CollectComplexValueErrors(code, complexValue, complexType, errorList); + + if (errorList.Count > 0) + { + attribute = null; + errors = errorList; + return false; + } + + attribute = new AttributeValue>(code, complexValue); + errors = null; + return true; + } + + public bool TryCreateAttribute(AttributeCode code, IReadOnlyList listValue, [NotNullWhen(true)] out AttributeValue>? attribute, [NotNullWhen(false)] out IReadOnlyList? errors) + { + if (!_attributesDefinitions.TryGetValue(code, out var definition)) + { + attribute = null; + errors = [$"Attribute '{code}' is not defined in the schema."]; + return false; + } + + if (definition.AttributeType is not ListAttributeType listType) + { + attribute = null; + errors = [$"Attribute '{code}' is not a list type."]; + return false; + } + + var errorList = new List(); + CollectListValueErrors(code, listValue, listType, errorList); + + if (errorList.Count > 0) + { + attribute = null; + errors = errorList; + return false; + } + + attribute = new AttributeValue>(code, listValue); + errors = null; + return true; + } + + public AttributeValueCollection CreateAttributes(IEnumerable attributes) => new AttributeValueCollection(attributes); + + private bool TryCreateScalarAttribute(AttributeCode code, T value, ScalarDataType dataType, out AttributeValue? attribute, out IReadOnlyList? errors) + { + if (!_attributesDefinitions.TryGetValue(code, out var definition)) + { + attribute = null; + errors = [$"Attribute '{code}' is not defined in the schema."]; + return false; + } + + if (definition.AttributeType is not ScalarAttributeType scalar || scalar.DataType != dataType) + { + var providedType = typeof(T).Name; + var definedType = definition.AttributeType is ScalarAttributeType s ? s.DataType.ToString() : definition.AttributeType.GetType().Name; + attribute = null; + errors = [$"Attribute '{code}' is defined as '{definedType}' but a '{providedType}' value was provided."]; + return false; + } + + attribute = new AttributeValue(code, value); + errors = null; + return true; + } + + private static void CollectComplexValueErrors(AttributeCode code, IReadOnlyDictionary value, ComplexAttributeType complexType, List errors) + { + foreach (var (key, propValue) in value) + { + if (!complexType.TryGetProperty(key, out _, out var prop)) + { + errors.Add($"Property '{key}' is not defined in complex attribute '{code}'."); + continue; + } + + var expectedType = GetExpectedTypeName(prop.Type); + + if (propValue is null) + { + errors.Add($"Property '{key}' in attribute '{code}' expects type '{expectedType}' but got 'null'."); + continue; + } + + if (!ValidateValueAgainstType(propValue, prop.Type)) + { + var actualType = propValue.GetType().Name; + errors.Add($"Property '{key}' in attribute '{code}' expects type '{expectedType}' but got '{actualType}'."); + } + } + } + + private static void CollectListValueErrors(AttributeCode code, IReadOnlyList value, ListAttributeType listType, List errors) + { + for (var i = 0; i < value.Count; i++) + { + var element = value[i]; + + if (listType.ElementType is ComplexAttributeType complexElementType) + { + if (element is null) + { + errors.Add($"Element at index {i} in list attribute '{code}' expects type 'Complex' but got 'null'."); + } + else if (element is IReadOnlyDictionary dict) + { + var before = errors.Count; + CollectComplexValueErrors(code, dict, complexElementType, errors); + // Prefix element-level context to any errors added + for (var j = before; j < errors.Count; j++) + { + errors[j] = $"Element at index {i}: {errors[j]}"; + } + } + else + { + var actualType = element.GetType().Name; + errors.Add($"Element at index {i} in list attribute '{code}' expects type 'Complex' but got '{actualType}'."); + } + } + else if (element is null) + { + var expectedType = GetExpectedTypeName(listType.ElementType); + errors.Add($"Element at index {i} in list attribute '{code}' expects type '{expectedType}' but got 'null'."); + } + else if (!ValidateValueAgainstType(element, listType.ElementType)) + { + var expectedType = GetExpectedTypeName(listType.ElementType); + var actualType = element.GetType().Name; + errors.Add($"Element at index {i} in list attribute '{code}' expects type '{expectedType}' but got '{actualType}'."); + } + } + } + + private static string GetExpectedTypeName(AttributeType type) => + type switch + { + ScalarAttributeType scalar => scalar.DataType.ToString(), + ComplexAttributeType => "Complex", + ListAttributeType => "List", + _ => type.GetType().Name + }; + + private static bool ValidateValueAgainstType(object value, AttributeType type) => + type switch + { + ScalarAttributeType scalar => ValidateScalarValue(value, scalar.DataType), + ComplexAttributeType complexType => value is IReadOnlyDictionary dict && ValidateComplexValue(dict, complexType), + ListAttributeType listType => value is IReadOnlyList list && list.All(el => ValidateValueAgainstType(el, listType.ElementType)), + _ => false + }; + + private static bool ValidateComplexValue(IReadOnlyDictionary value, ComplexAttributeType complexType) + { + foreach (var (key, propValue) in value) + { + if (!complexType.TryGetProperty(key, out _, out var prop)) + { + return false; + } + + if (!ValidateValueAgainstType(propValue, prop.Type)) + { + return false; + } + } + + return true; + } + + private static bool ValidateScalarValue(object value, ScalarDataType dataType) => + dataType switch + { + ScalarDataType.Boolean => value is bool, + ScalarDataType.Date => value is DateOnly, + ScalarDataType.DateTime => value is DateTimeOffset, + ScalarDataType.Decimal => value is decimal, + ScalarDataType.Integer => value is int, + ScalarDataType.String => value is string, + _ => false + }; + + public static AttributeSchema Load(IEnumerable attributes) => new(attributes, []); + + public static AttributeSchema Load(IEnumerable attributes, IEnumerable groups) => new(attributes, groups); +} diff --git a/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeDefinitionDso.cs b/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeDefinitionDso.cs new file mode 100644 index 000000000..6e68f3218 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeDefinitionDso.cs @@ -0,0 +1,17 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue.Internal.Storage; + +public static class AttributeDefinitionDso +{ + public sealed record V1( + string Code, + AttributeTypeDso Type, + string? Description, + bool IsUnique, + IReadOnlyList Tags, + string? GroupCode, + int Order, + string? DisplayName); +} diff --git a/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeGroupDso.cs b/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeGroupDso.cs new file mode 100644 index 000000000..fe9177a0a --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeGroupDso.cs @@ -0,0 +1,9 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue.Internal.Storage; + +public static class AttributeGroupDso +{ + public sealed record V1(string Code, string? DisplayName, string? Description, int Order); +} diff --git a/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeSchemaDso.cs b/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeSchemaDso.cs new file mode 100644 index 000000000..1e2de9d64 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeSchemaDso.cs @@ -0,0 +1,16 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; + +namespace Duende.Storage.EntityAttributeValue.Internal.Storage; + +public static class AttributeSchemaDso +{ + public static readonly EntityType EntityType = new(1501, "UserProfileSchemaDso"); + + public sealed record V1(ICollection AttributeDefinitions, ICollection Groups) : IDataStorageObject + { + public static DataStorageObjectVersion DsoVersion { get; } = new(EntityType, 1); + } +} diff --git a/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeTypeDso.cs b/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeTypeDso.cs new file mode 100644 index 000000000..d2bc22986 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeTypeDso.cs @@ -0,0 +1,25 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue.Internal.Storage; + +/// +/// Persisted representation of an . +/// EnumValues and ConstrainedValues are reserved for future enum/constrained-string +/// attribute types and are currently always null. +/// +public sealed record AttributeTypeDso( + string Kind, + string? ScalarDataType, + IReadOnlyList? EnumValues, + IReadOnlyList? ConstrainedValues, + Dictionary? Properties, + AttributeTypeDso? ElementType); + +/// +/// Persisted representation of a sub-property within a complex attribute type. +/// +public sealed record ComplexPropertyDso( + AttributeTypeDso Type, + string? DisplayName, + string? Description); diff --git a/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeTypeResolver.cs b/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeTypeResolver.cs new file mode 100644 index 000000000..74b3fa7bd --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeTypeResolver.cs @@ -0,0 +1,130 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.EntityAttributeValue.Internal.Storage; + +/// +/// Resolves SCIM attribute paths to Faro Field types based on the dynamic user schema. +/// Unlike other implementations which map fixed SCIM User schema attributes, +/// this resolver dynamically maps user-defined schema attributes to their Faro field types. +/// Supports dotted paths (e.g., address.city, phones.type) for complex and list types. +/// +public sealed class AttributeTypeResolver : IScimAttributeTypeResolver +{ + private readonly IReadOnlyDictionary _attributeDefinitions; + + /// + /// Creates a new resolver with the given user schema attribute definitions. + /// + /// The schema attribute definitions for the user schema. + public AttributeTypeResolver( + IReadOnlyDictionary attributeDefinitions) => + _attributeDefinitions = attributeDefinitions ?? throw new ArgumentNullException(nameof(attributeDefinitions)); + + /// + public Field ResolveField(string attributePath) + { + var normalized = attributePath.Trim(); + + // Handle built-in SCIM User fields that are stored as first-class columns + if (string.Equals(normalized, "username", StringComparison.OrdinalIgnoreCase)) + { + return new StringField("userName"); + } + + // Handle system timestamp fields + if (normalized == SystemFields.CreatedAttributeName) + { + return SystemFields.CreatedAtField; + } + if (normalized == SystemFields.LastUpdatedAttributeName) + { + return SystemFields.LastUpdatedAtField; + } + + // Split on '.' to handle dotted paths (e.g., "address.city", "phones.type") + var segments = normalized.Split('.'); + var rootSegment = segments[0]; + + // Resolve the root attribute from dynamic schema + if (!AttributeCode.TryCreate(rootSegment, out var schemaCode) || + !_attributeDefinitions.TryGetValue(schemaCode, out var definition)) + { + throw new NotSupportedException($"Unknown user attribute: {attributePath}"); + } + + // Walk the remaining segments through the type tree + var isArray = false; + var currentType = definition.AttributeType; + + for (var i = 1; i < segments.Length; i++) + { + var segment = segments[i]; + + // Unwrap any list wrapper before navigating into the segment + if (currentType is ListAttributeType listWrapper) + { + isArray = true; + currentType = listWrapper.ElementType; + } + + switch (currentType) + { + case ComplexAttributeType complex: + if (!complex.TryGetProperty(segment, out _, out var prop)) + { + throw new NotSupportedException( + $"Unknown property '{segment}' on complex attribute '{attributePath}'."); + } + currentType = prop.Type; + break; + + default: + throw new NotSupportedException( + $"Cannot navigate into segment '{segment}' of type '{currentType.GetType().Name}' in path '{attributePath}'."); + } + } + + // Unwrap any trailing list wrapper (e.g., root type is List with no sub-path). + // For a root list attribute (e.g., "tags" where tags: List), we resolve to a + // multi-valued field so the evaluator checks all array-indexed entries. + if (currentType is ListAttributeType trailingList) + { + isArray = true; + currentType = trailingList.ElementType; + } + + return MapToField(currentType, normalized, isArray); + } + + private static Field MapToField(AttributeType type, string fullPath, bool isArray) => + type switch + { + ScalarAttributeType scalar => MapScalarToField(scalar.DataType, fullPath, isArray), + + ComplexAttributeType => throw new NotSupportedException( + $"Cannot query a complex attribute directly at path '{fullPath}'. Use a sub-property path."), + + ListAttributeType => throw new NotSupportedException( + $"Cannot query a list attribute directly at path '{fullPath}'. Use a sub-property path."), + + _ => throw new NotSupportedException($"Unsupported schema attribute type: {type.GetType().Name}") + }; + + private static Field MapScalarToField(ScalarDataType dataType, string fullPath, bool isMultiValued) => + dataType switch + { + ScalarDataType.Boolean => new BooleanField(fullPath, isMultiValued), + ScalarDataType.Date => new DateTimeField(fullPath, isMultiValued), + ScalarDataType.DateTime => new DateTimeField(fullPath, isMultiValued), + ScalarDataType.Decimal => new NumberField(fullPath, isMultiValued), + ScalarDataType.Integer => new NumberField(fullPath, isMultiValued), + ScalarDataType.String => new StringField(fullPath, isMultiValued), + _ => throw new NotSupportedException( + $"Unsupported schema attribute data type: {dataType} for path {fullPath}") + }; +} diff --git a/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeValueDskV1.cs b/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeValueDskV1.cs new file mode 100644 index 000000000..3a79156e0 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeValueDskV1.cs @@ -0,0 +1,36 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Globalization; +using Duende.Storage.Internal; + +namespace Duende.Storage.EntityAttributeValue.Internal.Storage; + +public sealed record AttributeValueDskV1 : IDataStorageKey +{ + private AttributeValueDskV1(string name, string value) + { + Name = name; + Value = value; + } + + public static DataStorageKeyVersion DskVersion { get; } = + new(new DataStorageKeyType(1u, "Attribute"), 1); + + public string Name { get; } + + public string Value { get; } + + public static AttributeValueDskV1 Create(AttributeValue attribute) => + new(attribute.Code.Value, ToInvariantString(attribute.UntypedValue)); + + public static AttributeValueDskV1 Create(AttributeCode code, object value) => + new(code.Value, ToInvariantString(value)); + + private static string ToInvariantString(object value) => + value switch + { + IFormattable formattable => formattable.ToString(null, CultureInfo.InvariantCulture), + _ => value.ToString()! + }; +} diff --git a/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeValueDso.cs b/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeValueDso.cs new file mode 100644 index 000000000..9691d6582 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/Internal/Storage/AttributeValueDso.cs @@ -0,0 +1,9 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue.Internal.Storage; + +public static class AttributeValueDso +{ + public sealed record V1(string Name, object? Value); +} diff --git a/storage/src/Storage/EntityAttributeValue/Internal/Storage/EnumValueDso.cs b/storage/src/Storage/EntityAttributeValue/Internal/Storage/EnumValueDso.cs new file mode 100644 index 000000000..ec08404ee --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/Internal/Storage/EnumValueDso.cs @@ -0,0 +1,6 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue.Internal.Storage; + +public sealed record EnumValueDso(string Key, string DisplayName); diff --git a/storage/src/Storage/EntityAttributeValue/ListAttributeType.cs b/storage/src/Storage/EntityAttributeValue/ListAttributeType.cs new file mode 100644 index 000000000..4497340c8 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/ListAttributeType.cs @@ -0,0 +1,28 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +/// +/// Represents a list attribute type with a single element type. +/// Lists cannot be nested inside other lists. +/// +public sealed record ListAttributeType : AttributeType +{ + public ListAttributeType(AttributeType ElementType) + { + ArgumentNullException.ThrowIfNull(ElementType); + + this.ElementType = ElementType; + + // Validate no list-in-list at construction time + ValidateNesting(); + } + + public AttributeType ElementType { get; } + + public bool Equals(ListAttributeType? other) => + other is not null && ElementType.Equals(other.ElementType); + + public override int GetHashCode() => HashCode.Combine(ElementType); +} diff --git a/storage/src/Storage/EntityAttributeValue/ScalarAttributeType.cs b/storage/src/Storage/EntityAttributeValue/ScalarAttributeType.cs new file mode 100644 index 000000000..dc278c4bb --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/ScalarAttributeType.cs @@ -0,0 +1,22 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +/// +/// Represents a scalar (primitive) attribute type. +/// +public sealed record ScalarAttributeType : AttributeType +{ + public ScalarAttributeType(ScalarDataType DataType) + { + if (!Enum.IsDefined(DataType)) + { + throw new ArgumentException($"Invalid ScalarDataType value: {DataType}.", nameof(DataType)); + } + + this.DataType = DataType; + } + + public ScalarDataType DataType { get; init; } +} diff --git a/storage/src/Storage/EntityAttributeValue/ScalarDataType.cs b/storage/src/Storage/EntityAttributeValue/ScalarDataType.cs new file mode 100644 index 000000000..d13411e75 --- /dev/null +++ b/storage/src/Storage/EntityAttributeValue/ScalarDataType.cs @@ -0,0 +1,18 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +/// +/// Represents a schema attribute data type. +/// +#pragma warning disable CA1720 // Identifiers should not contain type names +public enum ScalarDataType +{ + Boolean, + Date, + DateTime, + Decimal, + Integer, + String, +} diff --git a/storage/src/Storage/IDatabaseSchema.cs b/storage/src/Storage/IDatabaseSchema.cs new file mode 100644 index 000000000..f623ce59f --- /dev/null +++ b/storage/src/Storage/IDatabaseSchema.cs @@ -0,0 +1,38 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; + +namespace Duende.Storage; + +public interface IDatabaseSchema +{ + /// + /// Checks the schema version of the database. + /// + /// The cancellation token. + Task CheckVersionAsync(Ct ct); + + /// + /// Migrates the database schema to the current version. Creates the schema if it does not exist, + /// upgrades it if behind, and is a no-op if already current. Calls + /// after migration and throws if verification finds errors. + /// + /// The cancellation token. + Task MigrateAsync(Ct ct); + + /// + /// Verifies that the actual database schema matches the expected structure. + /// Returns a list of discrepancies (missing tables, columns, wrong types, missing indexes, etc.). + /// + /// The cancellation token. + Task VerifySchemaAsync(Ct ct); + + /// + /// Builds a SQL migration script that brings the database from to the current version. + /// Pass to generate the full script for a fresh database. + /// Each migration step is gated on the schema version number, not object existence. + /// + /// The version to migrate from. + string BuildMigrationScript(DatabaseSchemaVersion fromVersion); +} diff --git a/storage/src/Storage/IStorageBuilder.cs b/storage/src/Storage/IStorageBuilder.cs new file mode 100644 index 000000000..788cca39b --- /dev/null +++ b/storage/src/Storage/IStorageBuilder.cs @@ -0,0 +1,11 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage; + +public interface IStorageBuilder +{ + public IServiceCollection Services { get; } +} diff --git a/storage/src/Storage/Internal/Builder/DataStorageTypeRegistry.cs b/storage/src/Storage/Internal/Builder/DataStorageTypeRegistry.cs new file mode 100644 index 000000000..0e74d1969 --- /dev/null +++ b/storage/src/Storage/Internal/Builder/DataStorageTypeRegistry.cs @@ -0,0 +1,23 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Builder; + +internal sealed class DataStorageTypeRegistry +{ + private readonly Dictionary _dsoRegistrations; + + public DataStorageTypeRegistry(IEnumerable dsoRegistrations) + => _dsoRegistrations = dsoRegistrations.ToDictionary(r => r.DsoVersion, r => r.DsoType); + + /// + /// Gets the registered type for the specified DSO version. + /// + /// The DSO version to look up. + /// The registered type. + /// Thrown when the DSO type is not registered. + public Type Get(DataStorageObjectVersion dsoVersion) => + !_dsoRegistrations.TryGetValue(dsoVersion, out var registeredType) + ? throw new InvalidOperationException($"DsoType {dsoVersion.EntityType.Name} with version {dsoVersion.SchemaVersion} is not registered.") + : registeredType; +} diff --git a/storage/src/Storage/Internal/Builder/DsoRegistration.cs b/storage/src/Storage/Internal/Builder/DsoRegistration.cs new file mode 100644 index 000000000..25c59811e --- /dev/null +++ b/storage/src/Storage/Internal/Builder/DsoRegistration.cs @@ -0,0 +1,10 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Builder; + +public sealed class DsoRegistration(Type dsoType, DataStorageObjectVersion dsoVersion) +{ + public Type DsoType { get; } = dsoType; + public DataStorageObjectVersion DsoVersion { get; } = dsoVersion; +} diff --git a/storage/src/Storage/Internal/Builder/DsoRegistrationServiceCollectionExtensions.cs b/storage/src/Storage/Internal/Builder/DsoRegistrationServiceCollectionExtensions.cs new file mode 100644 index 000000000..94416a125 --- /dev/null +++ b/storage/src/Storage/Internal/Builder/DsoRegistrationServiceCollectionExtensions.cs @@ -0,0 +1,21 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Microsoft.Extensions.DependencyInjection; + +#pragma warning disable IDE0130 +namespace Duende.Storage.Internal.Builder; +#pragma warning restore IDE0130 + +public static class DsoRegistrationServiceCollectionExtensions +{ + extension(IServiceCollection services) + { + public void AddDsoRegistration() where TDso : IDataStorageObject + { + var dsoRegistration = new DsoRegistration(typeof(TDso), TDso.DsoVersion); + var dsoRegistrationServiceDescriptor = new ServiceDescriptor(typeof(DsoRegistration), dsoRegistration); + services.Add(dsoRegistrationServiceDescriptor); + } + } +} diff --git a/storage/src/Storage/Internal/Builder/GetRegisteredTypeForDso.cs b/storage/src/Storage/Internal/Builder/GetRegisteredTypeForDso.cs new file mode 100644 index 000000000..f788b9971 --- /dev/null +++ b/storage/src/Storage/Internal/Builder/GetRegisteredTypeForDso.cs @@ -0,0 +1,6 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Builder; + +public delegate Type GetRegisteredTypeForDso(DataStorageObjectVersion version); diff --git a/storage/src/Storage/Internal/CheckSchemaVersionResult.cs b/storage/src/Storage/Internal/CheckSchemaVersionResult.cs new file mode 100644 index 000000000..d8963958c --- /dev/null +++ b/storage/src/Storage/Internal/CheckSchemaVersionResult.cs @@ -0,0 +1,19 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +public sealed class CheckSchemaVersionResult +{ + internal CheckSchemaVersionResult(uint currentVersion, uint requiredVersion) + { + CurrentVersion = currentVersion; + RequiredVersion = requiredVersion; + } + + public uint CurrentVersion { get; } + + public uint RequiredVersion { get; } + + public bool IsCompatible => CurrentVersion == RequiredVersion; +} diff --git a/storage/src/Storage/Internal/DataStorageKey.cs b/storage/src/Storage/Internal/DataStorageKey.cs new file mode 100644 index 000000000..a77812970 --- /dev/null +++ b/storage/src/Storage/Internal/DataStorageKey.cs @@ -0,0 +1,35 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Text.Json; + +namespace Duende.Storage.Internal; + +public sealed class DataStorageKey +{ + private DataStorageKey(DataStorageKeyVersion version, Guid value, string? keyJsonValue) + { + DskVersion = version; + Value = value; + KeyJsonValue = keyJsonValue; + } + + public DataStorageKeyVersion DskVersion { get; private set; } + + public Guid Value { get; private set; } + + public string? KeyJsonValue { get; private set; } + + public static DataStorageKey Create(T dsk) where T : IDataStorageKey + { + if (dsk is IGuidDataStorageKey guidDsk) + { + return new DataStorageKey(T.DskVersion, guidDsk.Value, null); + } + + var json = JsonSerializer.Serialize(dsk); + var guid = DeterministicGuidGenerator.Create(json); + + return new DataStorageKey(T.DskVersion, guid, json); + } +} diff --git a/storage/src/Storage/Internal/DataStorageKeyType.cs b/storage/src/Storage/Internal/DataStorageKeyType.cs new file mode 100644 index 000000000..9080c57b2 --- /dev/null +++ b/storage/src/Storage/Internal/DataStorageKeyType.cs @@ -0,0 +1,24 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Globalization; + +namespace Duende.Storage.Internal; + +/// +/// The type of DSK that's being stored. +/// +/// A number representation for the DSK type. +/// The name of the DSK type. This name is only used for display purposes and should never change. +public readonly record struct DataStorageKeyType(uint Id, string Name) +{ + [Obsolete("Don't use this constructor")] + public DataStorageKeyType() : this(0!, null!) => throw new InvalidOperationException("Cannot instantiate DSKType without parameters"); + + public static DataStorageKeyType BuildFrom(Enum @enum) => + new((uint)Convert.ToInt32(@enum, CultureInfo.InvariantCulture), @enum.ToString()); + + public static DataStorageKeyType FromEnum(Enum value) => BuildFrom(value); + + public static implicit operator DataStorageKeyType(Enum value) => BuildFrom(value); +} diff --git a/storage/src/Storage/Internal/DataStorageKeyVersion.cs b/storage/src/Storage/Internal/DataStorageKeyVersion.cs new file mode 100644 index 000000000..4d992b4e7 --- /dev/null +++ b/storage/src/Storage/Internal/DataStorageKeyVersion.cs @@ -0,0 +1,10 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +public sealed record DataStorageKeyVersion(DataStorageKeyType KeyType, uint SchemaVersion) +{ + public override string ToString() => $"{KeyType.Name}({KeyType.Id}) v{SchemaVersion}"; + +} diff --git a/storage/src/Storage/Internal/DataStorageObjectVersion.cs b/storage/src/Storage/Internal/DataStorageObjectVersion.cs new file mode 100644 index 000000000..2d60e7ce2 --- /dev/null +++ b/storage/src/Storage/Internal/DataStorageObjectVersion.cs @@ -0,0 +1,12 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +/// +/// Represents a versioned DSO type. +/// +public sealed record DataStorageObjectVersion(EntityType EntityType, uint SchemaVersion) +{ + public override string ToString() => $"{EntityType.Name}({EntityType.Id}) v{SchemaVersion}"; +} diff --git a/storage/src/Storage/Internal/DeterministicGuidGenerator.cs b/storage/src/Storage/Internal/DeterministicGuidGenerator.cs new file mode 100644 index 000000000..498d64262 --- /dev/null +++ b/storage/src/Storage/Internal/DeterministicGuidGenerator.cs @@ -0,0 +1,31 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Security.Cryptography; +using System.Text; + +namespace Duende.Storage.Internal; + +public static class DeterministicGuidGenerator +{ + public static Guid Create(string name) + { + if (string.IsNullOrEmpty(name)) + { + throw new ArgumentException("Value cannot be null or empty.", nameof(name)); + } + //use MD5 hash to get a 16-byte hash of the string: + + var bytes = Encoding.Default.GetBytes(name); + +#pragma warning disable CA5351 // MD5 is used to produce a deterministic 128-bit hash for stable GUID generation from names, not for cryptographic security + var hashBytes = MD5.HashData(bytes); +#pragma warning restore CA5351 + + //generate a guid from the hash: + + var hashGuid = new Guid(hashBytes); + + return hashGuid; + } +} diff --git a/storage/src/Storage/Internal/EntityType.cs b/storage/src/Storage/Internal/EntityType.cs new file mode 100644 index 000000000..4444639f0 --- /dev/null +++ b/storage/src/Storage/Internal/EntityType.cs @@ -0,0 +1,16 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +/// +/// The type of document that's being stored. +/// Each library defines its own entity types as static fields on the containing DSO class. +/// +/// A number representation for the DSO type. Must be unique across all entity types. +/// The name of the DSO type. Used for analytics and display purposes. Should never change once entities exist. +public readonly record struct EntityType(uint Id, string Name) +{ + [Obsolete("Don't use this constructor")] + public EntityType() : this(0, null!) => throw new InvalidOperationException("Cannot instantiate EntityType without parameters"); +} diff --git a/storage/src/Storage/Internal/Expiration.cs b/storage/src/Storage/Internal/Expiration.cs new file mode 100644 index 000000000..4ad941f8d --- /dev/null +++ b/storage/src/Storage/Internal/Expiration.cs @@ -0,0 +1,99 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +/// +/// Represents an expiration policy for a store entity. +/// Use for a fixed point in time, +/// for a duration from now, +/// or to explicitly indicate no expiration. +/// +public abstract record Expiration +{ + // Prevent external subclassing. + private Expiration() { } + + /// + /// Resolves this expiration to an absolute (UTC), + /// or null if the entity should never expire. + /// + /// The time provider used to determine the current time for relative expirations. + /// The absolute expiration time, or null if the entity should never expire. + public abstract DateTimeOffset? Resolve(TimeProvider timeProvider); + + /// + /// Creates an expiration at a specific absolute point in time. + /// + /// The absolute expiration time. Must have of (UTC). + /// An instance. + public static Expiration AtAbsolute(DateTimeOffset expiresAt) => new AbsoluteExpiration(expiresAt); + + /// + /// Creates an expiration relative to the current time. + /// + /// The duration from now until expiration. Must be strictly positive. + /// A instance. + public static Expiration InRelative(TimeSpan lifetime) => new RelativeExpiration(lifetime); + + /// + /// A sentinel value indicating the entity should never expire. + /// On Create, this means the entity lives forever. + /// On Update, this explicitly clears any existing expiration. + /// + public static readonly Expiration NoExpiration = new NeverExpiration(); + + /// + /// An expiration at a fixed absolute point in time (UTC). + /// + internal sealed record AbsoluteExpiration : Expiration + { + public DateTimeOffset ExpiresAt { get; } + + public AbsoluteExpiration(DateTimeOffset expiresAt) + { + if (expiresAt.Offset != TimeSpan.Zero) + { + throw new ArgumentException( + "Expiration must be in UTC (Offset must be TimeSpan.Zero).", + nameof(expiresAt)); + } + + ExpiresAt = expiresAt; + } + + public override DateTimeOffset? Resolve(TimeProvider timeProvider) => ExpiresAt; + } + + /// + /// An expiration relative to the current time. + /// + internal sealed record RelativeExpiration : Expiration + { + public TimeSpan Lifetime { get; } + + public RelativeExpiration(TimeSpan lifetime) + { + if (lifetime <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException( + nameof(lifetime), + lifetime, + "Lifetime must be strictly positive."); + } + + Lifetime = lifetime; + } + + public override DateTimeOffset? Resolve(TimeProvider timeProvider) => + timeProvider.GetUtcNow() + Lifetime; + } + + /// + /// Sentinel indicating no expiration. Resolves to null. + /// + internal sealed record NeverExpiration : Expiration + { + public override DateTimeOffset? Resolve(TimeProvider timeProvider) => null; + } +} diff --git a/storage/src/Storage/Internal/FieldType.cs b/storage/src/Storage/Internal/FieldType.cs new file mode 100644 index 000000000..2b8b7e0dc --- /dev/null +++ b/storage/src/Storage/Internal/FieldType.cs @@ -0,0 +1,40 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +/// +/// Specifies the type of a field, corresponding to the typed columns in the search_values table. +/// This ensures QueryFields reads from the correct column instead of checking the first non-null value. +/// +public enum FieldType +{ + /// + /// String field, stored in the string_value column. + /// +#pragma warning disable CA1720 // Identifier 'String' contains type name + String, +#pragma warning restore CA1720 + + /// + /// Number field, stored in the number_value column. + /// + Number, + + /// + /// DateTime field, stored in the datetime_value column. + /// + DateTime, + + /// + /// Boolean field, stored in the boolean_value column. + /// + Boolean, + + /// + /// Guid field, stored in the guid_value column. + /// +#pragma warning disable CA1720 // Identifier 'Guid' contains type name + Guid +#pragma warning restore CA1720 +} diff --git a/storage/src/Storage/Internal/Filtering/ComparisonOperator.cs b/storage/src/Storage/Internal/Filtering/ComparisonOperator.cs new file mode 100644 index 000000000..e71c11f20 --- /dev/null +++ b/storage/src/Storage/Internal/Filtering/ComparisonOperator.cs @@ -0,0 +1,18 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Filtering; + +public enum ComparisonOperator +{ + Equal, + NotEqual, + Contains, + StartsWith, + EndsWith, + GreaterThan, + GreaterThanOrEqual, + LessThan, + LessThanOrEqual, + Present +} diff --git a/storage/src/Storage/Internal/Filtering/Expressions/AttributePathExpression.cs b/storage/src/Storage/Internal/Filtering/Expressions/AttributePathExpression.cs new file mode 100644 index 000000000..1f2ef3d0a --- /dev/null +++ b/storage/src/Storage/Internal/Filtering/Expressions/AttributePathExpression.cs @@ -0,0 +1,11 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Filtering.Expressions; + +public sealed class AttributePathExpression(string path) : FilterExpression +{ + public string Path { get; } = path ?? throw new ArgumentNullException(nameof(path)); + + public override string ToString() => Path; +} diff --git a/storage/src/Storage/Internal/Filtering/Expressions/ComparisonExpression.cs b/storage/src/Storage/Internal/Filtering/Expressions/ComparisonExpression.cs new file mode 100644 index 000000000..391ce1fba --- /dev/null +++ b/storage/src/Storage/Internal/Filtering/Expressions/ComparisonExpression.cs @@ -0,0 +1,16 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Filtering.Expressions; + +public sealed class ComparisonExpression(AttributePathExpression attributePath, ComparisonOperator op, object? value) + : FilterExpression +{ + public AttributePathExpression AttributePath { get; } = attributePath ?? throw new ArgumentNullException(nameof(attributePath)); + + public ComparisonOperator Operator { get; } = op; + + public object? Value { get; } = value; + + public override string ToString() => $"{AttributePath} {Operator.ToFilterString()} {Value}"; +} diff --git a/storage/src/Storage/Internal/Filtering/Expressions/ComplexAttributeExpression.cs b/storage/src/Storage/Internal/Filtering/Expressions/ComplexAttributeExpression.cs new file mode 100644 index 000000000..a4ef696ff --- /dev/null +++ b/storage/src/Storage/Internal/Filtering/Expressions/ComplexAttributeExpression.cs @@ -0,0 +1,16 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Filtering.Expressions; + +public sealed class ComplexAttributeExpression(AttributePathExpression attributePath, FilterExpression filter) + : FilterExpression +{ + public AttributePathExpression AttributePath { get; } + = attributePath ?? throw new ArgumentNullException(nameof(attributePath)); + + public FilterExpression Filter { get; } + = filter ?? throw new ArgumentNullException(nameof(filter)); + + public override string ToString() => $"{AttributePath}[{Filter}]"; +} diff --git a/storage/src/Storage/Internal/Filtering/Expressions/FilterExpression.cs b/storage/src/Storage/Internal/Filtering/Expressions/FilterExpression.cs new file mode 100644 index 000000000..5b484f37c --- /dev/null +++ b/storage/src/Storage/Internal/Filtering/Expressions/FilterExpression.cs @@ -0,0 +1,8 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Filtering.Expressions; + +public abstract class FilterExpression +{ +} diff --git a/storage/src/Storage/Internal/Filtering/Expressions/LogicalExpression.cs b/storage/src/Storage/Internal/Filtering/Expressions/LogicalExpression.cs new file mode 100644 index 000000000..0cf0ca36d --- /dev/null +++ b/storage/src/Storage/Internal/Filtering/Expressions/LogicalExpression.cs @@ -0,0 +1,38 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Filtering.Expressions; + +public sealed class LogicalExpression : FilterExpression +{ + public LogicalOperator Operator { get; } + + public FilterExpression Left { get; } + + public FilterExpression? Right { get; } + + public LogicalExpression(LogicalOperator op, FilterExpression left, FilterExpression? right = null) + { + Operator = op; + Left = left ?? throw new ArgumentNullException(nameof(left)); + Right = right; + + if (op != LogicalOperator.Not && right == null) + { + throw new ArgumentException($"{op} operator requires a right operand", nameof(right)); + } + if (op == LogicalOperator.Not && right != null) + { + throw new ArgumentException("NOT operator should not have a right operand", nameof(right)); + } + } + + public override string ToString() => + Operator switch + { + LogicalOperator.And => $"({Left} AND {Right})", + LogicalOperator.Or => $"({Left} OR {Right})", + LogicalOperator.Not => $"NOT ({Left})", + _ => $"Unknown operator: {Operator}" + }; +} diff --git a/storage/src/Storage/Internal/Filtering/FilterExpressionParser.cs b/storage/src/Storage/Internal/Filtering/FilterExpressionParser.cs new file mode 100644 index 000000000..56f0a1d92 --- /dev/null +++ b/storage/src/Storage/Internal/Filtering/FilterExpressionParser.cs @@ -0,0 +1,370 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Filtering.Expressions; +using Duende.Storage.Querying; + +namespace Duende.Storage.Internal.Filtering; + +public static class FilterExpressionParser +{ + public static FilterExpression Parse(string filter) + { + if (string.IsNullOrWhiteSpace(filter)) + { + throw new ArgumentException("Filter cannot be null or whitespace.", nameof(filter)); + } + + try + { + var tokens = FilterLexer.Tokenize(filter); + var parser = new Parser(tokens, filter); + var expression = parser.ParseFilter(); + + if (!parser.IsAtEnd) + { + var current = parser.Peek(); + throw new FilterParseException( + $"Unexpected token '{current.Value}' at position {current.Position}; expected end of input"); + } + + return expression; + } + catch (Exception ex) when (ex is not FilterParseException and not ArgumentException) + { + throw new FilterParseException($"Invalid filter syntax: {ex.Message}", ex); + } + } + + public static bool TryParse(string filter, out FilterExpression? expression) + { + try + { + expression = Parse(filter); + return true; + } +#pragma warning disable CA1031 // TryParse is designed to catch all exceptions + catch (Exception) +#pragma warning restore CA1031 + { + expression = null; + return false; + } + } + + private sealed class Parser(List tokens, string input) + { + private const int MaxDepth = 100; + private int _position; + private int _depth; + + internal bool IsAtEnd => _position >= tokens.Count; + + internal LexToken Peek() + { + if (IsAtEnd) + { + return new LexToken(FilterToken.None, string.Empty, input.Length); + } + return tokens[_position]; + } + + private LexToken Advance() + { + if (IsAtEnd) + { + throw new FilterParseException("Unexpected end of input"); + } + return tokens[_position++]; + } + + private LexToken Expect(FilterToken type) + { + var token = Peek(); + if (token.Type != type) + { + var expected = type.ToString(); + if (IsAtEnd) + { + throw new FilterParseException( + $"Unexpected end of input; expected {expected}"); + } + throw new FilterParseException( + $"Unexpected token '{token.Value}' at position {token.Position}; expected {expected}"); + } + return Advance(); + } + + private bool Check(FilterToken type) => !IsAtEnd && Peek().Type == type; + + internal FilterExpression ParseFilter() => ParseOrExpression(); + + private FilterExpression ParseOrExpression() + { + var left = ParseAndExpression(); + while (Check(FilterToken.Or)) + { + _ = Advance(); + var right = ParseAndExpression(); + left = new LogicalExpression(LogicalOperator.Or, left, right); + } + return left; + } + + private FilterExpression ParseAndExpression() + { + var left = ParseUnaryExpression(); + while (Check(FilterToken.And)) + { + _ = Advance(); + var right = ParseUnaryExpression(); + left = new LogicalExpression(LogicalOperator.And, left, right); + } + return left; + } + + private FilterExpression ParseUnaryExpression() + { + if (Check(FilterToken.Not)) + { + _ = Advance(); + var expr = ParsePrimaryExpression(); + return new LogicalExpression(LogicalOperator.Not, expr); + } + return ParsePrimaryExpression(); + } + + private FilterExpression ParsePrimaryExpression() + { + if (Check(FilterToken.LParen)) + { + return ParseGroupedExpression(); + } + return ParseAttributeExpression(); + } + + private FilterExpression ParseGroupedExpression() + { + _ = Expect(FilterToken.LParen); + if (++_depth > MaxDepth) + { + throw new FilterParseException( + $"Filter expression exceeds maximum nesting depth of {MaxDepth}"); + } + try + { + var expr = ParseFilter(); + _ = Expect(FilterToken.RParen); + return expr; + } + finally + { + _depth--; + } + } + + private FilterExpression ParseAttributeExpression() + { + var attrPath = ParseAttributePath(); + + // Complex attribute filter: emails[type eq "work"] + if (Check(FilterToken.LBracket)) + { + return ParseComplexFilter(attrPath); + } + + // Present operator: title pr + if (Check(FilterToken.Pr)) + { + _ = Advance(); + return new ComparisonExpression(attrPath, ComparisonOperator.Present, null); + } + + // Comparison: attrPath op value + if (IsComparisonOperator(Peek().Type)) + { + var op = ParseComparisonOperator(); + var value = ParseCompValue(); + return new ComparisonExpression(attrPath, op, value); + } + + var current = Peek(); + if (IsAtEnd) + { + throw new FilterParseException("Unexpected end of input; expected operator"); + } + throw new FilterParseException( + $"Unexpected token '{current.Value}' at position {current.Position}; expected operator"); + } + + private ComplexAttributeExpression ParseComplexFilter(AttributePathExpression attrPath) + { + _ = Expect(FilterToken.LBracket); + if (++_depth > MaxDepth) + { + throw new FilterParseException( + $"Filter expression exceeds maximum nesting depth of {MaxDepth}"); + } + try + { + var filter = ParseFilter(); + _ = Expect(FilterToken.RBracket); + return new ComplexAttributeExpression(attrPath, filter); + } + finally + { + _depth--; + } + } + + private AttributePathExpression ParseAttributePath() + { + var first = Expect(FilterToken.Identifier).Value; + + while (Check(FilterToken.Dot) || Check(FilterToken.Colon)) + { + if (Check(FilterToken.Dot)) + { + // Standard sub-attribute: attrName.subAttr + _ = Advance(); + first += "." + Expect(FilterToken.Identifier).Value; + } + else // Colon + { + _ = Advance(); + if (Check(FilterToken.Identifier)) + { + first += ":" + Advance().Value; + } + else if (Check(FilterToken.Number)) + { + // Version segment in URI, e.g. "2" or "2.0" in "core:2.0:User" + first += ":" + Advance().Value; + } + else + { + var current = Peek(); + throw new FilterParseException( + $"Unexpected token '{current.Value}' at position {current.Position}; expected identifier or number in attribute path"); + } + } + } + + return new AttributePathExpression(first); + } + + private static bool IsComparisonOperator(FilterToken type) => + type is FilterToken.Eq or FilterToken.Ne or FilterToken.Co or FilterToken.Sw + or FilterToken.Ew or FilterToken.Gt or FilterToken.Ge or FilterToken.Lt + or FilterToken.Le; + + private ComparisonOperator ParseComparisonOperator() + { + var token = Advance(); + return token.Type switch + { + FilterToken.Eq => ComparisonOperator.Equal, + FilterToken.Ne => ComparisonOperator.NotEqual, + FilterToken.Co => ComparisonOperator.Contains, + FilterToken.Sw => ComparisonOperator.StartsWith, + FilterToken.Ew => ComparisonOperator.EndsWith, + FilterToken.Gt => ComparisonOperator.GreaterThan, + FilterToken.Ge => ComparisonOperator.GreaterThanOrEqual, + FilterToken.Lt => ComparisonOperator.LessThan, + FilterToken.Le => ComparisonOperator.LessThanOrEqual, + _ => throw new FilterParseException( + $"Unexpected token '{token.Value}' at position {token.Position}; expected comparison operator") + }; + } + + private object? ParseCompValue() + { + var token = Peek(); + switch (token.Type) + { + case FilterToken.StringLiteral: + _ = Advance(); + return UnescapeString(token.Value); + + case FilterToken.Number: + _ = Advance(); + return ParseNumber(token.Value, token.Position); + + case FilterToken.True: + _ = Advance(); + return true; + + case FilterToken.False: + _ = Advance(); + return false; + + case FilterToken.Null: + _ = Advance(); + return null; + + default: + if (IsAtEnd) + { + throw new FilterParseException("Unexpected end of input; expected value"); + } + throw new FilterParseException( + $"Unexpected token '{token.Value}' at position {token.Position}; expected value"); + } + } + + private static string UnescapeString(string raw) + { + // Strip surrounding quotes + var inner = raw[1..^1]; + + if (!inner.Contains('\\', StringComparison.Ordinal)) + { + return inner; + } + + var result = new System.Text.StringBuilder(inner.Length); + for (var i = 0; i < inner.Length; i++) + { + if (inner[i] == '\\' && i + 1 < inner.Length) + { + i++; + _ = result.Append(inner[i]); + } + else + { + _ = result.Append(inner[i]); + } + } + return result.ToString(); + } + + private static object ParseNumber(string text, int position) + { + try + { + if (text.Contains('.', StringComparison.Ordinal)) + { + return double.Parse(text, System.Globalization.CultureInfo.InvariantCulture); + } + + if (int.TryParse(text, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var i)) + { + return i; + } + + if (long.TryParse(text, System.Globalization.NumberStyles.Integer, System.Globalization.CultureInfo.InvariantCulture, out var l)) + { + return l; + } + + throw new FormatException($"Cannot parse number: {text}"); + } + catch (Exception ex) when (ex is FormatException or OverflowException) + { + throw new FilterParseException( + $"Invalid number '{text}' at position {position}", + ex); + } + } + } +} diff --git a/storage/src/Storage/Internal/Filtering/FilterLexer.cs b/storage/src/Storage/Internal/Filtering/FilterLexer.cs new file mode 100644 index 000000000..6b88af557 --- /dev/null +++ b/storage/src/Storage/Internal/Filtering/FilterLexer.cs @@ -0,0 +1,156 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Querying; + +namespace Duende.Storage.Internal.Filtering; + +internal static class FilterLexer +{ + public static List Tokenize(string input) + { + var tokens = new List(); + var position = 0; + + while (position < input.Length) + { + var ch = input[position]; + + // Skip whitespace + if (char.IsWhiteSpace(ch)) + { + position++; + continue; + } + + // Single-character punctuation + switch (ch) + { + case '(': + tokens.Add(new LexToken(FilterToken.LParen, "(", position)); + position++; + continue; + case ')': + tokens.Add(new LexToken(FilterToken.RParen, ")", position)); + position++; + continue; + case '[': + tokens.Add(new LexToken(FilterToken.LBracket, "[", position)); + position++; + continue; + case ']': + tokens.Add(new LexToken(FilterToken.RBracket, "]", position)); + position++; + continue; + case '.': + tokens.Add(new LexToken(FilterToken.Dot, ".", position)); + position++; + continue; + case ':': + tokens.Add(new LexToken(FilterToken.Colon, ":", position)); + position++; + continue; + } + + // String literal + if (ch == '"') + { + var start = position; + position++; // skip opening quote + while (position < input.Length) + { + if (input[position] == '\\') + { + position += 2; // skip escape sequence + continue; + } + if (input[position] == '"') + { + break; + } + position++; + } + + if (position >= input.Length) + { + throw new FilterParseException( + $"Unterminated string literal at position {start}"); + } + + position++; // skip closing quote + var value = input[start..position]; + tokens.Add(new LexToken(FilterToken.StringLiteral, value, start)); + continue; + } + + // Number (digit or - followed by digit) + if (char.IsDigit(ch) || (ch == '-' && position + 1 < input.Length && char.IsDigit(input[position + 1]))) + { + var start = position; + if (ch == '-') + { + position++; + } + while (position < input.Length && char.IsDigit(input[position])) + { + position++; + } + // Optional decimal part + if (position < input.Length && input[position] == '.' && position + 1 < input.Length && char.IsDigit(input[position + 1])) + { + position++; // skip dot + while (position < input.Length && char.IsDigit(input[position])) + { + position++; + } + } + var value = input[start..position]; + tokens.Add(new LexToken(FilterToken.Number, value, start)); + continue; + } + + // Identifier or keyword + if (char.IsLetter(ch) || ch == '_') + { + var start = position; + position++; + while (position < input.Length && (char.IsLetterOrDigit(input[position]) || input[position] == '_' || input[position] == '-')) + { + position++; + } + var word = input[start..position]; + + // Check for keywords (case-insensitive) + var tokenType = word.ToUpperInvariant() switch + { + "AND" => FilterToken.And, + "OR" => FilterToken.Or, + "NOT" => FilterToken.Not, + "EQ" => FilterToken.Eq, + "NE" => FilterToken.Ne, + "CO" => FilterToken.Co, + "SW" => FilterToken.Sw, + "EW" => FilterToken.Ew, + "PR" => FilterToken.Pr, + "GT" => FilterToken.Gt, + "GE" => FilterToken.Ge, + "LT" => FilterToken.Lt, + "LE" => FilterToken.Le, + "TRUE" => FilterToken.True, + "FALSE" => FilterToken.False, + "NULL" => FilterToken.Null, + _ => FilterToken.Identifier + }; + + tokens.Add(new LexToken(tokenType, word, start)); + continue; + } + + // Unexpected character + throw new FilterParseException( + $"Unexpected character '{ch}' at position {position}"); + } + + return tokens; + } +} diff --git a/storage/src/Storage/Internal/Filtering/FilterToken.cs b/storage/src/Storage/Internal/Filtering/FilterToken.cs new file mode 100644 index 000000000..d4077c7b3 --- /dev/null +++ b/storage/src/Storage/Internal/Filtering/FilterToken.cs @@ -0,0 +1,34 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Filtering; + +internal enum FilterToken +{ + None, + Identifier, + StringLiteral, + Number, + True, + False, + Null, + Eq, + Ne, + Co, + Sw, + Ew, + Pr, + Gt, + Ge, + Lt, + Le, + And, + Or, + Not, + LParen, + RParen, + LBracket, + RBracket, + Dot, + Colon +} diff --git a/storage/src/Storage/Internal/Filtering/FilterTranslator.cs b/storage/src/Storage/Internal/Filtering/FilterTranslator.cs new file mode 100644 index 000000000..10e3e17b5 --- /dev/null +++ b/storage/src/Storage/Internal/Filtering/FilterTranslator.cs @@ -0,0 +1,222 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Filtering.Expressions; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Expressions; +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Filtering; + +public sealed class FilterTranslator +{ + private readonly IScimAttributeTypeResolver _resolver; + + public FilterTranslator(IScimAttributeTypeResolver resolver) => + _resolver = resolver ?? throw new ArgumentNullException(nameof(resolver)); + + public IQueryFilterExpression? Translate(string? filter) + { + if (string.IsNullOrWhiteSpace(filter)) + { + return null; + } + + var expression = FilterExpressionParser.Parse(filter); + return Translate(expression); + } + + public IQueryFilterExpression Translate(FilterExpression expression) + { + ArgumentNullException.ThrowIfNull(expression); + + return expression switch + { + ComparisonExpression comparison => TranslateComparison(comparison), + LogicalExpression logical => TranslateLogical(logical), + ComplexAttributeExpression complex => TranslateComplexAttribute(complex), + _ => throw new NotSupportedException($"Unsupported expression type: {expression.GetType().Name}") + }; + } + + private IQueryFilterExpression TranslateComparison(ComparisonExpression comparison) + { + var field = _resolver.ResolveField(comparison.AttributePath.Path); + + return comparison.Operator switch + { + ComparisonOperator.Equal => TranslateEqual(field, comparison.Value), + ComparisonOperator.NotEqual => comparison.Value is null + ? new PresentExpression(field) + : new NotExpression(TranslateEqual(field, comparison.Value)), + ComparisonOperator.Contains => TranslateStringOp(field, comparison.Value, + static (sf, v) => new ContainsExpression(sf, v)), + ComparisonOperator.StartsWith => TranslateStringOp(field, comparison.Value, + static (sf, v) => new StartsWithExpression(sf, v)), + ComparisonOperator.EndsWith => TranslateStringOp(field, comparison.Value, + static (sf, v) => new EndsWithExpression(sf, v)), + ComparisonOperator.GreaterThan => new GreaterThanExpression(field, ConvertValue(field, comparison.Value)), + ComparisonOperator.GreaterThanOrEqual => new GreaterOrEqualExpression(field, ConvertValue(field, comparison.Value)), + ComparisonOperator.LessThan => new LessThanExpression(field, ConvertValue(field, comparison.Value)), + ComparisonOperator.LessThanOrEqual => new LessOrEqualExpression(field, ConvertValue(field, comparison.Value)), + ComparisonOperator.Present => new PresentExpression(field), + _ => throw new NotSupportedException($"Unsupported comparison operator: {comparison.Operator}") + }; + } + + private static IQueryFilterExpression TranslateEqual(Field field, object? value) + { + // title eq null → not present + if (value is null) + { + return new NotExpression(new PresentExpression(field)); + } + + // For string array fields, "eq" means "any element equals the value" + if (field is StringArrayField arrayField && value is string strValue) + { + return new ArrayContainsExpression(arrayField, strValue.ToUpperInvariant()); + } + + // For multi-valued fields resolved from schema (not StringArrayField), equality + // works via the standard EqualExpression with IsMultiValued producing item_index >= 0 + return new EqualExpression(field, ConvertValue(field, value)); + } + + private static IQueryFilterExpression TranslateStringOp( + Field field, + object? value, + Func factory) + { + if (value is not string stringValue) + { + throw new InvalidOperationException( + $"String operator requires a string value, but got {value?.GetType().Name ?? "null"}"); + } + + // For string array fields, string operations (co, sw, ew) fall back to exact element + // matching via ArrayContainsExpression. True substring semantics on individual array + // elements are not yet supported by the indexing model. + if (field is StringArrayField arrayField) + { + return new ArrayContainsExpression(arrayField, stringValue.ToUpperInvariant()); + } + + if (field is not StringField stringField) + { + throw new InvalidOperationException( + $"String operator cannot be applied to field '{field.Path}' of type {field.Type}"); + } + + return factory(stringField, stringValue.ToUpperInvariant()); + } + + private IQueryFilterExpression TranslateLogical(LogicalExpression logical) => + logical.Operator switch + { + LogicalOperator.And => new AndExpression(Translate(logical.Left), Translate(logical.Right!)), + LogicalOperator.Or => new OrExpression(Translate(logical.Left), Translate(logical.Right!)), + LogicalOperator.Not => new NotExpression(Translate(logical.Left)), + _ => throw new NotSupportedException($"Unsupported logical operator: {logical.Operator}") + }; + + private ArrayFilterExpression TranslateComplexAttribute(ComplexAttributeExpression complex) + { + var prefixedFilter = PrefixAttributePaths(complex.Filter, complex.AttributePath.Path); + var scopedTranslator = new FilterTranslator(new ArrayElementResolver(_resolver, complex.AttributePath.Path)); + var innerFilter = scopedTranslator.Translate(prefixedFilter); + return new ArrayFilterExpression(complex.AttributePath.Path, innerFilter); + } + + private static FilterExpression PrefixAttributePaths(FilterExpression expression, string prefix) => + expression switch + { + ComparisonExpression comparison => new ComparisonExpression( + new AttributePathExpression($"{prefix}.{comparison.AttributePath.Path}"), + comparison.Operator, + comparison.Value), + LogicalExpression logical => logical.Operator == LogicalOperator.Not + ? new LogicalExpression(LogicalOperator.Not, PrefixAttributePaths(logical.Left, prefix)) + : new LogicalExpression(logical.Operator, + PrefixAttributePaths(logical.Left, prefix), + PrefixAttributePaths(logical.Right!, prefix)), + _ => expression + }; + + /// + /// Wraps a resolver to strip the array prefix from resolved field paths. + /// Inside an array filter, fields are per-element and the SQL builder + /// re-adds the array prefix when building the query. + /// + private sealed class ArrayElementResolver(IScimAttributeTypeResolver inner, string arrayPrefix) : IScimAttributeTypeResolver + { + public Field ResolveField(string attributePath) + { + var field = inner.ResolveField(attributePath); + var strippedPath = StripPrefix(field.Path); + return field switch + { + StringField => new StringField(strippedPath), + NumberField => new NumberField(strippedPath), + BooleanField => new BooleanField(strippedPath), + DateTimeField => new DateTimeField(strippedPath), + GuidField => new GuidField(strippedPath), + _ => field + }; + } + + private string StripPrefix(string path) + { + var prefix = arrayPrefix.ToUpperInvariant() + "."; + return path.StartsWith(prefix, StringComparison.OrdinalIgnoreCase) + ? path[prefix.Length..] + : path; + } + } + + private static object ConvertValue(Field field, object? value) + { + if (value is null) + { + throw new ArgumentNullException(nameof(value), "Cannot convert null value for comparison"); + } + + return field.Type switch + { + FieldType.String => (value is string s + ? s + : Convert.ToString(value, System.Globalization.CultureInfo.InvariantCulture)!).ToUpperInvariant(), + + FieldType.Number => value switch + { + decimal d => d, + int i => (decimal)i, + long l => (decimal)l, + double d => decimal.Parse(d.ToString(System.Globalization.CultureInfo.InvariantCulture), System.Globalization.CultureInfo.InvariantCulture), + float f => decimal.Parse(f.ToString(System.Globalization.CultureInfo.InvariantCulture), System.Globalization.CultureInfo.InvariantCulture), + string s => decimal.Parse(s, System.Globalization.CultureInfo.InvariantCulture), + _ => Convert.ToDecimal(value, System.Globalization.CultureInfo.InvariantCulture) + }, + + FieldType.Boolean => value switch + { + bool b => b, + string s => bool.Parse(s), + _ => Convert.ToBoolean(value, System.Globalization.CultureInfo.InvariantCulture) + }, + + FieldType.DateTime => value switch + { + DateTimeOffset dto => dto, + DateTime dt => new DateTimeOffset(dt, TimeSpan.Zero), + // Use AssumeUniversal so date-only strings (e.g. "2024-01-01") are interpreted as UTC, + // matching how DateOnly values are stored as DateTimeOffset at midnight UTC. + string s => DateTimeOffset.Parse(s, System.Globalization.CultureInfo.InvariantCulture, + System.Globalization.DateTimeStyles.AssumeUniversal), + _ => throw new InvalidOperationException($"Cannot convert {value.GetType().Name} to DateTimeOffset") + }, + + _ => value + }; + } +} diff --git a/storage/src/Storage/Internal/Filtering/LexToken.cs b/storage/src/Storage/Internal/Filtering/LexToken.cs new file mode 100644 index 000000000..0e28f1513 --- /dev/null +++ b/storage/src/Storage/Internal/Filtering/LexToken.cs @@ -0,0 +1,11 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Filtering; + +internal readonly record struct LexToken(FilterToken Type, string Value, int Position) +{ + [Obsolete("Don't use parameterless constructor")] + public LexToken() : this(default, string.Empty, 0) => + throw new InvalidOperationException("Don't use parameterless constructor"); +} diff --git a/storage/src/Storage/Internal/Filtering/LogicalOperator.cs b/storage/src/Storage/Internal/Filtering/LogicalOperator.cs new file mode 100644 index 000000000..53dd1ed8c --- /dev/null +++ b/storage/src/Storage/Internal/Filtering/LogicalOperator.cs @@ -0,0 +1,11 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Filtering; + +public enum LogicalOperator +{ + And, + Or, + Not +} diff --git a/storage/src/Storage/Internal/Filtering/OperatorExtensions.cs b/storage/src/Storage/Internal/Filtering/OperatorExtensions.cs new file mode 100644 index 000000000..5369fea4f --- /dev/null +++ b/storage/src/Storage/Internal/Filtering/OperatorExtensions.cs @@ -0,0 +1,32 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Filtering; + +internal static class OperatorExtensions +{ + public static string ToFilterString(this ComparisonOperator op) => + op switch + { + ComparisonOperator.Equal => "eq", + ComparisonOperator.NotEqual => "ne", + ComparisonOperator.Contains => "co", + ComparisonOperator.StartsWith => "sw", + ComparisonOperator.EndsWith => "ew", + ComparisonOperator.GreaterThan => "gt", + ComparisonOperator.GreaterThanOrEqual => "ge", + ComparisonOperator.LessThan => "lt", + ComparisonOperator.LessThanOrEqual => "le", + ComparisonOperator.Present => "pr", + _ => throw new ArgumentOutOfRangeException(nameof(op)) + }; + + public static string ToFilterString(this LogicalOperator op) => + op switch + { + LogicalOperator.And => "and", + LogicalOperator.Or => "or", + LogicalOperator.Not => "not", + _ => throw new ArgumentOutOfRangeException(nameof(op)) + }; +} diff --git a/storage/src/Storage/Internal/IDataStorageKey.cs b/storage/src/Storage/Internal/IDataStorageKey.cs new file mode 100644 index 000000000..183dde991 --- /dev/null +++ b/storage/src/Storage/Internal/IDataStorageKey.cs @@ -0,0 +1,16 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +/// +/// Interface for DSK (Data Store Key) types. +/// +public interface IDataStorageKey +{ + /// + /// The version of the DSK. DSK's (once released) must be immutable, so we need + /// to keep track of versions. + /// + static abstract DataStorageKeyVersion DskVersion { get; } +} diff --git a/storage/src/Storage/Internal/IDataStorageObject.cs b/storage/src/Storage/Internal/IDataStorageObject.cs new file mode 100644 index 000000000..6e2a949e2 --- /dev/null +++ b/storage/src/Storage/Internal/IDataStorageObject.cs @@ -0,0 +1,12 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +/// +/// Represents a Data Storage Object (DSO). +/// +public interface IDataStorageObject +{ + static abstract DataStorageObjectVersion DsoVersion { get; } +} diff --git a/storage/src/Storage/Internal/IGuidDataStorageKey.cs b/storage/src/Storage/Internal/IGuidDataStorageKey.cs new file mode 100644 index 000000000..7e312d1fd --- /dev/null +++ b/storage/src/Storage/Internal/IGuidDataStorageKey.cs @@ -0,0 +1,13 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +/// +/// Marks a dsk as only being a guid value. This means it won't be stored as serialized json +/// but only uses the guid value. IE: UserSubjectId, which is already a Guid. +/// +public interface IGuidDataStorageKey : IDataStorageKey +{ + Guid Value { get; } +} diff --git a/storage/src/Storage/Internal/IPooledStore.cs b/storage/src/Storage/Internal/IPooledStore.cs new file mode 100644 index 000000000..993fab58d --- /dev/null +++ b/storage/src/Storage/Internal/IPooledStore.cs @@ -0,0 +1,9 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +public interface IPooledStore : IDatabaseSchema +{ + IStore OpenPool(PoolId poolId); +} diff --git a/storage/src/Storage/Internal/IScimAttributeTypeResolver.cs b/storage/src/Storage/Internal/IScimAttributeTypeResolver.cs new file mode 100644 index 000000000..947474d58 --- /dev/null +++ b/storage/src/Storage/Internal/IScimAttributeTypeResolver.cs @@ -0,0 +1,21 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal; + +/// +/// Resolves SCIM attribute paths to Faro Field types. +/// Implementations define the schema-specific mapping for a resource type (User, Group, etc.). +/// +public interface IScimAttributeTypeResolver +{ + /// + /// Resolves a SCIM attribute path to its corresponding Faro Field. + /// + /// The SCIM attribute path (e.g., "userName", "name.familyName", "emails.value"). + /// The corresponding Faro Field instance. + /// Thrown when the attribute path is not recognized. + Field ResolveField(string attributePath); +} diff --git a/storage/src/Storage/Internal/IStore.cs b/storage/src/Storage/Internal/IStore.cs new file mode 100644 index 000000000..520bf6576 --- /dev/null +++ b/storage/src/Storage/Internal/IStore.cs @@ -0,0 +1,280 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Outbox; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Internal.Querying.SearchFields; +using Duende.Storage.Internal.Querying.Sorting; +using Duende.Storage.Pagination; +using Duende.Storage.Querying; +using OutboxEventId = Duende.Storage.Internal.Outbox.OutboxEventId; + +namespace Duende.Storage.Internal; + +public interface IStore +{ + internal void SetPoolId(PoolId poolId); + + /// + /// Creates a new entity in the store, writing outbox events atomically. + /// If the resolved expiration is already in the past, the entity is not stored (noop) and + /// is returned. + /// + /// The type of the DSO to create. + /// The unique identifier for the entity. + /// The DSO value to store. + /// The collection of keys for alternate lookups. + /// Optional search field values that can be used for querying. + /// The expiration policy for the entity. + /// Use for entities that should never expire. + /// Outbox events to INSERT atomically within the same transaction. + /// Pass [] when no events are needed. + /// Silently ignored when the outbox is not enabled. + /// The cancellation token. + /// The result of the create operation. + public Task CreateAsync( + UuidV7 id, + TDso value, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration expiration, + IReadOnlyList outboxEvents, + Ct ct) where TDso : IDataStorageObject; + + /// + /// Attempts to retrieve an entity from the store by its unique identifier. + /// + /// The type of entity to retrieve. + /// The unique identifier of the entity. + /// Cancellation token. + /// A indicating whether the entity was found and, if so, its value and version. + public Task TryReadAsync( + EntityType type, + UuidV7 id, + Ct ct); + + /// + /// Attempts to retrieve an entity from the store by an alternate key. + /// + /// The type of entity to retrieve. + /// The alternate key to look up. + /// Cancellation token. + /// A indicating whether the entity was found and, if so, its value and version. + public Task TryReadAsync( + EntityType type, + DataStorageKey key, + Ct ct); + + /// + /// Retrieves multiple entities from the store by their unique identifiers. + /// Only entities that exist in the store are included in the result; + /// IDs that do not match any stored entity are silently omitted. + /// + /// The type of entities to retrieve. + /// The unique identifiers of the entities to retrieve. Using a set ensures no duplicate IDs are requested. + /// The maximum number of IDs allowed in a single request. + /// An is thrown if the count of exceeds this value. + /// Cancellation token. + /// + /// A list of containing only the entities that were found. + /// The result list may contain fewer items than if some IDs + /// do not exist in the store. Returns an empty list if no matching entities are found + /// or if is empty. + /// + public Task> TryReadManyAsync( + EntityType entityType, + IReadOnlySet ids, + int maximum, + Ct ct); + + /// + /// Updates an existing entity in the store, writing outbox events atomically. + /// The expiration value is stored as provided. Expired records remain visible on reads + /// (TTL is best-effort) and are removed by the background purge job. + /// + /// The type of the DSO to update. + /// The unique identifier for the entity. + /// The new DSO value to store. + /// The expected version for optimistic concurrency control. + /// The collection of keys for alternate lookups. + /// Optional search field values that can be used for querying. + /// These values replace any existing search fields for this entity. + /// The expiration policy for the entity. + /// null means "don't change existing expiration". + /// explicitly clears any existing expiration. + /// or sets a new expiration. + /// Outbox events to INSERT atomically within the same transaction. + /// Pass [] when no events are needed. + /// Silently ignored when the outbox is not enabled. + /// The cancellation token. + /// The result of the update operation. + public Task UpdateAsync( + UuidV7 id, + TDso dso, + int expectedEntityVersion, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration? expiration, + IReadOnlyList outboxEvents, + Ct ct) where TDso : IDataStorageObject; + + /// + /// Deletes an entity from the store by its unique identifier, writing outbox events atomically. + /// + /// The type of entity to delete. + /// The unique identifier of the entity to delete. + /// Outbox events to INSERT atomically within the same transaction. + /// Pass [] when no events are needed. + /// Silently ignored when the outbox is not enabled. + /// Cancellation token. + /// The result of the delete operation. + public Task DeleteAsync(EntityType entityType, UuidV7 id, IReadOnlyList outboxEvents, Ct ct); + + /// + /// Deletes an entity from the store by an alternate key, writing outbox events atomically. + /// + /// The type of entity to delete. + /// The alternate key identifying the entity to delete. + /// Outbox events to INSERT atomically within the same transaction. + /// Pass [] when no events are needed. + /// Silently ignored when the outbox is not enabled. + /// Cancellation token. + /// The result of the delete operation. + public Task DeleteAsync(EntityType entityType, DataStorageKey key, IReadOnlyList outboxEvents, Ct ct); + + /// + /// Creates a link between two entities, writing outbox events atomically. + /// The link is unique per (LinkType, LeftId, RightId). + /// No referential integrity check — the entities do not need to exist. + /// + /// The link definition describing the relationship schema. + /// The ID of the left-side entity. + /// The ID of the right-side entity. + /// Outbox events to INSERT atomically within the same transaction. + /// Pass [] when no events are needed. + /// Silently ignored when the outbox is not enabled. + /// Cancellation token. + /// if created, if the exact link already exists. + Task LinkAsync(LinkDefinition definition, UuidV7 leftEntityId, UuidV7 rightEntityId, IReadOnlyList outboxEvents, Ct ct); + + /// + /// Removes a link between two entities, writing outbox events atomically. + /// Returns success even if the link did not exist (idempotent). + /// + /// The link definition describing the relationship schema. + /// The ID of the left-side entity. + /// The ID of the right-side entity. + /// Outbox events to INSERT atomically within the same transaction. + /// Pass [] when no events are needed. + /// Silently ignored when the outbox is not enabled. + /// Cancellation token. + /// Always returns . + Task UnlinkAsync(LinkDefinition definition, UuidV7 leftEntityId, UuidV7 rightEntityId, IReadOnlyList outboxEvents, Ct ct); + + /// + /// Purges a batch of expired entities atomically. Within a single transaction: + /// locks expired rows, deletes associated entity links, and deletes expired entities. + /// + /// Max expired entities to process (1–1000). + /// Cancellation token. + /// Number of entities deleted. + Task PurgeExpiredAsync(int batchSize, Ct ct); + + /// + /// Executes multiple operations atomically in a single transaction, writing outbox events atomically. + /// Operations are executed in order until completion or first failure. + /// If any operation fails, execution stops immediately and all operations are rolled back. + /// Outbox events are INSERTed after all operations succeed but before the transaction is committed. + /// + /// The operations to execute. + /// Outbox events to INSERT atomically within the same transaction, + /// after all operations succeed. Pass [] when no events are needed. + /// Silently ignored when the outbox is not enabled. + /// The cancellation token. + /// + /// A BatchResult indicating overall success/failure with per-operation outcomes. + /// When Success is false, Results contains outcomes only for operations attempted + /// (up to and including the failed operation). No changes have been persisted. + /// + Task ExecuteBatchAsync(IReadOnlyList operations, IReadOnlyList outboxEvents, Ct ct); + + /// + /// Retrieves the oldest page of outbox events for a specific subscriber, ordered by sequence number. + /// The intended usage pattern is: get oldest page → process events → delete them → repeat. + /// + /// The subscriber name to filter events for. + /// The maximum number of events to return. + /// Cancellation token. + /// A page of outbox events and whether more events exist beyond this page. + Task GetOutboxEventsForSubscriberAsync(SubscriberName subscriberName, int count, Ct ct); + + /// + /// Deletes outbox events by their message IDs (the store-generated per-row identifiers). + /// + /// The message IDs of the outbox events to delete. + /// Cancellation token. + Task DeleteOutboxEventsAsync(IReadOnlyList ids, Ct ct); + + /// + /// Queries entities with the specified pagination strategy. + /// + /// The type of the DSO to query. + /// The entity type to query. + /// The filter expression to apply. + /// Sort parameter. Use SortParameter.Empty if no sorting is required. + /// The pagination strategy (page, offset, or continuation token). + /// The cancellation token. + /// A query result containing items and pagination metadata. + Task>> QueryAsync( + EntityType entityType, + IQueryExpression filter, + SortParameter sort, + DataRange dataRange, + Ct ct) where TDso : IDataStorageObject; + + /// + /// Queries for specific field values with the specified pagination strategy. + /// Returns projected results instead of full entities. + /// + /// The entity type to query. + /// The fields to project in the results. + /// The filter expression to apply. + /// Sort parameter. Use SortParameter.Empty if no sorting is required. + /// The pagination strategy (page, offset, or continuation token). + /// The cancellation token. + /// A query result containing projected field values and pagination metadata. + Task> QueryFieldsAsync( + EntityType entityType, + IReadOnlyCollection fields, + IQueryExpression filter, + SortParameter sort, + DataRange dataRange, + Ct ct); + + /// + /// Queries for entities reachable via a chain of link traversals. + /// + /// The source entity type to return. + /// The link query descriptor describing the traversal chain and filter. + /// The pagination strategy. + /// Cancellation token. + /// A query result containing entities of the source type reachable via the link chain. + Task>> QueryLinksAsync( + LinkQueryDescriptor query, + DataRange dataRange, + Ct ct) where TDso : IDataStorageObject; + + /// + /// Counts entities matching the specified filter, or all entities if no filter is provided. + /// + /// The entity type to count. + /// Optional filter expression. If null, counts all entities of the specified type. + /// The cancellation token. + /// The number of matching entities. + Task CountAsync( + EntityType entityType, + IQueryExpression? filter, + Ct ct); +} diff --git a/storage/src/Storage/Internal/InstrumentedStore.cs b/storage/src/Storage/Internal/InstrumentedStore.cs new file mode 100644 index 000000000..7a72ab6a2 --- /dev/null +++ b/storage/src/Storage/Internal/InstrumentedStore.cs @@ -0,0 +1,519 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Diagnostics; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Outbox; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Internal.Querying.SearchFields; +using Duende.Storage.Internal.Querying.Sorting; +using Duende.Storage.Internal.Telemetry; +using Duende.Storage.Pagination; +using Duende.Storage.Querying; + +namespace Duende.Storage.Internal; + +/// +/// Decorates an with tracing and metrics instrumentation. +/// +public sealed class InstrumentedStore(IStore inner, StorageMetrics metrics, string dbSystem) : IStore +{ + public IStore Inner => inner; + + public void SetPoolId(PoolId poolId) => inner.SetPoolId(poolId); + + public async Task CreateAsync( + UuidV7 id, + TDso value, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration expiration, + IReadOnlyList outboxEvents, + Ct ct) where TDso : IDataStorageObject + { + var entityType = TDso.DsoVersion.EntityType.Name; + using var activity = StartActivity("Store.Create", entityType, StorageTelemetryConstants.Operations.Create); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.CreateAsync(id, value, keys, searchFieldCollection, expiration, outboxEvents, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.Create, dbSystem, entityType); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.Create, dbSystem, ex, entityType); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.Create, Elapsed(start), dbSystem, result, entityType); + } + } + + public async Task TryReadAsync(EntityType type, UuidV7 id, Ct ct) + { + var entityType = type.Name; + using var activity = StartActivity("Store.Read", entityType, StorageTelemetryConstants.Operations.Read); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.TryReadAsync(type, id, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.Read, dbSystem, entityType); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.Read, dbSystem, ex, entityType); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.Read, Elapsed(start), dbSystem, result, entityType); + } + } + + public async Task TryReadAsync(EntityType type, DataStorageKey key, Ct ct) + { + var entityType = type.Name; + using var activity = StartActivity("Store.Read", entityType, StorageTelemetryConstants.Operations.Read); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.TryReadAsync(type, key, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.Read, dbSystem, entityType); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.Read, dbSystem, ex, entityType); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.Read, Elapsed(start), dbSystem, result, entityType); + } + } + + public async Task> TryReadManyAsync( + EntityType entityType, + IReadOnlySet ids, + int maximum, + Ct ct) + { + var entityTypeName = entityType.Name; + using var activity = StartActivity("Store.ReadMany", entityTypeName, StorageTelemetryConstants.Operations.ReadMany); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.TryReadManyAsync(entityType, ids, maximum, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.ReadMany, dbSystem, entityTypeName); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.ReadMany, dbSystem, ex, entityTypeName); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.ReadMany, Elapsed(start), dbSystem, result, entityTypeName); + } + } + + public async Task UpdateAsync( + UuidV7 id, + TDso dso, + int expectedEntityVersion, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration? expiration, + IReadOnlyList outboxEvents, + Ct ct) where TDso : IDataStorageObject + { + var entityType = TDso.DsoVersion.EntityType.Name; + using var activity = StartActivity("Store.Update", entityType, StorageTelemetryConstants.Operations.Update); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.UpdateAsync(id, dso, expectedEntityVersion, keys, searchFieldCollection, expiration, outboxEvents, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.Update, dbSystem, entityType); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.Update, dbSystem, ex, entityType); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.Update, Elapsed(start), dbSystem, result, entityType); + } + } + + public async Task DeleteAsync(EntityType entityType, UuidV7 id, IReadOnlyList outboxEvents, Ct ct) + { + var entityTypeName = entityType.Name; + using var activity = StartActivity("Store.Delete", entityTypeName, StorageTelemetryConstants.Operations.Delete); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.DeleteAsync(entityType, id, outboxEvents, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.Delete, dbSystem, entityTypeName); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.Delete, dbSystem, ex, entityTypeName); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.Delete, Elapsed(start), dbSystem, result, entityTypeName); + } + } + + public async Task DeleteAsync(EntityType entityType, DataStorageKey key, IReadOnlyList outboxEvents, Ct ct) + { + var entityTypeName = entityType.Name; + using var activity = StartActivity("Store.Delete", entityTypeName, StorageTelemetryConstants.Operations.Delete); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.DeleteAsync(entityType, key, outboxEvents, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.Delete, dbSystem, entityTypeName); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.Delete, dbSystem, ex, entityTypeName); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.Delete, Elapsed(start), dbSystem, result, entityTypeName); + } + } + + public async Task LinkAsync(LinkDefinition definition, UuidV7 leftEntityId, UuidV7 rightEntityId, IReadOnlyList outboxEvents, Ct ct) + { + using var activity = StartActivity("Store.Link", null, StorageTelemetryConstants.Operations.Link); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.LinkAsync(definition, leftEntityId, rightEntityId, outboxEvents, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.Link, dbSystem, null); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.Link, dbSystem, ex, null); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.Link, Elapsed(start), dbSystem, result, null); + } + } + + public async Task UnlinkAsync(LinkDefinition definition, UuidV7 leftEntityId, UuidV7 rightEntityId, IReadOnlyList outboxEvents, Ct ct) + { + using var activity = StartActivity("Store.Unlink", null, StorageTelemetryConstants.Operations.Unlink); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.UnlinkAsync(definition, leftEntityId, rightEntityId, outboxEvents, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.Unlink, dbSystem, null); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.Unlink, dbSystem, ex, null); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.Unlink, Elapsed(start), dbSystem, result, null); + } + } + + public async Task PurgeExpiredAsync(int batchSize, Ct ct) + { + using var activity = StartActivity("Store.PurgeExpired", null, StorageTelemetryConstants.Operations.PurgeExpired); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.PurgeExpiredAsync(batchSize, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.PurgeExpired, dbSystem, null); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.PurgeExpired, dbSystem, ex, null); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.PurgeExpired, Elapsed(start), dbSystem, result, null); + } + } + + public async Task ExecuteBatchAsync(IReadOnlyList operations, IReadOnlyList outboxEvents, Ct ct) + { + using var activity = StartActivity("Store.ExecuteBatch", null, StorageTelemetryConstants.Operations.Batch); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.ExecuteBatchAsync(operations, outboxEvents, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.Batch, dbSystem, null); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.Batch, dbSystem, ex, null); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.Batch, Elapsed(start), dbSystem, result, null); + } + } + + public async Task GetOutboxEventsForSubscriberAsync(SubscriberName subscriberName, int count, Ct ct) + { + using var activity = StartActivity("Store.GetOutboxEvents", null, StorageTelemetryConstants.Operations.OutboxGet); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.GetOutboxEventsForSubscriberAsync(subscriberName, count, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.OutboxGet, dbSystem, null); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.OutboxGet, dbSystem, ex, null); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.OutboxGet, Elapsed(start), dbSystem, result, null); + } + } + + public async Task DeleteOutboxEventsAsync(IReadOnlyList ids, Ct ct) + { + using var activity = StartActivity("Store.DeleteOutboxEvents", null, StorageTelemetryConstants.Operations.OutboxDelete); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + await inner.DeleteOutboxEventsAsync(ids, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.OutboxDelete, dbSystem, null); + succeeded = true; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.OutboxDelete, dbSystem, ex, null); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.OutboxDelete, Elapsed(start), dbSystem, result, null); + } + } + + public async Task>> QueryAsync( + EntityType entityType, + IQueryExpression filter, + SortParameter sort, + DataRange dataRange, + Ct ct) where TDso : IDataStorageObject + { + var entityTypeName = entityType.Name; + using var activity = StartActivity("QueryStore.Query", entityTypeName, StorageTelemetryConstants.Operations.Query); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.QueryAsync(entityType, filter, sort, dataRange, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.Query, dbSystem, entityTypeName); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.Query, dbSystem, ex, entityTypeName); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.Query, Elapsed(start), dbSystem, result, entityTypeName); + } + } + + public async Task> QueryFieldsAsync( + EntityType entityType, + IReadOnlyCollection fields, + IQueryExpression filter, + SortParameter sort, + DataRange dataRange, + Ct ct) + { + var entityTypeName = entityType.Name; + using var activity = StartActivity("QueryStore.QueryFields", entityTypeName, StorageTelemetryConstants.Operations.QueryFields); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.QueryFieldsAsync(entityType, fields, filter, sort, dataRange, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.QueryFields, dbSystem, entityTypeName); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.QueryFields, dbSystem, ex, entityTypeName); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.QueryFields, Elapsed(start), dbSystem, result, entityTypeName); + } + } + + public async Task>> QueryLinksAsync( + LinkQueryDescriptor query, + DataRange dataRange, + Ct ct) where TDso : IDataStorageObject + { + using var activity = StartActivity("QueryStore.QueryLinks", null, StorageTelemetryConstants.Operations.QueryLinks); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.QueryLinksAsync(query, dataRange, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.QueryLinks, dbSystem, null); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.QueryLinks, dbSystem, ex, null); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.QueryLinks, Elapsed(start), dbSystem, result, null); + } + } + + public async Task CountAsync(EntityType entityType, IQueryExpression? filter, Ct ct) + { + var entityTypeName = entityType.Name; + using var activity = StartActivity("QueryStore.Count", entityTypeName, StorageTelemetryConstants.Operations.Count); + var start = Stopwatch.GetTimestamp(); + var succeeded = false; + try + { + var result = await inner.CountAsync(entityType, filter, ct); + metrics.RecordSuccess(StorageTelemetryConstants.Operations.Count, dbSystem, entityTypeName); + succeeded = true; + return result; + } + catch (Exception ex) + { + RecordException(activity, ex); + metrics.RecordError(StorageTelemetryConstants.Operations.Count, dbSystem, ex, entityTypeName); + throw; + } + finally + { + var result = succeeded ? StorageTelemetryConstants.TagValues.Success : StorageTelemetryConstants.TagValues.Error; + metrics.RecordDuration(StorageTelemetryConstants.Operations.Count, Elapsed(start), dbSystem, result, entityTypeName); + } + } + + private Activity? StartActivity(string name, string? entityType, string operation) + { + var activity = StorageTracing.ActivitySource.StartActivity(name); + if (activity is not null) + { + _ = activity.SetTag(StorageTelemetryConstants.Tags.DbSystem, dbSystem); + _ = activity.SetTag(StorageTelemetryConstants.Tags.Operation, operation); + if (entityType is not null) + { + _ = activity.SetTag(StorageTelemetryConstants.Tags.EntityType, entityType); + } + } + + return activity; + } + + private static void RecordException(Activity? activity, Exception ex) + { + if (activity is not null) + { + _ = activity.SetStatus(ActivityStatusCode.Error, ex.Message); + _ = activity.SetTag(StorageTelemetryConstants.Tags.ErrorType, ex.GetType().Name); + } + } + + private static double Elapsed(long start) => Stopwatch.GetElapsedTime(start).TotalSeconds; +} diff --git a/storage/src/Storage/Internal/LinkDefinition.cs b/storage/src/Storage/Internal/LinkDefinition.cs new file mode 100644 index 000000000..92be12af3 --- /dev/null +++ b/storage/src/Storage/Internal/LinkDefinition.cs @@ -0,0 +1,20 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +/// +/// Defines the schema for a link type — binding a with its left and right s. +/// Define link definitions once as static instances and reference them everywhere. +/// +public sealed record LinkDefinition +{ + /// The entity type on the left side of the link. + public required EntityType Left { get; init; } + + /// The entity type on the right side of the link. + public required EntityType Right { get; init; } + + /// The link type identifying this relationship. + public required LinkType Link { get; init; } +} diff --git a/storage/src/Storage/Internal/LinkResult.cs b/storage/src/Storage/Internal/LinkResult.cs new file mode 100644 index 000000000..98f8c22fe --- /dev/null +++ b/storage/src/Storage/Internal/LinkResult.cs @@ -0,0 +1,16 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +/// +/// The result of a Link operation on . +/// +public enum LinkResult +{ + /// The link was created successfully. + Success, + + /// The exact same link already exists (idempotent — not an error, but callers can detect duplicates). + AlreadyLinked, +} diff --git a/storage/src/Storage/Internal/LinkType.cs b/storage/src/Storage/Internal/LinkType.cs new file mode 100644 index 000000000..fcfdf3bb1 --- /dev/null +++ b/storage/src/Storage/Internal/LinkType.cs @@ -0,0 +1,20 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +/// +/// The type of link between two entities. +/// +/// A number representation for the link type. +/// The name of the link type. This name is only used for display purposes and should never change. +public readonly record struct LinkType(uint Id, string Name) +{ + [Obsolete("Don't use this constructor")] + public LinkType() : this(0!, null!) => throw new InvalidOperationException("Cannot instantiate LinkType without parameters"); + + public static LinkType ToLinkType(LinkTypeRegistry registry) => + new((uint)registry, registry.ToString()); + + public static implicit operator LinkType(LinkTypeRegistry value) => ToLinkType(value); +} diff --git a/storage/src/Storage/Internal/LinkTypeRegistry.cs b/storage/src/Storage/Internal/LinkTypeRegistry.cs new file mode 100644 index 000000000..d5c237e0a --- /dev/null +++ b/storage/src/Storage/Internal/LinkTypeRegistry.cs @@ -0,0 +1,20 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +/// +/// A registry of all link types stored in the system. Each link type must have a unique integer identifier. +/// +/// Once a link type is assigned an identifier, it must never be changed or reused for a different link type. +/// +#pragma warning disable CA1008 // Enums should have zero value. We explicitly want to avoid assigning a zero value to any link type, to catch uninitialized values. +public enum LinkTypeRegistry +#pragma warning restore CA1008 +{ + // user profile link types (1500-1599 range, aligned with entity types) + GroupRole = 1502, + + MembershipRole = 1503, + MembershipGroup = 1504, +} diff --git a/storage/src/Storage/Internal/Operations/BatchResult.cs b/storage/src/Storage/Internal/Operations/BatchResult.cs new file mode 100644 index 000000000..c6fe09db0 --- /dev/null +++ b/storage/src/Storage/Internal/Operations/BatchResult.cs @@ -0,0 +1,27 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Operations; + +/// +/// Represents the result of a batch operation. +/// +/// True if all operations succeeded; false if any failed (and all were rolled back). +/// The outcome of each operation, in order. +public sealed record BatchResult(bool Success, IReadOnlyList Results) +{ + /// + /// Creates a successful for a batch of the specified size. + /// + /// The number of operations in the batch. + /// A with all operations having . + public static BatchResult Successful(int count) + { + var results = new OperationResult[count]; + for (var i = 0; i < count; i++) + { + results[i] = new OperationResult(i, OperationOutcome.Success); + } + return new BatchResult(true, results); + } +} diff --git a/storage/src/Storage/Internal/Operations/CreateOperation.cs b/storage/src/Storage/Internal/Operations/CreateOperation.cs new file mode 100644 index 000000000..634b67bb4 --- /dev/null +++ b/storage/src/Storage/Internal/Operations/CreateOperation.cs @@ -0,0 +1,112 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.SearchFields; + +namespace Duende.Storage.Internal.Operations; + +/// +/// Represents a create operation for batch processing. +/// +public sealed class CreateOperation : IStoreOperation +{ + private CreateOperation( + EntityType entityType, + UuidV7 id, + object value, + DataStorageObjectVersion dsoVersion, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration expiration) + { + EntityType = entityType; + Id = id; + Value = value; + DsoVersion = dsoVersion; + Keys = keys; + SearchFieldCollection = searchFieldCollection; + Expiration = expiration; + } + + /// + /// Gets the entity type for this operation. + /// + public EntityType EntityType { get; } + + /// + /// Gets the unique identifier for the entity to create. + /// + public UuidV7 Id { get; } + + /// + /// Gets the DSO value to store. + /// + internal object Value { get; } + + /// + /// Gets the DSO version for serialization. + /// + internal DataStorageObjectVersion DsoVersion { get; } + + /// + /// Gets the collection of keys for alternate lookups. + /// + internal IReadOnlyCollection Keys { get; } + + /// + /// Gets the search field values for querying. + /// + internal SearchFieldCollection SearchFieldCollection { get; } + + /// + /// Gets the expiration policy for the entity. + /// + internal Expiration Expiration { get; } + + /// + /// Creates a new create operation for the specified DSO type. + /// + /// The unique identifier for the entity. + /// The DSO value to store. + /// The collection of keys for alternate lookups. + /// The search field values for querying. + /// The expiration policy for the entity. + /// A new create operation. + public static CreateOperation For( + UuidV7 id, + TypedDso dso, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration expiration) => new CreateOperation( + dso.EntityType, + id, + dso.Value, + dso.Version, + keys, + searchFieldCollection, + expiration); + + /// + /// Creates a new create operation for the specified DSO type. + /// + /// The type of the DSO to create. + /// The unique identifier for the entity. + /// The DSO value to store. + /// The collection of keys for alternate lookups. + /// The search field values for querying. + /// The expiration policy for the entity. + /// A new create operation. + public static CreateOperation For( + UuidV7 id, + TDso value, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration expiration) where TDso : IDataStorageObject => new CreateOperation( + TDso.DsoVersion.EntityType, + id, + value, + TDso.DsoVersion, + keys, + searchFieldCollection, + expiration); +} diff --git a/storage/src/Storage/Internal/Operations/CreateResult.cs b/storage/src/Storage/Internal/Operations/CreateResult.cs new file mode 100644 index 000000000..0781fabaf --- /dev/null +++ b/storage/src/Storage/Internal/Operations/CreateResult.cs @@ -0,0 +1,12 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Operations; + +public enum CreateResult +{ + Success, + AlreadyExists, + KeyConflict, + ConcurrencyConflict +} diff --git a/storage/src/Storage/Internal/Operations/DeleteOperation.cs b/storage/src/Storage/Internal/Operations/DeleteOperation.cs new file mode 100644 index 000000000..11c7d8ef2 --- /dev/null +++ b/storage/src/Storage/Internal/Operations/DeleteOperation.cs @@ -0,0 +1,48 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Operations; + +/// +/// Represents a delete operation for batch processing. +/// +public sealed class DeleteOperation : IStoreOperation +{ + private DeleteOperation(EntityType entityType, UuidV7? id, DataStorageKey? key) + { + EntityType = entityType; + Id = id; + Key = key; + } + + /// + /// Gets the entity type for this operation. + /// + public EntityType EntityType { get; } + + /// + /// Gets the unique identifier for the entity to delete. + /// + public UuidV7? Id { get; } + + /// + /// Gets the alternate key for the entity to delete. + /// + public DataStorageKey? Key { get; } + + /// + /// Creates a delete operation that targets an entity by its primary ID. + /// + /// The entity type. + /// The unique identifier of the entity to delete. + /// A new delete operation. + public static DeleteOperation ById(EntityType entityType, UuidV7 id) => new DeleteOperation(entityType, id, null); + + /// + /// Creates a delete operation that targets an entity by an alternate key. + /// + /// The entity type. + /// The alternate key of the entity to delete. + /// A new delete operation. + public static DeleteOperation ByKey(EntityType entityType, DataStorageKey key) => new DeleteOperation(entityType, null, key); +} diff --git a/storage/src/Storage/Internal/Operations/DeleteResult.cs b/storage/src/Storage/Internal/Operations/DeleteResult.cs new file mode 100644 index 000000000..3fc0b25da --- /dev/null +++ b/storage/src/Storage/Internal/Operations/DeleteResult.cs @@ -0,0 +1,9 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Operations; + +public enum DeleteResult +{ + Success +} diff --git a/storage/src/Storage/Internal/Operations/IStoreOperation.cs b/storage/src/Storage/Internal/Operations/IStoreOperation.cs new file mode 100644 index 000000000..3628c6cc8 --- /dev/null +++ b/storage/src/Storage/Internal/Operations/IStoreOperation.cs @@ -0,0 +1,15 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Operations; + +/// +/// Marker interface for batch operations. +/// +public interface IStoreOperation +{ + /// + /// Gets the entity type for this operation. + /// + EntityType EntityType { get; } +} diff --git a/storage/src/Storage/Internal/Operations/LinkOperation.cs b/storage/src/Storage/Internal/Operations/LinkOperation.cs new file mode 100644 index 000000000..1869daa30 --- /dev/null +++ b/storage/src/Storage/Internal/Operations/LinkOperation.cs @@ -0,0 +1,48 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Operations; + +/// +/// Represents a link operation for batch processing. +/// Creates a link between two entities as part of an atomic batch. +/// +public sealed class LinkOperation : IStoreOperation +{ + private LinkOperation(LinkDefinition definition, UuidV7 leftEntityId, UuidV7 rightEntityId) + { + Definition = definition; + LeftEntityId = leftEntityId; + RightEntityId = rightEntityId; + } + + /// + /// Gets the entity type for this operation (the left entity type, for the IStoreOperation contract). + /// + public EntityType EntityType => Definition.Left; + + /// + /// Gets the link definition describing the relationship schema. + /// + public LinkDefinition Definition { get; } + + /// + /// Gets the ID of the left-side entity. + /// + public UuidV7 LeftEntityId { get; } + + /// + /// Gets the ID of the right-side entity. + /// + public UuidV7 RightEntityId { get; } + + /// + /// Creates a new link operation. + /// + /// The link definition. + /// The ID of the left-side entity. + /// The ID of the right-side entity. + /// A new link operation. + public static LinkOperation For(LinkDefinition definition, UuidV7 leftId, UuidV7 rightId) => + new LinkOperation(definition, leftId, rightId); +} diff --git a/storage/src/Storage/Internal/Operations/OperationOutcome.cs b/storage/src/Storage/Internal/Operations/OperationOutcome.cs new file mode 100644 index 000000000..e91c0e461 --- /dev/null +++ b/storage/src/Storage/Internal/Operations/OperationOutcome.cs @@ -0,0 +1,40 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Operations; + +/// +/// Represents the outcome of an individual operation within a batch. +/// +public enum OperationOutcome +{ + /// + /// The operation succeeded. + /// + Success, + + /// + /// Create failed: an entity with the same ID already exists. + /// + AlreadyExists, + + /// + /// Create or Update failed: a key is already used by another entity. + /// + KeyConflict, + + /// + /// Update or Delete failed: the entity was not found. + /// + DoesNotExist, + + /// + /// Update failed: the entity version did not match the expected version. + /// + UnexpectedVersion, + + /// + /// Link failed: an identical link (same LinkType, LeftId, RightId) already exists. + /// + AlreadyLinked +} diff --git a/storage/src/Storage/Internal/Operations/OperationResult.cs b/storage/src/Storage/Internal/Operations/OperationResult.cs new file mode 100644 index 000000000..b55a6aba6 --- /dev/null +++ b/storage/src/Storage/Internal/Operations/OperationResult.cs @@ -0,0 +1,11 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Operations; + +/// +/// Represents the result of an individual operation within a batch. +/// +/// The zero-based index of the operation in the batch. +/// The outcome of the operation. +public sealed record OperationResult(int Index, OperationOutcome Outcome); diff --git a/storage/src/Storage/Internal/Operations/StoreGetResult.cs b/storage/src/Storage/Internal/Operations/StoreGetResult.cs new file mode 100644 index 000000000..d2d8be7c8 --- /dev/null +++ b/storage/src/Storage/Internal/Operations/StoreGetResult.cs @@ -0,0 +1,46 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.Internal.Operations; + +/// +/// Wraps the result of a Get operation from the store. +/// +public sealed record StoreGetResult +{ + public StoreGetResult(IDataStorageObject dso, Guid id, int version, DateTimeOffset createdAt, DateTimeOffset lastUpdatedAt) + { + Found = true; + Dso = dso; + Id = id; + Version = version; + CreatedAt = createdAt; + LastUpdatedAt = lastUpdatedAt; + } + + private StoreGetResult() + { + } + + public static StoreGetResult NotFound() => new(); + + [MemberNotNullWhen(true, nameof(Dso))] + [MemberNotNullWhen(true, nameof(Id))] + [MemberNotNullWhen(true, nameof(Version))] + public bool Found { get; } + + public IDataStorageObject? Dso { get; } + + public Guid? Id { get; } + + public int? Version { get; } + + public DateTimeOffset CreatedAt { get; } + + public DateTimeOffset LastUpdatedAt { get; } + + public static StoreGetResult IsFound(IDataStorageObject item, Guid id, int version, DateTimeOffset createdAt, DateTimeOffset lastUpdatedAt) => + new(item, id, version, createdAt, lastUpdatedAt); +} diff --git a/storage/src/Storage/Internal/Operations/UnlinkOperation.cs b/storage/src/Storage/Internal/Operations/UnlinkOperation.cs new file mode 100644 index 000000000..764b8ec5c --- /dev/null +++ b/storage/src/Storage/Internal/Operations/UnlinkOperation.cs @@ -0,0 +1,48 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Operations; + +/// +/// Represents an unlink operation for batch processing. +/// Removes a link between two entities as part of an atomic batch. +/// +public sealed class UnlinkOperation : IStoreOperation +{ + private UnlinkOperation(LinkDefinition definition, UuidV7 leftEntityId, UuidV7 rightEntityId) + { + Definition = definition; + LeftEntityId = leftEntityId; + RightEntityId = rightEntityId; + } + + /// + /// Gets the entity type for this operation (the left entity type, for the IStoreOperation contract). + /// + public EntityType EntityType => Definition.Left; + + /// + /// Gets the link definition describing the relationship schema. + /// + public LinkDefinition Definition { get; } + + /// + /// Gets the ID of the left-side entity. + /// + public UuidV7 LeftEntityId { get; } + + /// + /// Gets the ID of the right-side entity. + /// + public UuidV7 RightEntityId { get; } + + /// + /// Creates a new unlink operation. + /// + /// The link definition. + /// The ID of the left-side entity. + /// The ID of the right-side entity. + /// A new unlink operation. + public static UnlinkOperation For(LinkDefinition definition, UuidV7 leftId, UuidV7 rightId) => + new UnlinkOperation(definition, leftId, rightId); +} diff --git a/storage/src/Storage/Internal/Operations/UpdateOperation.cs b/storage/src/Storage/Internal/Operations/UpdateOperation.cs new file mode 100644 index 000000000..90a12eff4 --- /dev/null +++ b/storage/src/Storage/Internal/Operations/UpdateOperation.cs @@ -0,0 +1,126 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.SearchFields; + +namespace Duende.Storage.Internal.Operations; + +/// +/// Represents an update operation for batch processing. +/// +public sealed class UpdateOperation : IStoreOperation +{ + private UpdateOperation( + EntityType entityType, + UuidV7 id, + object value, + DataStorageObjectVersion dsoVersion, + int expectedEntityVersion, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration? expiration) + { + EntityType = entityType; + Id = id; + Value = value; + DsoVersion = dsoVersion; + ExpectedEntityVersion = expectedEntityVersion; + Keys = keys; + SearchFieldCollection = searchFieldCollection; + Expiration = expiration; + } + + /// + /// Gets the entity type for this operation. + /// + public EntityType EntityType { get; } + + /// + /// Gets the unique identifier for the entity to update. + /// + public UuidV7 Id { get; } + + /// + /// Gets the DSO value to store. + /// + internal object Value { get; } + + /// + /// Gets the DSO version for serialization. + /// + internal DataStorageObjectVersion DsoVersion { get; } + + /// + /// Gets the expected version for optimistic concurrency control. + /// + internal int ExpectedEntityVersion { get; } + + /// + /// Gets the collection of keys for alternate lookups. + /// + internal IReadOnlyCollection Keys { get; } + + /// + /// Gets the search field values for querying. + /// + internal SearchFieldCollection SearchFieldCollection { get; } + + /// + /// Gets the expiration policy for the entity. + /// null means "don't change existing expiration". + /// + internal Expiration? Expiration { get; } + + /// + /// Creates a new update operation for the specified DSO type. + /// + /// The unique identifier for the entity. + /// The typed DSO wrapper containing the value to update. + /// The expected entity version for optimistic concurrency. + /// The collection of keys for alternate lookups. + /// The search field values for querying. + /// The expiration policy. null means "don't change". + /// A new update operation. + public static UpdateOperation For( + UuidV7 id, + TypedDso dso, + int expectedEntityVersion, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration? expiration) => new UpdateOperation( + dso.EntityType, + id, + dso.Value, + dso.Version, + expectedEntityVersion, + keys, + searchFieldCollection, + expiration); + + /// + /// Creates a new update operation for the specified DSO type. + /// + /// The type of the DSO to update. + /// The unique identifier for the entity. + /// The new DSO value to store. + /// The expected version for optimistic concurrency control. + /// The collection of keys for alternate lookups. + /// The search field values for querying. + /// The expiration policy. null means "don't change". + /// A new update operation. + public static UpdateOperation For( + UuidV7 id, + TDso value, + int expectedEntityVersion, + IReadOnlyCollection keys, + SearchFieldCollection searchFieldCollection, + Expiration? expiration) where TDso : IDataStorageObject => new UpdateOperation( + TDso.DsoVersion.EntityType, + id, + value, + TDso.DsoVersion, + expectedEntityVersion, + keys, + searchFieldCollection, + expiration); +} diff --git a/storage/src/Storage/Internal/Operations/UpdateResult.cs b/storage/src/Storage/Internal/Operations/UpdateResult.cs new file mode 100644 index 000000000..f7fd5514d --- /dev/null +++ b/storage/src/Storage/Internal/Operations/UpdateResult.cs @@ -0,0 +1,12 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Operations; + +public enum UpdateResult +{ + Success, + DoesNotExist, + UnexpectedVersion, + KeyConflict +} diff --git a/storage/src/Storage/Internal/Outbox/HandleOutcomeResult.cs b/storage/src/Storage/Internal/Outbox/HandleOutcomeResult.cs new file mode 100644 index 000000000..116319a0e --- /dev/null +++ b/storage/src/Storage/Internal/Outbox/HandleOutcomeResult.cs @@ -0,0 +1,44 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Outbox; + +/// +/// Represents the outcome of a handler processing an outbox event. +/// Handlers return one of: , , or . +/// +public abstract record HandleOutcomeResult +{ + private HandleOutcomeResult() { } + + /// Returns a success outcome — the event was processed and should be deleted. + public static HandleOutcomeResult Success() => SuccessResult.Instance; + + /// + /// Returns a retry outcome — the event could not be processed and should be retried after a delay. + /// + /// Human-readable reason for the retry request. + public static HandleOutcomeResult Retry(string reason) => new RetryResult(reason); + + /// + /// Returns a drop outcome — the event cannot be processed and should be deleted without retry. + /// + /// Human-readable reason for dropping the event. + public static HandleOutcomeResult Drop(string reason) => new DropResult(reason); + + /// Successful processing outcome. + public sealed record SuccessResult : HandleOutcomeResult + { + internal static readonly SuccessResult Instance = new(); + + private SuccessResult() { } + } + + /// Retry-requested outcome. + /// Human-readable reason for the retry request. + public sealed record RetryResult(string Reason) : HandleOutcomeResult; + + /// Drop-requested outcome. + /// Human-readable reason for dropping the event. + public sealed record DropResult(string Reason) : HandleOutcomeResult; +} diff --git a/storage/src/Storage/Internal/Outbox/IOutboxSubscriber.cs b/storage/src/Storage/Internal/Outbox/IOutboxSubscriber.cs new file mode 100644 index 000000000..28afbc08d --- /dev/null +++ b/storage/src/Storage/Internal/Outbox/IOutboxSubscriber.cs @@ -0,0 +1,34 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Outbox; + +/// +/// Declares a subscriber that wants to receive outbox events via the fanout mechanism. +/// Subscribers are registered in DI and matched against outbox events by event name and entity type. +/// +public interface IOutboxSubscriber +{ + /// The unique name identifying this subscriber. + SubscriberName SubscriberName { get; } + + /// + /// Whether this subscriber is currently enabled for outbox event delivery. + /// Disabled subscribers are excluded from fanout and will not receive any events. + /// + bool IsEnabled { get; } + + /// + /// The event names this subscriber listens to. + /// An empty set means the subscriber receives all events (wildcard). + /// A non-empty set means only the specified event names are received. + /// + IReadOnlySet EventNames { get; } + + /// + /// The entity type IDs this subscriber listens to. + /// An empty set means the subscriber receives events for all entity types (wildcard). + /// A non-empty set means only the specified entity type IDs are received. + /// + IReadOnlySet EntityTypeIds { get; } +} diff --git a/storage/src/Storage/Internal/Outbox/IOutboxSubscriberHandler.cs b/storage/src/Storage/Internal/Outbox/IOutboxSubscriberHandler.cs new file mode 100644 index 000000000..29c24bf4c --- /dev/null +++ b/storage/src/Storage/Internal/Outbox/IOutboxSubscriberHandler.cs @@ -0,0 +1,25 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Outbox; + +/// +/// Defines the handler contract for processing outbox events for a specific subscriber. +/// Implementations are registered in DI using keyed services, keyed by subscriber name. +/// +public interface IOutboxSubscriberHandler +{ + /// Handles a single outbox event delivered to this subscriber. + /// The outbox event to process. + /// Cancellation token. + /// + /// A indicating how the job should proceed: + /// + /// — event processed; will be deleted. + /// — transient failure; retry after a delay. + /// — permanent failure; delete without retry. + /// + /// Unhandled exceptions are treated as . + /// + Task HandleAsync(PersistedOutboxEvent item, Ct ct); +} diff --git a/storage/src/Storage/Internal/Outbox/OutboxEvent.cs b/storage/src/Storage/Internal/Outbox/OutboxEvent.cs new file mode 100644 index 000000000..2d57ddd90 --- /dev/null +++ b/storage/src/Storage/Internal/Outbox/OutboxEvent.cs @@ -0,0 +1,33 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Outbox; + +/// +/// Represents an event to be written to the outbox table atomically alongside a domain operation. +/// The store implementation stamps PoolId from the ambient pool context, and the +/// database automatically assigns the SequenceNumber on insert. +/// +public sealed record OutboxEvent +{ + /// The unique identifier for this outbox event. + public required OutboxEventId Id { get; init; } + + /// When the event occurred. + public required DateTimeOffset Timestamp { get; init; } + + /// The name of the domain event (e.g. "UserCreated"). + public required OutboxEventName EventName { get; init; } + + /// The ID of the entity that is the subject of this event. + public required UuidV7 SubjectId { get; init; } + + /// The name of the entity type (e.g. "User"). + public required string EntityTypeName { get; init; } + + /// The numeric ID of the entity type. + public required int EntityTypeId { get; init; } + + /// The serialized event payload. Must be valid JSON. + public required string Payload { get; init; } +} diff --git a/storage/src/Storage/Internal/Outbox/OutboxEventId.cs b/storage/src/Storage/Internal/Outbox/OutboxEventId.cs new file mode 100644 index 000000000..b759e73dd --- /dev/null +++ b/storage/src/Storage/Internal/Outbox/OutboxEventId.cs @@ -0,0 +1,13 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Outbox; + +[ValueOf] +public partial record OutboxEventId +{ + /// + /// Creates a new OutboxEventId using a version 7 UUID. + /// + public static OutboxEventId New() => new(Guid.CreateVersion7()); +} diff --git a/storage/src/Storage/Internal/Outbox/OutboxEventId.g.cs b/storage/src/Storage/Internal/Outbox/OutboxEventId.g.cs new file mode 100644 index 000000000..0bb86a352 --- /dev/null +++ b/storage/src/Storage/Internal/Outbox/OutboxEventId.g.cs @@ -0,0 +1,72 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Collections.Generic; +using Duende.Storage; +using System.Globalization; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.Internal.Outbox; + +[System.ComponentModel.TypeConverter(typeof(ValueOfTypeConverter))] +partial record OutboxEventId : IValueOf +{ + // Constructor for controlled creation + private OutboxEventId(global::System.Guid value) => Value = value; + + public global::System.Guid Value { get; } + + public static OutboxEventId Create(string s) + { + if (!TryCreate(s, out var result, out var errors)) + { + throw new FormatException($"The value '{s}' is not a valid '{nameof(OutboxEventId)}'. {string.Join(" ", errors)}"); + } + return result; + } + + public static bool TryCreate(string? s, [NotNullWhen(true)] out OutboxEventId? result) + => TryCreate(s, out result, out _); + + public static bool TryCreate(string? s, [NotNullWhen(true)] out OutboxEventId? result, [NotNullWhen(false)] out IReadOnlyList? errors) + { + result = null; + errors = null; + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["A value is required."]; + return false; + } + + if (global::System.Guid.TryParse(s, CultureInfo.InvariantCulture, out var value)) + { + var instance = new OutboxEventId(value); + result = instance; + return true; + } + + errors = ["The value could not be parsed."]; + return false; + } + + public static implicit operator OutboxEventId(global::System.Guid value) + { + return new OutboxEventId(value); + } + + public override string ToString() => Value.ToString(null, CultureInfo.InvariantCulture); + + public static OutboxEventId? CreateOrDefault(string? input) + { + if (string.IsNullOrEmpty(input)) + { + return null; + } + + return Create(input); + } + + internal static OutboxEventId Load(global::System.Guid value) => new OutboxEventId(value); +} diff --git a/storage/src/Storage/Internal/Outbox/OutboxEventName.cs b/storage/src/Storage/Internal/Outbox/OutboxEventName.cs new file mode 100644 index 000000000..b8a6d3768 --- /dev/null +++ b/storage/src/Storage/Internal/Outbox/OutboxEventName.cs @@ -0,0 +1,21 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Text.RegularExpressions; + +namespace Duende.Storage.Internal.Outbox; + +/// +/// The name of a domain event written to the outbox. Only alphanumeric characters, underscores, and hyphens are allowed. +/// +[StringValue] +public partial record OutboxEventName +{ + [GeneratedRegex(@"^[a-zA-Z0-9_\-]+$")] + private static partial Regex Regex(); + + /// + /// The well-known event name written when an entity is purged due to expiration. + /// + public static readonly OutboxEventName EntityExpired = Create("EntityExpired"); +} diff --git a/storage/src/Storage/Internal/Outbox/OutboxEventName.g.cs b/storage/src/Storage/Internal/Outbox/OutboxEventName.g.cs new file mode 100644 index 000000000..7a1079f1a --- /dev/null +++ b/storage/src/Storage/Internal/Outbox/OutboxEventName.g.cs @@ -0,0 +1,72 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Collections.Generic; +using Duende.Storage; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.Internal.Outbox; + +[System.ComponentModel.TypeConverter(typeof(ValueOfTypeConverter))] +partial record OutboxEventName : IStringValue +{ + // Constructor for controlled creation + private OutboxEventName(string value) => Value = value; + + public string Value { get; } + + public static OutboxEventName Create(string s) + { + if (!TryCreate(s, out var result, out var errors)) + { + throw new FormatException($"The value '{s}' is not a valid OutboxEventName. {string.Join(" ", errors)}"); + } + return result; + } + + public static bool TryCreate(string? s, [NotNullWhen(true)] out OutboxEventName? result) + => TryCreate(s, out result, out _); + + public static bool TryCreate(string? s, [NotNullWhen(true)] out OutboxEventName? result, [NotNullWhen(false)] out IReadOnlyList? errors) + { + result = null; + errors = null; + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["A value is required."]; + return false; + } + + var validationErrors = new List(); + if (!Regex().IsMatch(s)) + { + validationErrors.Add("Must match the required pattern."); + } + if (validationErrors.Count > 0) + { + errors = validationErrors; + return false; + } + result = new OutboxEventName(s); + return true; + } + + public static implicit operator OutboxEventName(string value) => Create(value); + + public override string ToString() => Value; + + public static OutboxEventName? CreateOrDefault(string? input) + { + if (string.IsNullOrEmpty(input)) + { + return null; + } + + return Create(input); + } + + internal static OutboxEventName Load(string value) => new OutboxEventName(value); + +} diff --git a/storage/src/Storage/Internal/Outbox/OutboxEventsPage.cs b/storage/src/Storage/Internal/Outbox/OutboxEventsPage.cs new file mode 100644 index 000000000..f54d98345 --- /dev/null +++ b/storage/src/Storage/Internal/Outbox/OutboxEventsPage.cs @@ -0,0 +1,11 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Outbox; + +/// +/// A paged result of outbox events ordered by sequence number. +/// +/// The outbox events in this page. +/// Whether there are more events beyond this page. +public sealed record OutboxEventsPage(IReadOnlyList Events, bool HasMore); diff --git a/storage/src/Storage/Internal/Outbox/PersistedOutboxEvent.cs b/storage/src/Storage/Internal/Outbox/PersistedOutboxEvent.cs new file mode 100644 index 000000000..5bae409b6 --- /dev/null +++ b/storage/src/Storage/Internal/Outbox/PersistedOutboxEvent.cs @@ -0,0 +1,44 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Outbox; + +/// +/// Represents an outbox event as read from the database, including the +/// database-assigned sequence number and the store-stamped space identifier. +/// +public sealed record PersistedOutboxEvent +{ + /// The store-generated unique identifier for this persisted message (one per subscriber fanout row). + public required OutboxEventId MessageId { get; init; } + + /// The caller-supplied event identifier, shared across all subscriber copies of the same logical event. + public required OutboxEventId EventId { get; init; } + + /// When the event occurred. + public required DateTimeOffset Timestamp { get; init; } + + /// Database-assigned monotonic sequence number for ordering and paging events. + public required long SequenceNumber { get; init; } + + /// The name of the domain event (e.g. "UserCreated"). + public required OutboxEventName EventName { get; init; } + + /// The ID of the entity that is the subject of this event. + public required UuidV7 SubjectId { get; init; } + + /// The name of the entity type (e.g. "User"). + public required string EntityTypeName { get; init; } + + /// The numeric ID of the entity type. + public required int EntityTypeId { get; init; } + + /// The pool this event belongs to, stamped by the store at write time. + public required PoolId PoolId { get; init; } + + /// The serialized event payload (typically JSON). + public required string Payload { get; init; } + + /// The name of the subscriber this event was addressed to at write time. + public required SubscriberName SubscriberName { get; init; } +} diff --git a/storage/src/Storage/Internal/Outbox/SubscriberName.cs b/storage/src/Storage/Internal/Outbox/SubscriberName.cs new file mode 100644 index 000000000..77de28734 --- /dev/null +++ b/storage/src/Storage/Internal/Outbox/SubscriberName.cs @@ -0,0 +1,16 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Text.RegularExpressions; + +namespace Duende.Storage.Internal.Outbox; + +/// +/// The unique name identifying an outbox subscriber. Only alphanumeric characters, underscores, and hyphens are allowed. +/// +[StringValue] +public partial record SubscriberName +{ + [GeneratedRegex(@"^[a-zA-Z0-9_\-]+$")] + private static partial Regex Regex(); +} diff --git a/storage/src/Storage/Internal/Outbox/SubscriberName.g.cs b/storage/src/Storage/Internal/Outbox/SubscriberName.g.cs new file mode 100644 index 000000000..0e8abd44c --- /dev/null +++ b/storage/src/Storage/Internal/Outbox/SubscriberName.g.cs @@ -0,0 +1,72 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Collections.Generic; +using Duende.Storage; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.Internal.Outbox; + +[System.ComponentModel.TypeConverter(typeof(ValueOfTypeConverter))] +partial record SubscriberName : IStringValue +{ + // Constructor for controlled creation + private SubscriberName(string value) => Value = value; + + public string Value { get; } + + public static SubscriberName Create(string s) + { + if (!TryCreate(s, out var result, out var errors)) + { + throw new FormatException($"The value '{s}' is not a valid SubscriberName. {string.Join(" ", errors)}"); + } + return result; + } + + public static bool TryCreate(string? s, [NotNullWhen(true)] out SubscriberName? result) + => TryCreate(s, out result, out _); + + public static bool TryCreate(string? s, [NotNullWhen(true)] out SubscriberName? result, [NotNullWhen(false)] out IReadOnlyList? errors) + { + result = null; + errors = null; + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["A value is required."]; + return false; + } + + var validationErrors = new List(); + if (!Regex().IsMatch(s)) + { + validationErrors.Add("Must match the required pattern."); + } + if (validationErrors.Count > 0) + { + errors = validationErrors; + return false; + } + result = new SubscriberName(s); + return true; + } + + public static implicit operator SubscriberName(string value) => Create(value); + + public override string ToString() => Value; + + public static SubscriberName? CreateOrDefault(string? input) + { + if (string.IsNullOrEmpty(input)) + { + return null; + } + + return Create(input); + } + + internal static SubscriberName Load(string value) => new SubscriberName(value); + +} diff --git a/storage/src/Storage/Internal/OutboxSubscribers.cs b/storage/src/Storage/Internal/OutboxSubscribers.cs new file mode 100644 index 000000000..ae66e89dc --- /dev/null +++ b/storage/src/Storage/Internal/OutboxSubscribers.cs @@ -0,0 +1,41 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Outbox; +using OutboxEventName = Duende.Storage.Internal.Outbox.OutboxEventName; + +namespace Duende.Storage.Internal; + +/// +/// Collects all enabled registrations from DI and provides +/// efficient lookup of matching subscribers for a given event and entity type. +/// Subscribers with set to false are excluded. +/// +internal sealed class OutboxSubscribers +{ + private readonly IReadOnlyList _subscribers; + + /// + /// Initializes the registry with the resolved set of subscribers, filtering out disabled ones. + /// + public OutboxSubscribers(IEnumerable subscribers) => + _subscribers = [.. subscribers.Where(s => s.IsEnabled)]; + + /// Returns true when no enabled subscribers are registered. + public bool IsEmpty => _subscribers.Count == 0; + + /// All enabled subscribers. + public IReadOnlyList Subscribers => _subscribers; + + /// + /// Returns all enabled subscribers that match the given event name and entity type ID. + /// An empty matches all entity types (wildcard). + /// An empty matches all event names (wildcard). + /// + public IEnumerable GetMatchingSubscribers(OutboxEventName eventName, int entityTypeId) => + _subscribers.Where(s => + (s.EntityTypeIds.Count == 0 || s.EntityTypeIds.Contains(entityTypeId)) && + (s.EventNames.Count == 0 || s.EventNames.Contains(eventName))); + + public bool HasSubscriber(OutboxEventName eventName, int entityTypeId) => GetMatchingSubscribers(eventName, entityTypeId).Any(); +} diff --git a/storage/src/Storage/Internal/PoolId.cs b/storage/src/Storage/Internal/PoolId.cs new file mode 100644 index 000000000..9d9f488fe --- /dev/null +++ b/storage/src/Storage/Internal/PoolId.cs @@ -0,0 +1,7 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +[ValueOf] +public partial record PoolId; diff --git a/storage/src/Storage/Internal/PoolId.g.cs b/storage/src/Storage/Internal/PoolId.g.cs new file mode 100644 index 000000000..222907498 --- /dev/null +++ b/storage/src/Storage/Internal/PoolId.g.cs @@ -0,0 +1,72 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Collections.Generic; +using Duende.Storage; +using System.Globalization; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.Internal; + +[System.ComponentModel.TypeConverter(typeof(ValueOfTypeConverter))] +partial record PoolId : IValueOf +{ + // Constructor for controlled creation + private PoolId(global::System.Int32 value) => Value = value; + + public global::System.Int32 Value { get; } + + public static PoolId Create(string s) + { + if (!TryCreate(s, out var result, out var errors)) + { + throw new FormatException($"The value '{s}' is not a valid '{nameof(PoolId)}'. {string.Join(" ", errors)}"); + } + return result; + } + + public static bool TryCreate(string? s, [NotNullWhen(true)] out PoolId? result) + => TryCreate(s, out result, out _); + + public static bool TryCreate(string? s, [NotNullWhen(true)] out PoolId? result, [NotNullWhen(false)] out IReadOnlyList? errors) + { + result = null; + errors = null; + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["A value is required."]; + return false; + } + + if (global::System.Int32.TryParse(s, CultureInfo.InvariantCulture, out var value)) + { + var instance = new PoolId(value); + result = instance; + return true; + } + + errors = ["The value could not be parsed."]; + return false; + } + + public static implicit operator PoolId(global::System.Int32 value) + { + return new PoolId(value); + } + + public override string ToString() => Value.ToString(null, CultureInfo.InvariantCulture); + + public static PoolId? CreateOrDefault(string? input) + { + if (string.IsNullOrEmpty(input)) + { + return null; + } + + return Create(input); + } + + internal static PoolId Load(global::System.Int32 value) => new PoolId(value); +} diff --git a/storage/src/Storage/Internal/PooledStore.cs b/storage/src/Storage/Internal/PooledStore.cs new file mode 100644 index 000000000..adbd58643 --- /dev/null +++ b/storage/src/Storage/Internal/PooledStore.cs @@ -0,0 +1,50 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.Internal; + +internal class PooledStore(IServiceProvider serviceProvider, object? serviceKey) : IPooledStore +{ + public IStore OpenPool(PoolId poolId) + { + var store = serviceKey is null + ? serviceProvider.GetRequiredService() + : serviceProvider.GetRequiredKeyedService(serviceKey); + store.SetPoolId(poolId); + return store; + } + + public Task CheckVersionAsync(Ct ct) + { + var databaseSchema = serviceKey is null + ? serviceProvider.GetRequiredService() + : serviceProvider.GetRequiredKeyedService(serviceKey); + return databaseSchema.CheckVersionAsync(ct); + } + + public Task MigrateAsync(Ct ct) + { + var databaseSchema = serviceKey is null + ? serviceProvider.GetRequiredService() + : serviceProvider.GetRequiredKeyedService(serviceKey); + return databaseSchema.MigrateAsync(ct); + } + + public Task VerifySchemaAsync(Ct ct) + { + var databaseSchema = serviceKey is null + ? serviceProvider.GetRequiredService() + : serviceProvider.GetRequiredKeyedService(serviceKey); + return databaseSchema.VerifySchemaAsync(ct); + } + + public string BuildMigrationScript(DatabaseSchemaVersion fromVersion) + { + var databaseSchema = serviceKey is null + ? serviceProvider.GetRequiredService() + : serviceProvider.GetRequiredKeyedService(serviceKey); + return databaseSchema.BuildMigrationScript(fromVersion); + } +} diff --git a/storage/src/Storage/Internal/Projection.cs b/storage/src/Storage/Internal/Projection.cs new file mode 100644 index 000000000..dc377ea19 --- /dev/null +++ b/storage/src/Storage/Internal/Projection.cs @@ -0,0 +1,39 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +/// +/// Specifies which attributes to include in query results. +/// When used, the query returns +/// instead of a fully-typed DTO. +/// +public sealed record Projection +{ + /// The attribute names to include. + public IReadOnlyList Attributes { get; } + + /// + /// Creates a projection for the specified attributes. + /// + public Projection(params string[] attributes) + { + ArgumentNullException.ThrowIfNull(attributes); + if (attributes.Length == 0) + { + throw new ArgumentException("At least one attribute must be specified.", nameof(attributes)); + } + + foreach (var attr in attributes) + { + ArgumentException.ThrowIfNullOrEmpty(attr); + } + + Attributes = attributes.ToArray(); + } + + /// + /// Creates a projection for the specified attributes. + /// + public static Projection Of(params string[] attributes) => new(attributes); +} diff --git a/storage/src/Storage/Internal/Querying/CursorToken.cs b/storage/src/Storage/Internal/Querying/CursorToken.cs new file mode 100644 index 000000000..2f5c337f2 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/CursorToken.cs @@ -0,0 +1,96 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Text; +using System.Text.Json; + +namespace Duende.Storage.Internal.Querying; + +/// +/// Represents a decoded cursor token containing the last-seen position for seek-based pagination. +/// +public sealed record CursorToken +{ + /// + /// The ID of the last entity on the previous page. + /// + public Guid Id { get; init; } + + /// + /// The sort field value of the last entity on the previous page (string type). + /// + public string? StringValue { get; init; } + + /// + /// The sort field value of the last entity on the previous page (number type). + /// + public decimal? NumberValue { get; init; } + + /// + /// The sort field value of the last entity on the previous page (datetime type). + /// + public DateTimeOffset? DateTimeValue { get; init; } + + /// + /// The sort field value of the last entity on the previous page (boolean type). + /// + public bool? BooleanValue { get; init; } + + /// + /// The sort field value of the last entity on the previous page (guid type). + /// + public Guid? GuidValue { get; init; } + + /// + /// Encodes the cursor token to an opaque base64 string. + /// + public string Encode() + { + var json = JsonSerializer.Serialize(this); + var bytes = Encoding.UTF8.GetBytes(json); + return Convert.ToBase64String(bytes); + } + + /// + /// Decodes an opaque base64 cursor token string. + /// + /// The encoded token string. + /// The decoded cursor token, or null if the token is invalid. + public static CursorToken? Decode(string token) + { + try + { + var bytes = Convert.FromBase64String(token); + var json = Encoding.UTF8.GetString(bytes); + return JsonSerializer.Deserialize(json); + } + catch (FormatException) + { + return null; + } + catch (JsonException) + { + return null; + } + } + + /// + /// Creates a cursor token from the last item's sort value and ID. + /// + public static CursorToken Create( + Guid id, + string? stringValue, + decimal? numberValue, + DateTimeOffset? dateTimeValue, + bool? booleanValue, + Guid? guidValue) => + new() + { + Id = id, + StringValue = stringValue, + NumberValue = numberValue, + DateTimeValue = dateTimeValue, + BooleanValue = booleanValue, + GuidValue = guidValue + }; +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/AllExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/AllExpression.cs new file mode 100644 index 000000000..541b0dd25 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/AllExpression.cs @@ -0,0 +1,20 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Singleton expression representing 'match all records' - no filter applied. +/// +public sealed record AllExpression : IQueryFilterExpression +{ + /// + /// Singleton instance of AllExpression. + /// + public static readonly AllExpression Instance = new(); + + // Private constructor to enforce singleton pattern + private AllExpression() + { + } +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/AndExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/AndExpression.cs new file mode 100644 index 000000000..f08130050 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/AndExpression.cs @@ -0,0 +1,67 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that combines multiple filter expressions with AND logic. +/// All conditions must be true for the expression to match. +/// +public sealed record AndExpression : IQueryFilterExpression +{ + /// + /// The collection of expressions that must all be true. + /// + public IReadOnlyList Parts { get; } + + public AndExpression(IReadOnlyList parts) + { + ArgumentNullException.ThrowIfNull(parts); + + if (parts.Count == 0) + { + throw new ArgumentException("AndExpression must have at least one part.", nameof(parts)); + } + + Parts = parts; + } + + public AndExpression(params IQueryFilterExpression[] parts) + : this((IReadOnlyList)parts) + { + } + + /// + /// Adds another condition with AND logic. + /// Smart accumulation: if called on an AndExpression, adds to Parts collection instead of nesting. + /// + public IQueryFilterExpression And(IQueryFilterExpression other) + { + ArgumentNullException.ThrowIfNull(other); + + // Smart accumulation: flatten nested AndExpressions + var newParts = new List(Parts); + + if (other is AndExpression andExpression) + { + // Flatten: add all parts from the nested AndExpression + newParts.AddRange(andExpression.Parts); + } + else + { + newParts.Add(other); + } + + return new AndExpression(newParts); + } + + /// + /// Combines this expression with another using OR logic. + /// + public IQueryFilterExpression Or(IQueryFilterExpression other) + { + ArgumentNullException.ThrowIfNull(other); + + return new OrExpression(this, other); + } +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/ArrayContainsExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/ArrayContainsExpression.cs new file mode 100644 index 000000000..586767a43 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/ArrayContainsExpression.cs @@ -0,0 +1,12 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that checks if a string array field contains an element equal to the specified value. +/// Generates an EXISTS subquery with item_index >= 0 to match any array element. +/// +public sealed record ArrayContainsExpression(StringArrayField Field, string Value) : IQueryExpression, IQueryFilterExpression; diff --git a/storage/src/Storage/Internal/Querying/Expressions/ArrayFilterExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/ArrayFilterExpression.cs new file mode 100644 index 000000000..554aad416 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/ArrayFilterExpression.cs @@ -0,0 +1,36 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that filters array items where all conditions must match within the same array element. +/// This correlates conditions within the same array item using the item_index column. +/// +/// Example: emails[type eq "work" and value co "@example.com"] +/// This ensures both conditions match the same email in the emails array. +/// +public sealed record ArrayFilterExpression : IQueryFilterExpression +{ + /// + /// The array field path (e.g., "emails"). + /// + public string ArrayFieldPath { get; } + + /// + /// The filter expression that applies to fields within the array. + /// Field paths in this expression are relative to the array (e.g., "type", "value"). + /// + public IQueryFilterExpression Filter { get; } + + public ArrayFilterExpression(string arrayFieldPath, IQueryFilterExpression filter) + { + if (string.IsNullOrWhiteSpace(arrayFieldPath)) + { + throw new ArgumentException("Array field path cannot be null or whitespace.", nameof(arrayFieldPath)); + } + + ArrayFieldPath = arrayFieldPath.ToUpperInvariant(); + Filter = filter ?? throw new ArgumentNullException(nameof(filter)); + } +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/BetweenExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/BetweenExpression.cs new file mode 100644 index 000000000..b417b1d8e --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/BetweenExpression.cs @@ -0,0 +1,28 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that checks if a field value is between two values (inclusive). +/// +public sealed record BetweenExpression : IQueryFilterExpression +{ + public Field Field { get; } + public object Min { get; } + public object Max { get; } + + public BetweenExpression(Field field, object min, object max) + { + Field = field ?? throw new ArgumentNullException(nameof(field)); + Min = min ?? throw new ArgumentNullException(nameof(min)); + Max = max ?? throw new ArgumentNullException(nameof(max)); + + if (field.Type == FieldType.String) + { + throw new ArgumentException("Between expression cannot be used with string fields.", nameof(field)); + } + } +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/ContainsExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/ContainsExpression.cs new file mode 100644 index 000000000..35a253674 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/ContainsExpression.cs @@ -0,0 +1,21 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that checks if a string field contains a specified substring. +/// +public sealed record ContainsExpression : IQueryFilterExpression +{ + public StringField Field { get; } + public string Value { get; } + + public ContainsExpression(StringField field, string value) + { + Field = field ?? throw new ArgumentNullException(nameof(field)); + Value = value ?? throw new ArgumentNullException(nameof(value)); + } +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/EndsWithExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/EndsWithExpression.cs new file mode 100644 index 000000000..6711f413f --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/EndsWithExpression.cs @@ -0,0 +1,22 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that checks if a string field ends with a specified suffix. +/// Used for the SCIM 'ew' (ends with) operator. +/// +public sealed record EndsWithExpression : IQueryFilterExpression +{ + public StringField Field { get; } + public string Value { get; } + + public EndsWithExpression(StringField field, string value) + { + Field = field ?? throw new ArgumentNullException(nameof(field)); + Value = value ?? throw new ArgumentNullException(nameof(value)); + } +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/EqualExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/EqualExpression.cs new file mode 100644 index 000000000..8158985b2 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/EqualExpression.cs @@ -0,0 +1,21 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that checks if a field equals a specified value. +/// +public sealed record EqualExpression : IQueryFilterExpression +{ + public Field Field { get; } + public object Value { get; } + + public EqualExpression(Field field, object value) + { + Field = field ?? throw new ArgumentNullException(nameof(field)); + Value = value ?? throw new ArgumentNullException(nameof(value)); + } +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/GreaterOrEqualExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/GreaterOrEqualExpression.cs new file mode 100644 index 000000000..36aec43a3 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/GreaterOrEqualExpression.cs @@ -0,0 +1,26 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that checks if a field is greater than or equal to a specified value. +/// +public sealed record GreaterOrEqualExpression : IQueryFilterExpression +{ + public Field Field { get; } + public object Value { get; } + + public GreaterOrEqualExpression(Field field, object value) + { + Field = field ?? throw new ArgumentNullException(nameof(field)); + Value = value ?? throw new ArgumentNullException(nameof(value)); + + if (field.Type == FieldType.String) + { + throw new ArgumentException("GreaterOrEqual expression cannot be used with string fields.", nameof(field)); + } + } +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/GreaterThanExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/GreaterThanExpression.cs new file mode 100644 index 000000000..2cdf2f3e2 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/GreaterThanExpression.cs @@ -0,0 +1,26 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that checks if a field is greater than a specified value. +/// +public sealed record GreaterThanExpression : IQueryFilterExpression +{ + public Field Field { get; } + public object Value { get; } + + public GreaterThanExpression(Field field, object value) + { + Field = field ?? throw new ArgumentNullException(nameof(field)); + Value = value ?? throw new ArgumentNullException(nameof(value)); + + if (field.Type == FieldType.String) + { + throw new ArgumentException("GreaterThan expression cannot be used with string fields.", nameof(field)); + } + } +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/InExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/InExpression.cs new file mode 100644 index 000000000..db5a5e09d --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/InExpression.cs @@ -0,0 +1,22 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Collections; +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that checks if a field value is in a specified collection. +/// +public sealed record InExpression : IQueryFilterExpression +{ + public Field Field { get; } + public IEnumerable Values { get; } + + public InExpression(Field field, IEnumerable values) + { + Field = field ?? throw new ArgumentNullException(nameof(field)); + Values = values ?? throw new ArgumentNullException(nameof(values)); + } +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/LessOrEqualExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/LessOrEqualExpression.cs new file mode 100644 index 000000000..b540a70bf --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/LessOrEqualExpression.cs @@ -0,0 +1,26 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that checks if a field is less than or equal to a specified value. +/// +public sealed record LessOrEqualExpression : IQueryFilterExpression +{ + public Field Field { get; } + public object Value { get; } + + public LessOrEqualExpression(Field field, object value) + { + Field = field ?? throw new ArgumentNullException(nameof(field)); + Value = value ?? throw new ArgumentNullException(nameof(value)); + + if (field.Type == FieldType.String) + { + throw new ArgumentException("LessOrEqual expression cannot be used with string fields.", nameof(field)); + } + } +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/LessThanExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/LessThanExpression.cs new file mode 100644 index 000000000..28d266330 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/LessThanExpression.cs @@ -0,0 +1,26 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that checks if a field is less than a specified value. +/// +public sealed record LessThanExpression : IQueryFilterExpression +{ + public Field Field { get; } + public object Value { get; } + + public LessThanExpression(Field field, object value) + { + Field = field ?? throw new ArgumentNullException(nameof(field)); + Value = value ?? throw new ArgumentNullException(nameof(value)); + + if (field.Type == FieldType.String) + { + throw new ArgumentException("LessThan expression cannot be used with string fields.", nameof(field)); + } + } +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/NotExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/NotExpression.cs new file mode 100644 index 000000000..b750135fa --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/NotExpression.cs @@ -0,0 +1,19 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that negates an inner filter expression. +/// Used for SCIM 'not' operator and 'ne' (as Not(Equal(...))). +/// +public sealed record NotExpression : IQueryFilterExpression +{ + /// + /// The inner expression to negate. + /// + public IQueryFilterExpression Inner { get; } + + public NotExpression(IQueryFilterExpression inner) => + Inner = inner ?? throw new ArgumentNullException(nameof(inner)); +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/OrExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/OrExpression.cs new file mode 100644 index 000000000..6136ae3f5 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/OrExpression.cs @@ -0,0 +1,67 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that combines multiple filter expressions with OR logic. +/// At least one condition must be true for the expression to match. +/// +public sealed record OrExpression : IQueryFilterExpression +{ + /// + /// The collection of expressions where at least one must be true. + /// + public IReadOnlyList Parts { get; } + + public OrExpression(IReadOnlyList parts) + { + ArgumentNullException.ThrowIfNull(parts); + + if (parts.Count == 0) + { + throw new ArgumentException("OrExpression must have at least one part.", nameof(parts)); + } + + Parts = parts; + } + + public OrExpression(params IQueryFilterExpression[] parts) + : this((IReadOnlyList)parts) + { + } + + /// + /// Combines this expression with another using AND logic. + /// + public IQueryFilterExpression And(IQueryFilterExpression other) + { + ArgumentNullException.ThrowIfNull(other); + + return new AndExpression(this, other); + } + + /// + /// Adds another condition with OR logic. + /// Smart accumulation: if called on an OrExpression, adds to Parts collection instead of nesting. + /// + public IQueryFilterExpression Or(IQueryFilterExpression other) + { + ArgumentNullException.ThrowIfNull(other); + + // Smart accumulation: flatten nested OrExpressions + var newParts = new List(Parts); + + if (other is OrExpression orExpression) + { + // Flatten: add all parts from the nested OrExpression + newParts.AddRange(orExpression.Parts); + } + else + { + newParts.Add(other); + } + + return new OrExpression(newParts); + } +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/PresentExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/PresentExpression.cs new file mode 100644 index 000000000..6f27c3cfa --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/PresentExpression.cs @@ -0,0 +1,20 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that checks if a field has a value (is present / not null). +/// Used for the SCIM 'pr' (present) operator. +/// For scalar fields, checks that a search_values row exists with a non-null typed column. +/// For array fields, checks that at least one row with item_index >= 0 exists. +/// +public sealed record PresentExpression : IQueryFilterExpression +{ + public Field Field { get; } + + public PresentExpression(Field field) => + Field = field ?? throw new ArgumentNullException(nameof(field)); +} diff --git a/storage/src/Storage/Internal/Querying/Expressions/StartsWithExpression.cs b/storage/src/Storage/Internal/Querying/Expressions/StartsWithExpression.cs new file mode 100644 index 000000000..e7626191f --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Expressions/StartsWithExpression.cs @@ -0,0 +1,21 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Querying.Expressions; + +/// +/// Expression that checks if a string field starts with a specified value. +/// +public sealed record StartsWithExpression : IQueryFilterExpression +{ + public StringField Field { get; } + public string Value { get; } + + public StartsWithExpression(StringField field, string value) + { + Field = field ?? throw new ArgumentNullException(nameof(field)); + Value = value ?? throw new ArgumentNullException(nameof(value)); + } +} diff --git a/storage/src/Storage/Internal/Querying/Fields/BooleanField.cs b/storage/src/Storage/Internal/Querying/Fields/BooleanField.cs new file mode 100644 index 000000000..d6671ffd6 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Fields/BooleanField.cs @@ -0,0 +1,31 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Expressions; + +namespace Duende.Storage.Internal.Querying.Fields; + +/// +/// Represents a boolean-valued field with boolean-specific operations. +/// +public sealed record BooleanField : Field +{ + public BooleanField(string path, bool isMultiValued = false) : base(path, FieldType.Boolean, isMultiValued) + { + } + + /// + /// Creates an expression that checks if the field equals the specified value. + /// + public IQueryFilterExpression Equals(bool value) => new EqualExpression(this, value); + + /// + /// Creates an expression that checks if the field is true. + /// + public IQueryFilterExpression IsTrue() => new EqualExpression(this, true); + + /// + /// Creates an expression that checks if the field is false. + /// + public IQueryFilterExpression IsFalse() => new EqualExpression(this, false); +} diff --git a/storage/src/Storage/Internal/Querying/Fields/DateTimeField.cs b/storage/src/Storage/Internal/Querying/Fields/DateTimeField.cs new file mode 100644 index 000000000..76490ce99 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Fields/DateTimeField.cs @@ -0,0 +1,54 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Expressions; + +namespace Duende.Storage.Internal.Querying.Fields; + +/// +/// Represents a DateTimeOffset-valued field with temporal comparison operations. +/// +public sealed record DateTimeField : Field +{ + public DateTimeField(string path, bool isMultiValued = false) : base(path, FieldType.DateTime, isMultiValued) + { + } + + /// + /// Creates an expression that checks if the field equals the specified value. + /// + public IQueryFilterExpression Equals(DateTimeOffset value) => new EqualExpression(this, value); + + /// + /// Creates an expression that checks if the field is greater than the specified value. + /// + public IQueryFilterExpression GreaterThan(DateTimeOffset value) => new GreaterThanExpression(this, value); + + /// + /// Creates an expression that checks if the field is less than the specified value. + /// + public IQueryFilterExpression LessThan(DateTimeOffset value) => new LessThanExpression(this, value); + + /// + /// Creates an expression that checks if the field is greater than or equal to the specified value. + /// + public IQueryFilterExpression GreaterOrEqual(DateTimeOffset value) => new GreaterOrEqualExpression(this, value); + + /// + /// Creates an expression that checks if the field is less than or equal to the specified value. + /// + public IQueryFilterExpression LessOrEqual(DateTimeOffset value) => new LessOrEqualExpression(this, value); + + /// + /// Creates an expression that checks if the field value is between the specified range (inclusive). + /// + public IQueryFilterExpression Between(DateTimeOffset min, DateTimeOffset max) + { + if (min > max) + { + throw new ArgumentOutOfRangeException(nameof(min), min, "Minimum value must not exceed maximum value."); + } + + return new BetweenExpression(this, min, max); + } +} diff --git a/storage/src/Storage/Internal/Querying/Fields/ExactMatchField.cs b/storage/src/Storage/Internal/Querying/Fields/ExactMatchField.cs new file mode 100644 index 000000000..e0d43486e --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Fields/ExactMatchField.cs @@ -0,0 +1,31 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Expressions; + +namespace Duende.Storage.Internal.Querying.Fields; + +/// +/// Represents a field that stores a deterministic GUID hash of a string value. +/// Only supports exact-match operations (Equals and In). +/// Queries use the guid_value column with hashed values for fast lookups. +/// No string_value is stored — only the deterministic hash in guid_value. +/// +public sealed record ExactMatchField : Field +{ + public ExactMatchField(string path) : base(path, FieldType.Guid) + { + } + + /// + /// Creates an expression that checks if the field equals the deterministic hash of the specified value. + /// + public IQueryFilterExpression Equals(string value) => + new EqualExpression(this, DeterministicGuidGenerator.Create(value.ToUpperInvariant())); + + /// + /// Creates an expression that checks if the field value hash is in the specified collection. + /// + public IQueryFilterExpression In(IReadOnlyCollection values) => + new InExpression(this, values.Select(v => DeterministicGuidGenerator.Create(v.ToUpperInvariant())).ToList()); +} diff --git a/storage/src/Storage/Internal/Querying/Fields/Field.cs b/storage/src/Storage/Internal/Querying/Fields/Field.cs new file mode 100644 index 000000000..edd624a43 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Fields/Field.cs @@ -0,0 +1,49 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Expressions; + +namespace Duende.Storage.Internal.Querying.Fields; + +/// +/// Base class for all field types, representing a queryable field path. +/// +public abstract record Field +{ + /// + /// The field path (e.g., "userName", "emails.value", "consoleProperties.id"). + /// + public string Path { get; } + + /// + /// The type of the field, indicating which typed column to read from (string_value, number_value, datetime_value, or boolean_value). + /// This ensures QueryFields reads from the correct column instead of checking the first non-null value. + /// + public FieldType Type { get; } + + /// + /// Indicates whether this field is multi-valued (i.e., stored with item_index >= 0). + /// When true, queries omit the item_index = -1 condition so any array element can match. + /// When false, queries include item_index = -1 to target scalar values only. + /// + public bool IsMultiValued { get; } + + protected Field(string path, FieldType type, bool isMultiValued = false) + { + if (string.IsNullOrWhiteSpace(path)) + { + throw new ArgumentException("Field path cannot be null or whitespace.", nameof(path)); + } + + Path = path.ToUpperInvariant(); + Type = type; + IsMultiValued = isMultiValued; + } + + /// + /// Creates an expression that checks if this field has a value (is present). + /// For scalar fields, checks that a non-null value exists. + /// For multi-valued fields, checks that at least one array element exists. + /// + public IQueryFilterExpression Present() => new PresentExpression(this); +} diff --git a/storage/src/Storage/Internal/Querying/Fields/GuidField.cs b/storage/src/Storage/Internal/Querying/Fields/GuidField.cs new file mode 100644 index 000000000..6932e0c34 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Fields/GuidField.cs @@ -0,0 +1,26 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Expressions; + +namespace Duende.Storage.Internal.Querying.Fields; + +/// +/// Represents a Guid-valued field stored in the guid_value column. +/// +public sealed record GuidField : Field +{ + public GuidField(string path, bool isMultiValued = false) : base(path, FieldType.Guid, isMultiValued) + { + } + + /// + /// Creates an expression that checks if the field equals the specified GUID value. + /// + public IQueryFilterExpression Equals(Guid value) => new EqualExpression(this, value); + + /// + /// Creates an expression that checks if the field value is in the specified collection. + /// + public IQueryFilterExpression In(IReadOnlyCollection values) => new InExpression(this, values.ToList()); +} diff --git a/storage/src/Storage/Internal/Querying/Fields/NumberField.cs b/storage/src/Storage/Internal/Querying/Fields/NumberField.cs new file mode 100644 index 000000000..c1e5fe2fc --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Fields/NumberField.cs @@ -0,0 +1,59 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Expressions; + +namespace Duende.Storage.Internal.Querying.Fields; + +/// +/// Represents a number-valued field with numeric comparison operations. +/// +public sealed record NumberField : Field +{ + public NumberField(string path, bool isMultiValued = false) : base(path, FieldType.Number, isMultiValued) + { + } + + /// + /// Creates an expression that checks if the field equals the specified value. + /// + public IQueryFilterExpression Equals(decimal value) => new EqualExpression(this, value); + + /// + /// Creates an expression that checks if the field is greater than the specified value. + /// + public IQueryFilterExpression GreaterThan(decimal value) => new GreaterThanExpression(this, value); + + /// + /// Creates an expression that checks if the field is less than the specified value. + /// + public IQueryFilterExpression LessThan(decimal value) => new LessThanExpression(this, value); + + /// + /// Creates an expression that checks if the field is greater than or equal to the specified value. + /// + public IQueryFilterExpression GreaterOrEqual(decimal value) => new GreaterOrEqualExpression(this, value); + + /// + /// Creates an expression that checks if the field is less than or equal to the specified value. + /// + public IQueryFilterExpression LessOrEqual(decimal value) => new LessOrEqualExpression(this, value); + + /// + /// Creates an expression that checks if the field value is between the specified range (inclusive). + /// + public IQueryFilterExpression Between(decimal min, decimal max) + { + if (min > max) + { + throw new ArgumentOutOfRangeException(nameof(min), min, "Minimum value must not exceed maximum value."); + } + + return new BetweenExpression(this, min, max); + } + + /// + /// Creates an expression that checks if the field value is in the specified collection. + /// + public IQueryFilterExpression In(IReadOnlyCollection values) => new InExpression(this, values); +} diff --git a/storage/src/Storage/Internal/Querying/Fields/StringArrayField.cs b/storage/src/Storage/Internal/Querying/Fields/StringArrayField.cs new file mode 100644 index 000000000..1306f25da --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Fields/StringArrayField.cs @@ -0,0 +1,24 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Expressions; + +namespace Duende.Storage.Internal.Querying.Fields; + +/// +/// Represents a string array field (multi-valued string). +/// This is a convenience subclass of with set to true. +/// +public sealed record StringArrayField : Field +{ + public StringArrayField(string path) : base(path, FieldType.String, isMultiValued: true) + { + } + + /// + /// Creates an expression that checks if the array contains an element equal to the specified value. + /// Uses the guid_value column via a deterministic hash for faster exact-match lookup. + /// + public IQueryFilterExpression Contains(string value) => + new EqualExpression(this, DeterministicGuidGenerator.Create(value.ToUpperInvariant())); +} diff --git a/storage/src/Storage/Internal/Querying/Fields/StringField.cs b/storage/src/Storage/Internal/Querying/Fields/StringField.cs new file mode 100644 index 000000000..b4e7d9669 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Fields/StringField.cs @@ -0,0 +1,45 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Expressions; + +namespace Duende.Storage.Internal.Querying.Fields; + +/// +/// Represents a string-valued field with string-specific operations. +/// +public sealed record StringField : Field +{ + public StringField(string path, bool isMultiValued = false) : base(path, FieldType.String, isMultiValued) + { + } + + /// + /// Creates an expression that checks if the field equals the specified value. + /// Uses the guid_value column via a deterministic hash for faster exact-match lookup. + /// + public IQueryFilterExpression Equals(string value) => + new EqualExpression(this, DeterministicGuidGenerator.Create(value.ToUpperInvariant())); + + /// + /// Creates an expression that checks if the field contains the specified substring. + /// + public IQueryFilterExpression Contains(string value) => new ContainsExpression(this, value.ToUpperInvariant()); + + /// + /// Creates an expression that checks if the field starts with the specified value. + /// + public IQueryFilterExpression StartsWith(string value) => new StartsWithExpression(this, value.ToUpperInvariant()); + + /// + /// Creates an expression that checks if the field ends with the specified suffix. + /// + public IQueryFilterExpression EndsWith(string value) => new EndsWithExpression(this, value.ToUpperInvariant()); + + /// + /// Creates an expression that checks if the field value is in the specified collection. + /// Uses the guid_value column via deterministic hashes for faster exact-match lookup. + /// + public IQueryFilterExpression In(IReadOnlyCollection values) => + new InExpression(this, values.Select(v => DeterministicGuidGenerator.Create(v.ToUpperInvariant())).ToList()); +} diff --git a/storage/src/Storage/Internal/Querying/IQueryExpression.cs b/storage/src/Storage/Internal/Querying/IQueryExpression.cs new file mode 100644 index 000000000..0bf1b8cf2 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/IQueryExpression.cs @@ -0,0 +1,11 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Querying; + +/// +/// Base marker interface for all query expressions. +/// +public interface IQueryExpression +{ +} diff --git a/storage/src/Storage/Internal/Querying/IQueryExpressionVisitor.cs b/storage/src/Storage/Internal/Querying/IQueryExpressionVisitor.cs new file mode 100644 index 000000000..5393443b6 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/IQueryExpressionVisitor.cs @@ -0,0 +1,99 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Expressions; + +namespace Duende.Storage.Internal.Querying; + +/// +/// Visitor interface for processing query expression trees. +/// Implementations can translate expressions to different formats (e.g., SQL, in-memory evaluation). +/// +/// The type returned by visiting an expression. +public interface IQueryExpressionVisitor +{ + /// + /// Visits an AllExpression that matches all records. + /// + TResult Visit(AllExpression expression); + + /// + /// Visits an EqualExpression that checks field equality. + /// + TResult Visit(EqualExpression expression); + + /// + /// Visits a ContainsExpression that checks if a string field contains a substring. + /// + TResult Visit(ContainsExpression expression); + + /// + /// Visits a StartsWithExpression that checks if a string field starts with a prefix. + /// + TResult Visit(StartsWithExpression expression); + + /// + /// Visits a GreaterThanExpression that checks if a field is greater than a value. + /// + TResult Visit(GreaterThanExpression expression); + + /// + /// Visits a LessThanExpression that checks if a field is less than a value. + /// + TResult Visit(LessThanExpression expression); + + /// + /// Visits a GreaterOrEqualExpression that checks if a field is greater than or equal to a value. + /// + TResult Visit(GreaterOrEqualExpression expression); + + /// + /// Visits a LessOrEqualExpression that checks if a field is less than or equal to a value. + /// + TResult Visit(LessOrEqualExpression expression); + + /// + /// Visits a BetweenExpression that checks if a field is between two values (inclusive). + /// + TResult Visit(BetweenExpression expression); + + /// + /// Visits an InExpression that checks if a field value is in a collection. + /// + TResult Visit(InExpression expression); + + /// + /// Visits an AndExpression that combines multiple expressions with AND logic. + /// + TResult Visit(AndExpression expression); + + /// + /// Visits an OrExpression that combines multiple expressions with OR logic. + /// + TResult Visit(OrExpression expression); + + /// + /// Visits an ArrayFilterExpression that filters array items where conditions match within the same element. + /// + TResult Visit(ArrayFilterExpression expression); + + /// + /// Visits a NotExpression that negates an inner expression. + /// + TResult Visit(NotExpression expression); + + /// + /// Visits an EndsWithExpression that checks if a string field ends with a suffix. + /// + TResult Visit(EndsWithExpression expression); + + /// + /// Visits a PresentExpression that checks if a field has a value. + /// + TResult Visit(PresentExpression expression); + + /// + /// Visits an ArrayContainsExpression that checks if a string array contains a specific value. + /// + TResult Visit(ArrayContainsExpression expression); +} diff --git a/storage/src/Storage/Internal/Querying/IQueryFilterExpression.cs b/storage/src/Storage/Internal/Querying/IQueryFilterExpression.cs new file mode 100644 index 000000000..4bf278a13 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/IQueryFilterExpression.cs @@ -0,0 +1,11 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Querying; + +/// +/// Marker interface for filter expressions that can be used in WHERE clauses. +/// +public interface IQueryFilterExpression : IQueryExpression +{ +} diff --git a/storage/src/Storage/Internal/Querying/ISqlDialect.cs b/storage/src/Storage/Internal/Querying/ISqlDialect.cs new file mode 100644 index 000000000..7cee0e584 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/ISqlDialect.cs @@ -0,0 +1,55 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Data.Common; + +namespace Duende.Storage.Internal.Querying; + +/// +/// Defines SQL dialect-specific behavior for query building. +/// +public interface ISqlDialect +{ + /// + /// The LIKE operator for case-insensitive matching. + /// PostgreSQL: "ILIKE", MSSQL: "LIKE" (relies on collation) + /// + string CaseInsensitiveLikeOperator { get; } + + /// + /// Escapes wildcard characters (%, _) in LIKE patterns according to the dialect. + /// PostgreSQL uses backslash escaping (\%), MSSQL uses bracket escaping ([%]). + /// + string EscapeLikeWildcards(string value); + + /// + /// Gets the SQL ESCAPE clause suffix to append after LIKE expressions. + /// SQLite requires an explicit ESCAPE clause (e.g. " ESCAPE '\'"), while PostgreSQL + /// and SQL Server handle escaping natively and return an empty string. + /// + string LikeEscapeClause => ""; + + /// + /// Adds a parameter to the command, handling type-specific conversions. + /// Implementations should handle DateTimeOffset, DateTime, and other types appropriately. + /// + void AddParameter(DbCommand command, string name, object value); + + /// + /// Converts a field path GUID to the appropriate parameter value for the dialect. + /// SQLite stores field paths as BLOBs (byte arrays), while other dialects use the native GUID type. + /// + object FieldPathToParameterValue(Guid fieldPathId) => fieldPathId; + + /// + /// Gets the SQL literal for TRUE. + /// PostgreSQL: "TRUE", MSSQL: "1=1" + /// + string TrueLiteral { get; } + + /// + /// Gets the SQL literal for FALSE. + /// PostgreSQL: "FALSE", MSSQL: "1=0" + /// + string FalseLiteral { get; } +} diff --git a/storage/src/Storage/Internal/Querying/LinkQuery.cs b/storage/src/Storage/Internal/Querying/LinkQuery.cs new file mode 100644 index 000000000..23c1d9089 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/LinkQuery.cs @@ -0,0 +1,23 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Querying; + +/// +/// Fluent entry point for building link traversal queries. +/// +/// +/// +/// var query = LinkQuery.From(userEntityType) +/// .Join(UserRoleDefinition) +/// .Where(roleEntityType, roleId) +/// .Build(); +/// +/// +public static class LinkQuery +{ + /// + /// Starts a new link query from the given source entity type. + /// + public static LinkQueryBuilder From(EntityType source) => new(source); +} diff --git a/storage/src/Storage/Internal/Querying/LinkQueryBuilder.cs b/storage/src/Storage/Internal/Querying/LinkQueryBuilder.cs new file mode 100644 index 000000000..bcae2c712 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/LinkQueryBuilder.cs @@ -0,0 +1,93 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Querying; + +/// +/// Builds a using a fluent API. +/// +public sealed class LinkQueryBuilder +{ + private readonly EntityType _source; + private readonly List _joins = []; + private EntityType? _currentEndpoint; + private EntityType? _whereEntityType; + private UuidV7? _whereEntityId; + private bool _whereSet; + + internal LinkQueryBuilder(EntityType source) + { + _source = source; + _currentEndpoint = source; + } + + /// + /// Adds a link traversal hop. The definition must connect to the current + /// chain endpoint — either definition.Left or definition.Right + /// must match. Direction is determined automatically. + /// + public LinkQueryBuilder Join(LinkDefinition definition) + { + ArgumentNullException.ThrowIfNull(definition); + + LinkJoinDirection direction; + if (definition.Left == _currentEndpoint) + { + direction = LinkJoinDirection.LeftToRight; + _currentEndpoint = definition.Right; + } + else if (definition.Right == _currentEndpoint) + { + direction = LinkJoinDirection.RightToLeft; + _currentEndpoint = definition.Left; + } + else + { + throw new InvalidOperationException( + $"LinkDefinition '{definition.Link.Name}' does not connect to the current chain endpoint '{_currentEndpoint?.Name}'. " + + $"Expected Left='{definition.Left.Name}' or Right='{definition.Right.Name}' to match."); + } + + _joins.Add(new LinkQueryJoin(definition, direction)); + return this; + } + + /// + /// Adds a filter: only return source entities reachable from this specific entity. + /// Can only be called once. The entity type must be the terminal type of the chain + /// (the endpoint after all joins). + /// + public LinkQueryBuilder Where(EntityType entityType, UuidV7 entityId) + { + if (_whereSet) + { + throw new InvalidOperationException("Where has already been set. Only one Where filter is allowed per query."); + } + + if (entityType != _currentEndpoint) + { + throw new InvalidOperationException( + $"Entity type '{entityType.Name}' is not the terminal type of the link query chain. " + + $"Where can only filter on the terminal type '{_currentEndpoint?.Name}'."); + } + + _whereEntityType = entityType; + _whereEntityId = entityId; + _whereSet = true; + return this; + } + + /// + /// Builds the . Requires at least one Join. + /// + public LinkQueryDescriptor Build() + { + if (_joins.Count == 0) + { + throw new InvalidOperationException("At least one Join is required to build a link query."); + } + + return new LinkQueryDescriptor(_source, _joins.AsReadOnly(), _whereEntityType, _whereEntityId); + } + +} diff --git a/storage/src/Storage/Internal/Querying/LinkQueryDescriptor.cs b/storage/src/Storage/Internal/Querying/LinkQueryDescriptor.cs new file mode 100644 index 000000000..6df7748af --- /dev/null +++ b/storage/src/Storage/Internal/Querying/LinkQueryDescriptor.cs @@ -0,0 +1,34 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Querying; + +/// +/// Describes a link traversal query — built by LinkQueryBuilder and +/// consumed by IQueryStore.QueryLinks. +/// +public sealed record LinkQueryDescriptor( + EntityType SourceEntityType, + IReadOnlyList Joins, + EntityType? WhereEntityType, + UuidV7? WhereEntityId); + +/// +/// A single hop in a link query chain, pairing a +/// with the direction it is traversed. +/// +public sealed record LinkQueryJoin( + LinkDefinition Definition, + LinkJoinDirection Direction); + +/// +/// Direction of a link traversal join. +/// +public enum LinkJoinDirection +{ + /// Traverse from the left entity to the right entity. + LeftToRight, + + /// Traverse from the right entity to the left entity. + RightToLeft +} diff --git a/storage/src/Storage/Internal/Querying/MetadataEnvelope.cs b/storage/src/Storage/Internal/Querying/MetadataEnvelope.cs new file mode 100644 index 000000000..36e8d4665 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/MetadataEnvelope.cs @@ -0,0 +1,15 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Querying; + +/// +/// Wraps a query result item with entity metadata: id, version, and timestamps. +/// +/// The type of the wrapped value. +public sealed record MetadataEnvelope( + TValue Value, + Guid Id, + int Version, + DateTimeOffset CreatedAt, + DateTimeOffset LastUpdatedAt); diff --git a/storage/src/Storage/Internal/Querying/ProjectedResult.cs b/storage/src/Storage/Internal/Querying/ProjectedResult.cs new file mode 100644 index 000000000..b9fa5dbaf --- /dev/null +++ b/storage/src/Storage/Internal/Querying/ProjectedResult.cs @@ -0,0 +1,32 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Querying; + +/// +/// Represents a query result with only selected field values rather than the full entity. +/// +public sealed record ProjectedResult +{ + /// + /// The ID of the entity. + /// + public Guid Id { get; } + + /// + /// The projected field values, keyed by field path. + /// + public IReadOnlyDictionary Fields { get; } + + /// + /// Creates a new projected result. + /// + /// The entity ID. + /// The projected field values. + public ProjectedResult(Guid id, IReadOnlyDictionary fields) + { + ArgumentNullException.ThrowIfNull(fields); + Id = id; + Fields = fields; + } +} diff --git a/storage/src/Storage/Internal/Querying/Query.cs b/storage/src/Storage/Internal/Querying/Query.cs new file mode 100644 index 000000000..6730ee407 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Query.cs @@ -0,0 +1,47 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Expressions; + +namespace Duende.Storage.Internal.Querying; + +/// +/// Static entry point for building query expressions. +/// Provides a fluent API for constructing query filter expressions. +/// +public static class Query +{ + /// + /// Creates a query expression that matches all records (no filter). + /// + public static IQueryExpression All() => AllExpression.Instance; + + /// + /// Creates a query expression starting with the specified filter. + /// + public static IQueryFilterExpression Where(IQueryFilterExpression filter) + { + ArgumentNullException.ThrowIfNull(filter); + + return filter; + } + + /// + /// Creates an array filter expression that correlates conditions within the same array item. + /// This is used for SCIM2-style array filters like: emails[type eq "work" and value co "@example.com"] + /// + /// The array field path (e.g., "emails"). + /// The filter expression that applies to fields within the array. + public static IQueryFilterExpression ArrayFilter(string arrayFieldPath, IQueryFilterExpression filter) => + new ArrayFilterExpression(arrayFieldPath, filter); + + /// + /// Negates the specified filter expression. + /// Used for SCIM 'not' operator and 'ne' (as Not(field.Equals("x"))). + /// + public static IQueryFilterExpression Not(IQueryFilterExpression expression) + { + ArgumentNullException.ThrowIfNull(expression); + return new NotExpression(expression); + } +} diff --git a/storage/src/Storage/Internal/Querying/QueryFilterExpressionExtensions.cs b/storage/src/Storage/Internal/Querying/QueryFilterExpressionExtensions.cs new file mode 100644 index 000000000..5bec1f5cb --- /dev/null +++ b/storage/src/Storage/Internal/Querying/QueryFilterExpressionExtensions.cs @@ -0,0 +1,65 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Expressions; + +namespace Duende.Storage.Internal.Querying; + +/// +/// Extension methods for fluent composition of query filter expressions. +/// +public static class QueryFilterExpressionExtensions +{ + /// + /// Combines this expression with another using AND logic. + /// Smart accumulation: if called on an AndExpression, adds to Parts collection instead of nesting. + /// + public static IQueryFilterExpression And(this IQueryFilterExpression left, IQueryFilterExpression right) + { + ArgumentNullException.ThrowIfNull(left); + ArgumentNullException.ThrowIfNull(right); + + // Smart accumulation for AndExpression + if (left is AndExpression andExpression) + { + return andExpression.And(right); + } + + // Smart accumulation: flatten if right is also AndExpression + if (right is AndExpression rightAnd) + { + var parts = new List { left }; + parts.AddRange(rightAnd.Parts); + return new AndExpression(parts); + } + + return new AndExpression(left, right); + } + + /// + /// Combines this expression with another using OR logic. + /// Smart accumulation: if called on an OrExpression, adds to Parts collection instead of nesting. + /// + public static IQueryFilterExpression Or(this IQueryFilterExpression left, IQueryFilterExpression right) + { + ArgumentNullException.ThrowIfNull(left); + ArgumentNullException.ThrowIfNull(right); + + // Smart accumulation for OrExpression + if (left is OrExpression orExpression) + { + return orExpression.Or(right); + } + + // Smart accumulation: flatten if right is also OrExpression + if (right is OrExpression rightOr) + { + var parts = new List { left }; + parts.AddRange(rightOr.Parts); + return new OrExpression(parts); + } + + return new OrExpression(left, right); + } + +} diff --git a/storage/src/Storage/Internal/Querying/SearchFields/SearchFieldCollection.cs b/storage/src/Storage/Internal/Querying/SearchFields/SearchFieldCollection.cs new file mode 100644 index 000000000..9cdb28669 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/SearchFields/SearchFieldCollection.cs @@ -0,0 +1,34 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Collections; +using System.Runtime.CompilerServices; + +namespace Duende.Storage.Internal.Querying.SearchFields; + +/// +/// Immutable collection of search field values that can be passed to IStore.Create/Update methods. +/// Use to construct instances. +/// +[CollectionBuilder(typeof(SearchFieldCollection), nameof(Create))] +public sealed class SearchFieldCollection(IReadOnlyList values) : IReadOnlyCollection +{ + /// + /// Gets the number of search field values in this collection. + /// + public int Count => values.Count; + + /// + /// Returns an enumerator that iterates through the search field values. + /// + public IEnumerator GetEnumerator() => values.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); + + /// + /// Creates an empty SearchFields collection. + /// + public static SearchFieldCollection Empty { get; } = new(Array.Empty()); + + public static SearchFieldCollection Create(ReadOnlySpan values) => new(values.ToArray()); +} diff --git a/storage/src/Storage/Internal/Querying/SearchFields/SearchFieldValue.cs b/storage/src/Storage/Internal/Querying/SearchFields/SearchFieldValue.cs new file mode 100644 index 000000000..cbe17ca06 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/SearchFields/SearchFieldValue.cs @@ -0,0 +1,243 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Querying.SearchFields; + +/// +/// Represents a single search field value that can be stored and queried. +/// This is a pure data structure for Entity-Attribute-Value (EAV) storage pattern. +/// +/// +/// For string fields, both StringValue and GuidValue are set — StringValue holds the raw (uppercased) +/// string for LIKE queries, and GuidValue holds a deterministic hash for fast exact-match queries. +/// For GuidField and ExactMatchField, only GuidValue is set. +/// For Number, DateTime, and Boolean fields, only the respective typed value is set. +/// +/// The ItemIndex is used to correlate fields within the same array item. For example: +/// - FieldPath = "emails.type", ItemIndex = 0, StringValue = "work" +/// - FieldPath = "emails.value", ItemIndex = 0, StringValue = "bob@work.com" +/// +/// For scalar (non-array) fields, ItemIndex should be null. +/// +public sealed record SearchFieldValue +{ + /// + /// The path to the field (e.g., "userName", "consoleProperties.id", "emails.type"). + /// Supports dotted paths for nested properties. + /// + public string FieldPath { get; } + + /// + /// A deterministic GUID computed from (uppercased). + /// Used to identify the field path column in storage without storing the raw string. + /// + public Guid FieldPathId { get; } + + /// + /// Optional index to correlate fields within the same array item. + /// Used for array properties to indicate which array element this value belongs to. + /// Should be null for scalar fields. + /// + public int? ItemIndex { get; } + + /// + /// String-typed value. Set only when the field type is string. + /// + public string? StringValue { get; } + + /// + /// Number-typed value. Set only when the field type is numeric (int, long, decimal, etc.). + /// + public decimal? NumberValue { get; } + + /// + /// DateTime-typed value. Set only when the field type is DateTime or DateTimeOffset. + /// + public DateTimeOffset? DateTimeValue { get; } + + /// + /// Boolean-typed value. Set only when the field type is bool. + /// + public bool? BooleanValue { get; } + + /// + /// Guid-typed value. Set for GuidField and ExactMatchField (only guid_value stored), + /// and also set alongside StringValue for string fields (deterministic hash for fast exact-match queries). + /// + internal Guid? GuidValue { get; } + + private SearchFieldValue( + string fieldPath, + int? itemIndex, + string? stringValue, + decimal? numberValue, + DateTimeOffset? dateTimeValue, + bool? booleanValue, + Guid? guidValue) + { + if (string.IsNullOrWhiteSpace(fieldPath)) + { + throw new ArgumentException("Field path cannot be null or whitespace.", nameof(fieldPath)); + } + + if (itemIndex.HasValue && itemIndex.Value < 0) + { + throw new ArgumentException("Item index must be non-negative.", nameof(itemIndex)); + } + + // Validate that at least one value is set and values are compatible. + // number/datetime/boolean are "exclusive" — only one can be set, and none can coexist with string/guid. + // string and guid may coexist (string fields store both for LIKE and hash-based equality). + // guid may appear alone (GuidField / ExactMatchField). + var exclusiveCount = 0; + if (numberValue.HasValue) { exclusiveCount++; } + if (dateTimeValue.HasValue) { exclusiveCount++; } + if (booleanValue.HasValue) { exclusiveCount++; } + + if (exclusiveCount > 1) + { + throw new ArgumentException("Only one of number, datetime, or boolean values can be set at a time."); + } + + var hasString = stringValue is not null; + var hasGuid = guidValue.HasValue; + var hasExclusive = exclusiveCount > 0; + + if (!hasString && !hasGuid && !hasExclusive) + { + throw new ArgumentException("At least one typed value must be set."); + } + + if (hasExclusive && (hasString || hasGuid)) + { + throw new ArgumentException("Number, datetime, and boolean values cannot coexist with string or guid values."); + } + + FieldPath = fieldPath.ToUpperInvariant(); + FieldPathId = DeterministicGuidGenerator.Create(FieldPath); + ItemIndex = itemIndex; + StringValue = stringValue?.ToUpperInvariant(); + NumberValue = numberValue; + DateTimeValue = dateTimeValue; + BooleanValue = booleanValue; + GuidValue = guidValue; + } + + /// + /// Creates a string-typed search field value for a scalar (non-array) field. + /// Populates both StringValue (for LIKE queries) and GuidValue (for fast exact-match queries). + /// +#pragma warning disable CA1720 // Identifier contains type name + public static SearchFieldValue String(string fieldPath, string value) +#pragma warning restore CA1720 + { + if (string.IsNullOrEmpty(value)) + { + throw new ArgumentException("String value cannot be null or empty.", nameof(value)); + } + + var guidValue = DeterministicGuidGenerator.Create(value.ToUpperInvariant()); + return new SearchFieldValue(fieldPath, null, value, null, null, null, guidValue); + } + + /// + /// Creates a string-typed search field value for an array field. + /// Populates both StringValue (for LIKE queries) and GuidValue (for fast exact-match queries). + /// +#pragma warning disable CA1720 // Identifier contains type name + public static SearchFieldValue String(string fieldPath, string value, int itemIndex) +#pragma warning restore CA1720 + { + if (string.IsNullOrEmpty(value)) + { + throw new ArgumentException("String value cannot be null or empty.", nameof(value)); + } + + var guidValue = DeterministicGuidGenerator.Create(value.ToUpperInvariant()); + return new SearchFieldValue(fieldPath, itemIndex, value, null, null, null, guidValue); + } + + /// + /// Creates a number-typed search field value for a scalar (non-array) field. + /// + public static SearchFieldValue Number(string fieldPath, decimal value) => + new(fieldPath, null, null, value, null, null, null); + + /// + /// Creates a number-typed search field value for an array field. + /// + public static SearchFieldValue Number(string fieldPath, decimal value, int itemIndex) => + new(fieldPath, itemIndex, null, value, null, null, null); + + /// + /// Creates a DateTime-typed search field value for a scalar (non-array) field. + /// + public static SearchFieldValue DateTime(string fieldPath, DateTimeOffset value) => + new(fieldPath, null, null, null, value, null, null); + + /// + /// Creates a DateTime-typed search field value for an array field. + /// + public static SearchFieldValue DateTime(string fieldPath, DateTimeOffset value, int itemIndex) => + new(fieldPath, itemIndex, null, null, value, null, null); + + /// + /// Creates a boolean-typed search field value for a scalar (non-array) field. + /// + public static SearchFieldValue Boolean(string fieldPath, bool value) => + new(fieldPath, null, null, null, null, value, null); + + /// + /// Creates a boolean-typed search field value for an array field. + /// + public static SearchFieldValue Boolean(string fieldPath, bool value, int itemIndex) => + new(fieldPath, itemIndex, null, null, null, value, null); + + /// + /// Creates a Guid-typed search field value for a scalar (non-array) field. + /// Only guid_value is stored — use for GuidField. + /// +#pragma warning disable CA1720 // Identifier contains type name + public static SearchFieldValue Guid(string fieldPath, Guid value) => +#pragma warning restore CA1720 + new(fieldPath, null, null, null, null, null, value); + + /// + /// Creates a Guid-typed search field value for an array field. + /// Only guid_value is stored — use for GuidField. + /// +#pragma warning disable CA1720 // Identifier contains type name + public static SearchFieldValue Guid(string fieldPath, Guid value, int itemIndex) => +#pragma warning restore CA1720 + new(fieldPath, itemIndex, null, null, null, null, value); + + /// + /// Creates an exact-match search field value that stores a deterministic GUID hash. + /// No string_value is stored — only the hash in guid_value. Use for ExactMatchField. + /// + internal static SearchFieldValue ExactMatch(string fieldPath, string value) + { + if (string.IsNullOrEmpty(value)) + { + throw new ArgumentException("String value cannot be null or empty.", nameof(value)); + } + + var hash = DeterministicGuidGenerator.Create(value.ToUpperInvariant()); + return new SearchFieldValue(fieldPath, null, null, null, null, null, hash); + } + + /// + /// Creates an exact-match search field value for an array field. + /// No string_value is stored — only the hash in guid_value. Use for ExactMatchField. + /// + internal static SearchFieldValue ExactMatch(string fieldPath, string value, int itemIndex) + { + if (string.IsNullOrEmpty(value)) + { + throw new ArgumentException("String value cannot be null or empty.", nameof(value)); + } + + var hash = DeterministicGuidGenerator.Create(value.ToUpperInvariant()); + return new SearchFieldValue(fieldPath, itemIndex, null, null, null, null, hash); + } +} diff --git a/storage/src/Storage/Internal/Querying/SearchFields/SearchFieldsBuilder.cs b/storage/src/Storage/Internal/Querying/SearchFields/SearchFieldsBuilder.cs new file mode 100644 index 000000000..284d61d75 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/SearchFields/SearchFieldsBuilder.cs @@ -0,0 +1,267 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Querying.SearchFields; + +/// +/// Builder for constructing collections. +/// Provides helper methods for adding scalar values, nested values, and array item values. +/// +public sealed class SearchFieldsBuilder +{ + private readonly List _values = new(); + private readonly HashSet _keys = new(); + + /// + /// Adds a string-typed search field value. + /// + /// The field path (e.g., "userName", "consoleProperties.id") + /// The string value + /// This builder for method chaining + /// Thrown when a duplicate key (fieldPath + itemIndex) is added + public SearchFieldsBuilder Add(string fieldPath, string value) + { + ValidateUniqueKey(fieldPath, null); + _values.Add(SearchFieldValue.String(fieldPath, value)); + return this; + } + + /// + /// Adds a string-typed search field value for an array item. + /// + /// The field path (e.g., "emails.type") + /// The array item index (0-based) + /// The string value + /// This builder for method chaining + /// Thrown when a duplicate key (fieldPath + itemIndex) is added + public SearchFieldsBuilder Add(string fieldPath, int itemIndex, string value) + { + ValidateUniqueKey(fieldPath, itemIndex); + _values.Add(SearchFieldValue.String(fieldPath, value, itemIndex)); + return this; + } + + /// + /// Adds a decimal-typed search field value. + /// + /// The field path + /// The decimal value + /// This builder for method chaining + /// Thrown when a duplicate key (fieldPath + itemIndex) is added + public SearchFieldsBuilder Add(string fieldPath, decimal value) + { + ValidateUniqueKey(fieldPath, null); + _values.Add(SearchFieldValue.Number(fieldPath, value)); + return this; + } + + /// + /// Adds a decimal-typed search field value for an array item. + /// + /// The field path + /// The array item index (0-based) + /// The decimal value + /// This builder for method chaining + /// Thrown when a duplicate key (fieldPath + itemIndex) is added + public SearchFieldsBuilder Add(string fieldPath, int itemIndex, decimal value) + { + ValidateUniqueKey(fieldPath, itemIndex); + _values.Add(SearchFieldValue.Number(fieldPath, value, itemIndex)); + return this; + } + + /// + /// Adds a DateTime-typed search field value. + /// + /// The field path + /// The DateTime value + /// This builder for method chaining + /// Thrown when a duplicate key (fieldPath + itemIndex) is added + public SearchFieldsBuilder Add(string fieldPath, DateTime value) + { + ValidateUniqueKey(fieldPath, null); + _values.Add(SearchFieldValue.DateTime(fieldPath, value)); + return this; + } + + /// + /// Adds a DateTime-typed search field value for an array item. + /// + /// The field path + /// The array item index (0-based) + /// The DateTime value + /// This builder for method chaining + /// Thrown when a duplicate key (fieldPath + itemIndex) is added + public SearchFieldsBuilder Add(string fieldPath, int itemIndex, DateTime value) + { + ValidateUniqueKey(fieldPath, itemIndex); + _values.Add(SearchFieldValue.DateTime(fieldPath, value, itemIndex)); + return this; + } + + /// + /// Adds a DateTimeOffset-typed search field value. + /// + /// The field path + /// The DateTimeOffset value + /// This builder for method chaining + /// Thrown when a duplicate key (fieldPath + itemIndex) is added + public SearchFieldsBuilder Add(string fieldPath, DateTimeOffset value) + { + ValidateUniqueKey(fieldPath, null); + _values.Add(SearchFieldValue.DateTime(fieldPath, value)); + return this; + } + + /// + /// Adds a DateTimeOffset-typed search field value for an array item. + /// + /// The field path + /// The array item index (0-based) + /// The DateTimeOffset value + /// This builder for method chaining + /// Thrown when a duplicate key (fieldPath + itemIndex) is added + public SearchFieldsBuilder Add(string fieldPath, int itemIndex, DateTimeOffset value) + { + ValidateUniqueKey(fieldPath, itemIndex); + _values.Add(SearchFieldValue.DateTime(fieldPath, value, itemIndex)); + return this; + } + + /// + /// Adds a boolean-typed search field value. + /// + /// The field path + /// The boolean value + /// This builder for method chaining + /// Thrown when a duplicate key (fieldPath + itemIndex) is added + public SearchFieldsBuilder Add(string fieldPath, bool value) + { + ValidateUniqueKey(fieldPath, null); + _values.Add(SearchFieldValue.Boolean(fieldPath, value)); + return this; + } + + /// + /// Adds a boolean-typed search field value for an array item. + /// + /// The field path + /// The array item index (0-based) + /// The boolean value + /// This builder for method chaining + /// Thrown when a duplicate key (fieldPath + itemIndex) is added + public SearchFieldsBuilder Add(string fieldPath, int itemIndex, bool value) + { + ValidateUniqueKey(fieldPath, itemIndex); + _values.Add(SearchFieldValue.Boolean(fieldPath, value, itemIndex)); + return this; + } + + /// + /// Adds a Guid-typed search field value for a scalar (non-array) field. + /// + /// The field path + /// The Guid value + /// This builder for method chaining + /// Thrown when a duplicate key (fieldPath + itemIndex) is added + public SearchFieldsBuilder Add(string fieldPath, Guid value) + { + ValidateUniqueKey(fieldPath, null); + _values.Add(SearchFieldValue.Guid(fieldPath, value)); + return this; + } + + /// + /// Adds a Guid-typed search field value for an array item. + /// + /// The field path + /// The array item index (0-based) + /// The Guid value + /// This builder for method chaining + /// Thrown when a duplicate key (fieldPath + itemIndex) is added + public SearchFieldsBuilder Add(string fieldPath, int itemIndex, Guid value) + { + ValidateUniqueKey(fieldPath, itemIndex); + _values.Add(SearchFieldValue.Guid(fieldPath, value, itemIndex)); + return this; + } + + /// + /// Adds an exact-match search field value (stores deterministic GUID hash) for a scalar (non-array) field. + /// Use this with ExactMatchField for fast exact-match string lookups without storing the full string. + /// + /// The field path + /// The string value to hash + /// This builder for method chaining + /// Thrown when a duplicate key (fieldPath + itemIndex) is added + public SearchFieldsBuilder AddExactMatch(string fieldPath, string value) + { + ValidateUniqueKey(fieldPath, null); + _values.Add(SearchFieldValue.ExactMatch(fieldPath, value)); + return this; + } + + /// + /// Adds an exact-match search field value (stores deterministic GUID hash) for an array item. + /// Use this with ExactMatchField for fast exact-match string lookups without storing the full string. + /// + /// The field path + /// The array item index (0-based) + /// The string value to hash + /// This builder for method chaining + /// Thrown when a duplicate key (fieldPath + itemIndex) is added + public SearchFieldsBuilder AddExactMatch(string fieldPath, int itemIndex, string value) + { + ValidateUniqueKey(fieldPath, itemIndex); + _values.Add(SearchFieldValue.ExactMatch(fieldPath, value, itemIndex)); + return this; + } + + /// + /// Reserved field paths that cannot be used as user-defined search field names. + /// These map to system columns on the entities table. + /// + private static readonly HashSet ReservedFieldNames = new(StringComparer.OrdinalIgnoreCase) + { + SystemFields.Created, SystemFields.LastUpdated, + SystemFields.CreatedAttributeName, SystemFields.LastUpdatedAttributeName + }; + + /// + /// Validates that the key (fieldPath + itemIndex) has not already been added. + /// Normalizes the fieldPath to upper-invariant to match the normalization applied by . + /// + /// The field path + /// The optional item index + private void ValidateUniqueKey(string fieldPath, int? itemIndex) + { + if (string.IsNullOrWhiteSpace(fieldPath)) + { + throw new ArgumentException("Field path cannot be null or whitespace.", nameof(fieldPath)); + } + + if (ReservedFieldNames.Contains(fieldPath)) + { + throw new ArgumentException($"Field path '{fieldPath}' is reserved and cannot be used as a search field name.", nameof(fieldPath)); + } + + var normalizedFieldPath = fieldPath.ToUpperInvariant(); + var key = new SearchFieldKey(normalizedFieldPath, itemIndex); + if (!_keys.Add(key)) + { + var indexInfo = itemIndex.HasValue ? $" with item index {itemIndex.Value}" : " (scalar field)"; + throw new ArgumentException($"Duplicate search field key detected: field path '{fieldPath}'{indexInfo} has already been added."); + } + } + + /// + /// Represents a unique key for a search field (fieldPath + itemIndex). + /// + private sealed record SearchFieldKey(string FieldPath, int? ItemIndex); + + /// + /// Builds the immutable collection. + /// + /// An immutable SearchFields collection containing all added values + public SearchFieldCollection Build() => new SearchFieldCollection(_values.ToArray()); +} diff --git a/storage/src/Storage/Internal/Querying/Sorting/SortParameter.cs b/storage/src/Storage/Internal/Querying/Sorting/SortParameter.cs new file mode 100644 index 000000000..d5c229314 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/Sorting/SortParameter.cs @@ -0,0 +1,56 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Diagnostics.CodeAnalysis; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Querying; + +namespace Duende.Storage.Internal.Querying.Sorting; + +/// +/// Specifies a field and direction for sorting query results. +/// +public sealed record SortParameter +{ + /// + /// Represents an empty sort parameter indicating no sorting should be applied. + /// + public static readonly SortParameter Empty = new(); + + /// + /// Gets a value indicating whether this is an empty sort parameter. + /// + [MemberNotNullWhen(false, nameof(Field))] + public bool IsEmpty => Field is null; + + /// + /// The field to sort by. + /// + public Field? Field { get; } + + /// + /// The direction to sort in. + /// + public SortDirection Direction { get; } + + /// + /// Private constructor for creating empty sort parameter. + /// + private SortParameter() + { + Field = null; + Direction = SortDirection.Ascending; + } + + /// + /// Creates a new sort parameter. + /// + /// The field to sort by. + /// The direction to sort in. Defaults to Ascending. + public SortParameter(Field field, SortDirection direction = SortDirection.Ascending) + { + ArgumentNullException.ThrowIfNull(field); + Field = field; + Direction = direction; + } +} diff --git a/storage/src/Storage/Internal/Querying/SqlWhereClauseBuilder.cs b/storage/src/Storage/Internal/Querying/SqlWhereClauseBuilder.cs new file mode 100644 index 000000000..7175d90fd --- /dev/null +++ b/storage/src/Storage/Internal/Querying/SqlWhereClauseBuilder.cs @@ -0,0 +1,990 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Collections; +using System.Data.Common; +using System.Globalization; +using System.Text; +using Duende.Storage.Internal.Querying.Expressions; +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Querying; + +/// +/// Visitor that translates query expression trees to SQL WHERE clauses with parameterized EXISTS subqueries. +/// All values are parameterized to prevent SQL injection. +/// +internal sealed class SqlWhereClauseBuilder(string schemaName, DbCommand command, ISqlDialect dialect) : IQueryExpressionVisitor +{ + private int _parameterCounter; + private int _subqueryCounter; + + /// + /// Returns the entity table column name for a system/reserved field path, or null if not a system field. + /// Throws if the field path is a system field but the field type is not DateTimeField. + /// + private static string? GetSystemColumnName(string fieldPath, Field? field) + { + if (string.Equals(fieldPath, SystemFields.Created, StringComparison.OrdinalIgnoreCase) || + string.Equals(fieldPath, SystemFields.CreatedAttributeName, StringComparison.OrdinalIgnoreCase)) + { + if (field is not null and not DateTimeField) + { + throw new InvalidOperationException($"System field '{fieldPath}' must use DateTimeField, not {field.GetType().Name}."); + } + + return "v.created_at"; + } + + if (string.Equals(fieldPath, SystemFields.LastUpdated, StringComparison.OrdinalIgnoreCase) || + string.Equals(fieldPath, SystemFields.LastUpdatedAttributeName, StringComparison.OrdinalIgnoreCase)) + { + if (field is not null and not DateTimeField) + { + throw new InvalidOperationException($"System field '{fieldPath}' must use DateTimeField, not {field.GetType().Name}."); + } + + return "v.last_updated_at"; + } + + return null; + } + + /// + /// Returns the item_index condition for a field. Scalar fields use item_index = -1 to hit + /// the partial index; multi-valued fields use item_index >= 0 to match any array element. + /// + private static string ItemIndexCondition(string alias, Field field) => + field.IsMultiValued + ? $"{alias}.item_index >= 0" + : $"{alias}.item_index = -1"; + + /// + /// Converts a field path string to the dialect-specific parameter value (GUID as byte[] for SQLite, Guid for others). + /// + private object FieldPathToParam(string fieldPath) => + dialect.FieldPathToParameterValue(DeterministicGuidGenerator.Create(fieldPath.ToUpperInvariant())); + + private static object FieldPathToParam(ISqlDialect dialect, string fieldPath) => + dialect.FieldPathToParameterValue(DeterministicGuidGenerator.Create(fieldPath.ToUpperInvariant())); + + /// + /// Builds the WHERE clause for the given expression. + /// + public string BuildWhereClause(IQueryExpression expression) + { + var clause = expression switch + { + AllExpression all => Visit(all), + EqualExpression eq => Visit(eq), + ContainsExpression contains => Visit(contains), + StartsWithExpression startsWith => Visit(startsWith), + EndsWithExpression endsWith => Visit(endsWith), + GreaterThanExpression gt => Visit(gt), + LessThanExpression lt => Visit(lt), + GreaterOrEqualExpression gte => Visit(gte), + LessOrEqualExpression lte => Visit(lte), + BetweenExpression between => Visit(between), + InExpression inExpr => Visit(inExpr), + NotExpression not => Visit(not), + AndExpression and => Visit(and), + OrExpression or => Visit(or), + ArrayFilterExpression arrayFilter => Visit(arrayFilter), + ArrayContainsExpression arrayContains => Visit(arrayContains), + PresentExpression present => Visit(present), + _ => throw new NotSupportedException($"Expression type {expression.GetType().Name} is not supported.") + }; + + return clause; + } + + // No filter - match all records + public string Visit(AllExpression expression) => dialect.TrueLiteral; + + public string Visit(EqualExpression expression) + { + var fieldPath = expression.Field.Path; + var value = expression.Value; + var (columnName, paramValue) = GetValueColumnAndParameter(value); + + var systemColumn = GetSystemColumnName(fieldPath, expression.Field); + if (systemColumn is not null) + { + var valueParam = AddParameter($"value_{_parameterCounter}", paramValue); + _parameterCounter++; + return $"{systemColumn} = {valueParam}"; + } + + var fieldPathParam = AddParameter($"field_path_{_parameterCounter}", FieldPathToParam(fieldPath)); + var valueParam2 = AddParameter($"value_{_parameterCounter}", paramValue); + _parameterCounter++; + + var subqueryId = _subqueryCounter++; + var itemIndexCond = ItemIndexCondition($"sv{subqueryId}", expression.Field); + + return $""" + EXISTS ( + SELECT 1 FROM {schemaName}.search_values sv{subqueryId} + WHERE sv{subqueryId}.entity_type_id = v.entity_type_id + AND sv{subqueryId}.pool_id = v.pool_id + AND sv{subqueryId}.entity_id = v.entity_id + AND sv{subqueryId}.field_path = {fieldPathParam} + AND {itemIndexCond} + AND sv{subqueryId}.{columnName} = {valueParam2} + ) + """; + } + + public string Visit(ContainsExpression expression) + { + var fieldPath = expression.Field.Path; + var value = expression.Value; + + var fieldPathParam = AddParameter($"field_path_{_parameterCounter}", FieldPathToParam(fieldPath)); + var escapedValue = dialect.EscapeLikeWildcards(value); + var valueParam = AddParameter($"value_{_parameterCounter}", $"%{escapedValue}%"); + _parameterCounter++; + + var subqueryId = _subqueryCounter++; + var itemIndexCond = ItemIndexCondition($"sv{subqueryId}", expression.Field); + + return $""" + EXISTS ( + SELECT 1 FROM {schemaName}.search_values sv{subqueryId} + WHERE sv{subqueryId}.entity_type_id = v.entity_type_id + AND sv{subqueryId}.pool_id = v.pool_id + AND sv{subqueryId}.entity_id = v.entity_id + AND sv{subqueryId}.field_path = {fieldPathParam} + AND {itemIndexCond} + AND sv{subqueryId}.string_value {dialect.CaseInsensitiveLikeOperator} {valueParam}{dialect.LikeEscapeClause} + ) + """; + } + + public string Visit(StartsWithExpression expression) + { + var fieldPath = expression.Field.Path; + var value = expression.Value; + + var fieldPathParam = AddParameter($"field_path_{_parameterCounter}", FieldPathToParam(fieldPath)); + var escapedValue = dialect.EscapeLikeWildcards(value); + var valueParam = AddParameter($"value_{_parameterCounter}", $"{escapedValue}%"); + _parameterCounter++; + + var subqueryId = _subqueryCounter++; + var itemIndexCond = ItemIndexCondition($"sv{subqueryId}", expression.Field); + + return $""" + EXISTS ( + SELECT 1 FROM {schemaName}.search_values sv{subqueryId} + WHERE sv{subqueryId}.entity_type_id = v.entity_type_id + AND sv{subqueryId}.pool_id = v.pool_id + AND sv{subqueryId}.entity_id = v.entity_id + AND sv{subqueryId}.field_path = {fieldPathParam} + AND {itemIndexCond} + AND sv{subqueryId}.string_value {dialect.CaseInsensitiveLikeOperator} {valueParam}{dialect.LikeEscapeClause} + ) + """; + } + + public string Visit(GreaterThanExpression expression) + { + var fieldPath = expression.Field.Path; + var value = expression.Value; + var (columnName, paramValue) = GetValueColumnAndParameter(value); + + var systemColumn = GetSystemColumnName(fieldPath, expression.Field); + if (systemColumn is not null) + { + var valueParam = AddParameter($"value_{_parameterCounter}", paramValue); + _parameterCounter++; + return $"{systemColumn} > {valueParam}"; + } + + var fieldPathParam = AddParameter($"field_path_{_parameterCounter}", FieldPathToParam(fieldPath)); + var valueParam2 = AddParameter($"value_{_parameterCounter}", paramValue); + _parameterCounter++; + + var subqueryId = _subqueryCounter++; + var itemIndexCond = ItemIndexCondition($"sv{subqueryId}", expression.Field); + + return $""" + EXISTS ( + SELECT 1 FROM {schemaName}.search_values sv{subqueryId} + WHERE sv{subqueryId}.entity_type_id = v.entity_type_id + AND sv{subqueryId}.pool_id = v.pool_id + AND sv{subqueryId}.entity_id = v.entity_id + AND sv{subqueryId}.field_path = {fieldPathParam} + AND {itemIndexCond} + AND sv{subqueryId}.{columnName} > {valueParam2} + ) + """; + } + + public string Visit(LessThanExpression expression) + { + var fieldPath = expression.Field.Path; + var value = expression.Value; + var (columnName, paramValue) = GetValueColumnAndParameter(value); + + var systemColumn = GetSystemColumnName(fieldPath, expression.Field); + if (systemColumn is not null) + { + var valueParam = AddParameter($"value_{_parameterCounter}", paramValue); + _parameterCounter++; + return $"{systemColumn} < {valueParam}"; + } + + var fieldPathParam = AddParameter($"field_path_{_parameterCounter}", FieldPathToParam(fieldPath)); + var valueParam2 = AddParameter($"value_{_parameterCounter}", paramValue); + _parameterCounter++; + + var subqueryId = _subqueryCounter++; + var itemIndexCond = ItemIndexCondition($"sv{subqueryId}", expression.Field); + + return $""" + EXISTS ( + SELECT 1 FROM {schemaName}.search_values sv{subqueryId} + WHERE sv{subqueryId}.entity_type_id = v.entity_type_id + AND sv{subqueryId}.pool_id = v.pool_id + AND sv{subqueryId}.entity_id = v.entity_id + AND sv{subqueryId}.field_path = {fieldPathParam} + AND {itemIndexCond} + AND sv{subqueryId}.{columnName} < {valueParam2} + ) + """; + } + + public string Visit(GreaterOrEqualExpression expression) + { + var fieldPath = expression.Field.Path; + var value = expression.Value; + var (columnName, paramValue) = GetValueColumnAndParameter(value); + + var systemColumn = GetSystemColumnName(fieldPath, expression.Field); + if (systemColumn is not null) + { + var valueParam = AddParameter($"value_{_parameterCounter}", paramValue); + _parameterCounter++; + return $"{systemColumn} >= {valueParam}"; + } + + var fieldPathParam = AddParameter($"field_path_{_parameterCounter}", FieldPathToParam(fieldPath)); + var valueParam2 = AddParameter($"value_{_parameterCounter}", paramValue); + _parameterCounter++; + + var subqueryId = _subqueryCounter++; + var itemIndexCond = ItemIndexCondition($"sv{subqueryId}", expression.Field); + + return $""" + EXISTS ( + SELECT 1 FROM {schemaName}.search_values sv{subqueryId} + WHERE sv{subqueryId}.entity_type_id = v.entity_type_id + AND sv{subqueryId}.pool_id = v.pool_id + AND sv{subqueryId}.entity_id = v.entity_id + AND sv{subqueryId}.field_path = {fieldPathParam} + AND {itemIndexCond} + AND sv{subqueryId}.{columnName} >= {valueParam2} + ) + """; + } + + public string Visit(LessOrEqualExpression expression) + { + var fieldPath = expression.Field.Path; + var value = expression.Value; + var (columnName, paramValue) = GetValueColumnAndParameter(value); + + var systemColumn = GetSystemColumnName(fieldPath, expression.Field); + if (systemColumn is not null) + { + var valueParam = AddParameter($"value_{_parameterCounter}", paramValue); + _parameterCounter++; + return $"{systemColumn} <= {valueParam}"; + } + + var fieldPathParam = AddParameter($"field_path_{_parameterCounter}", FieldPathToParam(fieldPath)); + var valueParam2 = AddParameter($"value_{_parameterCounter}", paramValue); + _parameterCounter++; + + var subqueryId = _subqueryCounter++; + var itemIndexCond = ItemIndexCondition($"sv{subqueryId}", expression.Field); + + return $""" + EXISTS ( + SELECT 1 FROM {schemaName}.search_values sv{subqueryId} + WHERE sv{subqueryId}.entity_type_id = v.entity_type_id + AND sv{subqueryId}.pool_id = v.pool_id + AND sv{subqueryId}.entity_id = v.entity_id + AND sv{subqueryId}.field_path = {fieldPathParam} + AND {itemIndexCond} + AND sv{subqueryId}.{columnName} <= {valueParam2} + ) + """; + } + + public string Visit(BetweenExpression expression) + { + var fieldPath = expression.Field.Path; + var minValue = expression.Min; + var maxValue = expression.Max; + var (columnName, minParamValue) = GetValueColumnAndParameter(minValue); + var (_, maxParamValue) = GetValueColumnAndParameter(maxValue); + + var systemColumn = GetSystemColumnName(fieldPath, expression.Field); + if (systemColumn is not null) + { + var minParam = AddParameter($"min_{_parameterCounter}", minParamValue); + var maxParam = AddParameter($"max_{_parameterCounter}", maxParamValue); + _parameterCounter++; + return $"{systemColumn} BETWEEN {minParam} AND {maxParam}"; + } + + var fieldPathParam = AddParameter($"field_path_{_parameterCounter}", FieldPathToParam(fieldPath)); + var minParam2 = AddParameter($"min_{_parameterCounter}", minParamValue); + var maxParam2 = AddParameter($"max_{_parameterCounter}", maxParamValue); + _parameterCounter++; + + var subqueryId = _subqueryCounter++; + var itemIndexCond = ItemIndexCondition($"sv{subqueryId}", expression.Field); + + return $""" + EXISTS ( + SELECT 1 FROM {schemaName}.search_values sv{subqueryId} + WHERE sv{subqueryId}.entity_type_id = v.entity_type_id + AND sv{subqueryId}.pool_id = v.pool_id + AND sv{subqueryId}.entity_id = v.entity_id + AND sv{subqueryId}.field_path = {fieldPathParam} + AND {itemIndexCond} + AND sv{subqueryId}.{columnName} BETWEEN {minParam2} AND {maxParam2} + ) + """; + } + + public string Visit(InExpression expression) + { + var fieldPath = expression.Field.Path; + var values = expression.Values; + + var valuesList = new List(); + foreach (var value in values) + { + valuesList.Add(value); + } + + if (valuesList.Count == 0) + { + // IN with empty list never matches + return dialect.FalseLiteral; + } + + var systemColumn = GetSystemColumnName(fieldPath, expression.Field); + if (systemColumn is not null) + { + var sysValueParams = new List(); + for (var i = 0; i < valuesList.Count; i++) + { + var (_, paramValue) = GetValueColumnAndParameter(valuesList[i]); + var valueParam = AddParameter($"in_value_{_parameterCounter}_{i}", paramValue); + sysValueParams.Add(valueParam); + } + + _parameterCounter++; + var sysInClause = string.Join(", ", sysValueParams); + return $"{systemColumn} IN ({sysInClause})"; + } + + var (columnName, _) = GetValueColumnAndParameter(valuesList[0]); + var fieldPathParam = AddParameter($"field_path_{_parameterCounter}", FieldPathToParam(fieldPath)); + + // Add parameters for all values + var valueParams = new List(); + for (var i = 0; i < valuesList.Count; i++) + { + var (_, paramValue) = GetValueColumnAndParameter(valuesList[i]); + var valueParam = AddParameter($"in_value_{_parameterCounter}_{i}", paramValue); + valueParams.Add(valueParam); + } + + _parameterCounter++; + var subqueryId = _subqueryCounter++; + var itemIndexCond = ItemIndexCondition($"sv{subqueryId}", expression.Field); + + var inClause = string.Join(", ", valueParams); + + return $""" + EXISTS ( + SELECT 1 FROM {schemaName}.search_values sv{subqueryId} + WHERE sv{subqueryId}.entity_type_id = v.entity_type_id + AND sv{subqueryId}.pool_id = v.pool_id + AND sv{subqueryId}.entity_id = v.entity_id + AND sv{subqueryId}.field_path = {fieldPathParam} + AND {itemIndexCond} + AND sv{subqueryId}.{columnName} IN ({inClause}) + ) + """; + } + + public string Visit(AndExpression expression) + { + var parts = new List(); + foreach (var part in expression.Parts) + { + var clause = BuildWhereClause(part); + parts.Add($"({clause})"); + } + + return string.Join(" AND ", parts); + } + + public string Visit(OrExpression expression) + { + var parts = new List(); + foreach (var part in expression.Parts) + { + var clause = BuildWhereClause(part); + parts.Add($"({clause})"); + } + + return string.Join(" OR ", parts); + } + + public string Visit(ArrayFilterExpression expression) + { + var arrayFieldPath = expression.ArrayFieldPath; + var filter = expression.Filter; + + // Create a scoped builder for array item filters + var arrayItemBuilder = new ArrayItemSqlBuilder(schemaName, command, dialect, arrayFieldPath, _parameterCounter, _subqueryCounter); + var filterClause = arrayItemBuilder.BuildArrayItemFilter(filter); + + // Update counters + _parameterCounter = arrayItemBuilder.ParameterCounter; + _subqueryCounter = arrayItemBuilder.SubqueryCounter; + + return filterClause; + } + + public string Visit(NotExpression expression) + { + var innerClause = BuildWhereClause(expression.Inner); + return $"NOT ({innerClause})"; + } + + public string Visit(EndsWithExpression expression) + { + var fieldPath = expression.Field.Path; + var value = expression.Value; + + var fieldPathParam = AddParameter($"field_path_{_parameterCounter}", FieldPathToParam(fieldPath)); + var escapedValue = dialect.EscapeLikeWildcards(value); + var valueParam = AddParameter($"value_{_parameterCounter}", $"%{escapedValue}"); + _parameterCounter++; + + var subqueryId = _subqueryCounter++; + var itemIndexCond = ItemIndexCondition($"sv{subqueryId}", expression.Field); + + return $""" + EXISTS ( + SELECT 1 FROM {schemaName}.search_values sv{subqueryId} + WHERE sv{subqueryId}.entity_type_id = v.entity_type_id + AND sv{subqueryId}.pool_id = v.pool_id + AND sv{subqueryId}.entity_id = v.entity_id + AND sv{subqueryId}.field_path = {fieldPathParam} + AND {itemIndexCond} + AND sv{subqueryId}.string_value {dialect.CaseInsensitiveLikeOperator} {valueParam}{dialect.LikeEscapeClause} + ) + """; + } + + public string Visit(PresentExpression expression) + { + var fieldPath = expression.Field.Path; + + // System fields are NOT NULL columns on the entities table — they are always present. + if (SystemFields.IsSystemField(fieldPath)) + { + return "1=1"; + } + + var fieldPathParam = AddParameter($"field_path_{_parameterCounter}", FieldPathToParam(fieldPath)); + _parameterCounter++; + + var subqueryId = _subqueryCounter++; + + if (expression.Field.IsMultiValued) + { + // For multi-valued fields, check that at least one array element exists (item_index >= 0) + return $""" + EXISTS ( + SELECT 1 FROM {schemaName}.search_values sv{subqueryId} + WHERE sv{subqueryId}.entity_type_id = v.entity_type_id + AND sv{subqueryId}.pool_id = v.pool_id + AND sv{subqueryId}.entity_id = v.entity_id + AND sv{subqueryId}.field_path = {fieldPathParam} + AND sv{subqueryId}.item_index >= 0 + ) + """; + } + + // For scalar fields, check that a non-null value exists with item_index = -1 + var columnName = GetColumnNameForFieldType(expression.Field.Type); + + return $""" + EXISTS ( + SELECT 1 FROM {schemaName}.search_values sv{subqueryId} + WHERE sv{subqueryId}.entity_type_id = v.entity_type_id + AND sv{subqueryId}.pool_id = v.pool_id + AND sv{subqueryId}.entity_id = v.entity_id + AND sv{subqueryId}.field_path = {fieldPathParam} + AND sv{subqueryId}.item_index = -1 + AND sv{subqueryId}.{columnName} IS NOT NULL + ) + """; + } + + public string Visit(ArrayContainsExpression expression) + { + var fieldPath = expression.Field.Path; + var value = expression.Value; + + var fieldPathParam = AddParameter($"field_path_{_parameterCounter}", FieldPathToParam(fieldPath)); + var guidValue = DeterministicGuidGenerator.Create(value.ToUpperInvariant()); + var valueParam = AddParameter($"value_{_parameterCounter}", guidValue); + _parameterCounter++; + + var subqueryId = _subqueryCounter++; + + return $""" + EXISTS ( + SELECT 1 FROM {schemaName}.search_values sv{subqueryId} + WHERE sv{subqueryId}.entity_type_id = v.entity_type_id + AND sv{subqueryId}.pool_id = v.pool_id + AND sv{subqueryId}.entity_id = v.entity_id + AND sv{subqueryId}.field_path = {fieldPathParam} + AND sv{subqueryId}.item_index >= 0 + AND sv{subqueryId}.guid_value = {valueParam} + ) + """; + } + + /// + /// Gets the appropriate column name and parameter value based on the value type. + /// + private static (string ColumnName, object ParameterValue) GetValueColumnAndParameter(object value) => value switch + { + Guid g => ("guid_value", g), + string => ("string_value", value), + decimal d => ("number_value", d), + DateTimeOffset dto => ("datetime_value", dto.UtcDateTime), + DateTime dt => ("datetime_value", dt.ToUniversalTime()), + bool b => ("boolean_value", b), + _ => throw new NotSupportedException($"Value type {value.GetType().Name} is not supported.") + }; + + /// + /// Gets the column name for the given field type. + /// + private static string GetColumnNameForFieldType(FieldType fieldType) => fieldType switch + { + FieldType.String => "string_value", + FieldType.Number => "number_value", + FieldType.DateTime => "datetime_value", + FieldType.Boolean => "boolean_value", + FieldType.Guid => "guid_value", + _ => throw new NotSupportedException($"Field type {fieldType} is not supported.") + }; + + /// + /// Adds a parameter to the command and returns the parameter name. + /// + private string AddParameter(string name, object value) + { + var paramName = $"@{name}"; + dialect.AddParameter(command, paramName, value); + return paramName; + } + + /// + /// Helper class for building SQL for array item filters with item_index correlation. + /// + private sealed class ArrayItemSqlBuilder( + string schemaName, + DbCommand command, + ISqlDialect dialect, + string arrayFieldPath, + int parameterCounter, + int subqueryCounter) + { + private int _parameterCounter = parameterCounter; + private int _subqueryCounter = subqueryCounter; + + public int ParameterCounter => _parameterCounter; + public int SubqueryCounter => _subqueryCounter; + + public string BuildArrayItemFilter(IQueryFilterExpression filter) + { + // Check if this is a top-level OR expression + if (filter is OrExpression orExpr) + { + return BuildOrFilter(orExpr); + } + + // For AND and leaf expressions, collect all conditions + var conditions = new List(); + CollectConditions(filter, conditions); + + if (conditions.Count == 0) + { + return dialect.FalseLiteral; + } + + return BuildAndFilter(conditions); + } + + private string BuildAndFilter(List conditions) + { + // Build the correlated subquery with JOINs for AND conditions + var subqueryId = _subqueryCounter++; + var sb = new StringBuilder(); + + _ = sb.AppendLine("EXISTS ("); + _ = sb.Append(CultureInfo.InvariantCulture, $" SELECT 1 FROM {schemaName}.search_values sv{subqueryId}_0"); + + // Add JOINs for additional conditions (all correlating on item_index) + for (var i = 1; i < conditions.Count; i++) + { + _ = sb.Append(CultureInfo.InvariantCulture, $""" + + INNER JOIN {schemaName}.search_values sv{subqueryId}_{i} + ON sv{subqueryId}_0.entity_type_id = sv{subqueryId}_{i}.entity_type_id + AND sv{subqueryId}_0.pool_id = sv{subqueryId}_{i}.pool_id + AND sv{subqueryId}_0.entity_id = sv{subqueryId}_{i}.entity_id + AND sv{subqueryId}_0.item_index = sv{subqueryId}_{i}.item_index + """); + } + + // WHERE clause + _ = sb.Append(CultureInfo.InvariantCulture, $""" + + WHERE sv{subqueryId}_0.entity_type_id = v.entity_type_id + AND sv{subqueryId}_0.pool_id = v.pool_id + AND sv{subqueryId}_0.entity_id = v.entity_id + AND sv{subqueryId}_0.item_index IS NOT NULL + """); + + // Add conditions for each joined table + for (var i = 0; i < conditions.Count; i++) + { + var condition = conditions[i]; + var fieldPath = $"{arrayFieldPath}.{condition.FieldPath}"; + var fieldPathParam = AddParameter($"array_field_{_parameterCounter}_{i}", FieldPathToParam(dialect, fieldPath)); + + var conditionClause = BuildConditionClause(condition, subqueryId, i); + _ = sb.Append(CultureInfo.InvariantCulture, $""" + + AND sv{subqueryId}_{i}.field_path = {fieldPathParam} + AND {conditionClause} + """); + } + + _parameterCounter++; + + _ = sb.AppendLine(); + _ = sb.Append(')'); + + return sb.ToString(); + } + + private string BuildOrFilter(OrExpression orExpr) + { + var parts = new List(); + + foreach (var part in orExpr.Parts) + { + // Each OR branch might be AND or a leaf condition + if (part is AndExpression || IsLeafExpression(part)) + { + var conditions = new List(); + CollectConditions(part, conditions); + + if (conditions.Count > 0) + { + parts.Add(BuildOrBranch(conditions)); + } + } + else if (part is OrExpression nestedOr) + { + // Flatten nested OR expressions + foreach (var nestedPart in nestedOr.Parts) + { + var conditions = new List(); + CollectConditions(nestedPart, conditions); + + if (conditions.Count > 0) + { + parts.Add(BuildOrBranch(conditions)); + } + } + } + else + { + throw new NotSupportedException($"Expression type {part.GetType().Name} is not supported in OR expressions within array filters."); + } + } + + if (parts.Count == 0) + { + return dialect.FalseLiteral; + } + + // Combine branches with OR - each branch is already wrapped in EXISTS + return string.Join(" OR ", parts.Select(p => $"({p})")); + } + + private string BuildOrBranch(List conditions) + { + var subqueryId = _subqueryCounter++; + var sb = new StringBuilder(); + + _ = sb.AppendLine("EXISTS ("); + _ = sb.Append(CultureInfo.InvariantCulture, $" SELECT 1 FROM {schemaName}.search_values sv{subqueryId}_0"); + + // Add JOINs for additional conditions in this branch + for (var i = 1; i < conditions.Count; i++) + { + _ = sb.Append(CultureInfo.InvariantCulture, $""" + + INNER JOIN {schemaName}.search_values sv{subqueryId}_{i} + ON sv{subqueryId}_0.entity_type_id = sv{subqueryId}_{i}.entity_type_id + AND sv{subqueryId}_0.pool_id = sv{subqueryId}_{i}.pool_id + AND sv{subqueryId}_0.entity_id = sv{subqueryId}_{i}.entity_id + AND sv{subqueryId}_0.item_index = sv{subqueryId}_{i}.item_index + """); + } + + // WHERE clause + _ = sb.Append(CultureInfo.InvariantCulture, $""" + + WHERE sv{subqueryId}_0.entity_type_id = v.entity_type_id + AND sv{subqueryId}_0.pool_id = v.pool_id + AND sv{subqueryId}_0.entity_id = v.entity_id + AND sv{subqueryId}_0.item_index IS NOT NULL + """); + + // Add conditions for this branch + for (var i = 0; i < conditions.Count; i++) + { + var condition = conditions[i]; + var fieldPath = $"{arrayFieldPath}.{condition.FieldPath}"; + var fieldPathParam = AddParameter($"array_field_{_parameterCounter}_{i}", FieldPathToParam(dialect, fieldPath)); + + var conditionClause = BuildConditionClause(condition, subqueryId, i); + _ = sb.Append(CultureInfo.InvariantCulture, $""" + + AND sv{subqueryId}_{i}.field_path = {fieldPathParam} + AND {conditionClause} + """); + } + + _parameterCounter++; + + _ = sb.AppendLine(); + _ = sb.Append(')'); + + return sb.ToString(); + } + + private static bool IsLeafExpression(IQueryFilterExpression expr) => + expr is EqualExpression + or ContainsExpression + or StartsWithExpression + or EndsWithExpression + or GreaterThanExpression + or LessThanExpression + or GreaterOrEqualExpression + or LessOrEqualExpression + or BetweenExpression + or InExpression + or PresentExpression + or NotExpression; + + private void CollectConditions(IQueryFilterExpression filter, List conditions) + { + switch (filter) + { + case EqualExpression eq: + conditions.Add(new ArrayItemCondition(eq.Field.Path, "=", eq.Value)); + break; + case ContainsExpression contains: + { + var escapedValue = dialect.EscapeLikeWildcards(contains.Value); + conditions.Add(new ArrayItemCondition(contains.Field.Path, dialect.CaseInsensitiveLikeOperator, $"%{escapedValue}%")); + break; + } + case StartsWithExpression startsWith: + { + var escapedValue = dialect.EscapeLikeWildcards(startsWith.Value); + conditions.Add(new ArrayItemCondition(startsWith.Field.Path, dialect.CaseInsensitiveLikeOperator, $"{escapedValue}%")); + break; + } + case EndsWithExpression endsWith: + { + var escapedValue = dialect.EscapeLikeWildcards(endsWith.Value); + conditions.Add(new ArrayItemCondition(endsWith.Field.Path, dialect.CaseInsensitiveLikeOperator, $"%{escapedValue}")); + break; + } + case GreaterThanExpression gt: + conditions.Add(new ArrayItemCondition(gt.Field.Path, ">", gt.Value)); + break; + case LessThanExpression lt: + conditions.Add(new ArrayItemCondition(lt.Field.Path, "<", lt.Value)); + break; + case GreaterOrEqualExpression gte: + conditions.Add(new ArrayItemCondition(gte.Field.Path, ">=", gte.Value)); + break; + case LessOrEqualExpression lte: + conditions.Add(new ArrayItemCondition(lte.Field.Path, "<=", lte.Value)); + break; + case BetweenExpression between: + // Split BETWEEN into two conditions + conditions.Add(new ArrayItemCondition(between.Field.Path, ">=", between.Min)); + conditions.Add(new ArrayItemCondition(between.Field.Path, "<=", between.Max)); + break; + case InExpression inExpr: + conditions.Add(new ArrayItemCondition(inExpr.Field.Path, "IN", inExpr.Values)); + break; + case NotExpression notExpr: + CollectNotConditions(notExpr, conditions); + break; + case PresentExpression present: + conditions.Add(new ArrayItemCondition(present.Field.Path, "IS NOT NULL", present.Field.Type)); + break; + case AndExpression and: + // For AND within array filter, collect all conditions (they must all match the same item) + foreach (var part in and.Parts) + { + CollectConditions(part, conditions); + } + break; + case OrExpression: + // OR expressions should be handled at a higher level in BuildArrayItemFilter + throw new InvalidOperationException("OR expressions should not reach CollectConditions. This is a bug."); + case ArrayFilterExpression: + throw new NotSupportedException("Nested array filters are not supported."); + default: + throw new NotSupportedException($"Expression type {filter.GetType().Name} is not supported in array filters."); + } + } + + private void CollectNotConditions(NotExpression notExpr, List conditions) + { + switch (notExpr.Inner) + { + case EqualExpression eq: + conditions.Add(new ArrayItemCondition(eq.Field.Path, "!=", eq.Value)); + break; + case ContainsExpression contains: + { + var escapedValue = dialect.EscapeLikeWildcards(contains.Value); + conditions.Add(new ArrayItemCondition(contains.Field.Path, $"NOT {dialect.CaseInsensitiveLikeOperator}", $"%{escapedValue}%")); + break; + } + case StartsWithExpression startsWith: + { + var escapedValue = dialect.EscapeLikeWildcards(startsWith.Value); + conditions.Add(new ArrayItemCondition(startsWith.Field.Path, $"NOT {dialect.CaseInsensitiveLikeOperator}", $"{escapedValue}%")); + break; + } + case EndsWithExpression endsWith: + { + var escapedValue = dialect.EscapeLikeWildcards(endsWith.Value); + conditions.Add(new ArrayItemCondition(endsWith.Field.Path, $"NOT {dialect.CaseInsensitiveLikeOperator}", $"%{escapedValue}")); + break; + } + case PresentExpression present: + conditions.Add(new ArrayItemCondition(present.Field.Path, "IS NULL", present.Field.Type)); + break; + default: + throw new NotSupportedException($"NOT wrapping {notExpr.Inner.GetType().Name} is not supported in array filters. Use NOT at the top level instead."); + } + } + + private string BuildConditionClause(ArrayItemCondition condition, int subqueryId, int tableIndex) + { + if (condition.Operator is "IS NOT NULL" or "IS NULL") + { + // For presence checks, the Value holds the FieldType enum to determine the column + var fieldType = (FieldType)condition.Value; + var columnName = fieldType switch + { + FieldType.String => "string_value", + FieldType.Number => "number_value", + FieldType.DateTime => "datetime_value", + FieldType.Boolean => "boolean_value", + FieldType.Guid => "guid_value", + _ => throw new NotSupportedException($"Field type {fieldType} is not supported.") + }; + return $"sv{subqueryId}_{tableIndex}.{columnName} {condition.Operator}"; + } + + if (condition.Operator == "IN") + { + var valuesList = new List(); + foreach (var value in (IEnumerable)condition.Value) + { + valuesList.Add(value); + } + + if (valuesList.Count == 0) + { + return dialect.FalseLiteral; + } + + // Get column name from first value in the list + var (columnName, _) = GetValueColumnAndParameter(valuesList[0]); + + var valueParams = new List(); + for (var i = 0; i < valuesList.Count; i++) + { + var (_, val) = GetValueColumnAndParameter(valuesList[i]); + var valueParam = AddParameter($"array_in_{_parameterCounter}_{tableIndex}_{i}", val); + valueParams.Add(valueParam); + } + + var inClause = string.Join(", ", valueParams); + return $"sv{subqueryId}_{tableIndex}.{columnName} IN ({inClause})"; + } + else + { + var (columnName, paramValue) = GetValueColumnAndParameter(condition.Value); + var valueParam = AddParameter($"array_value_{_parameterCounter}_{tableIndex}", paramValue); + var likeEscape = condition.Operator.Contains("LIKE", StringComparison.OrdinalIgnoreCase) + ? dialect.LikeEscapeClause + : ""; + return $"sv{subqueryId}_{tableIndex}.{columnName} {condition.Operator} {valueParam}{likeEscape}"; + } + } + + private static (string ColumnName, object ParameterValue) GetValueColumnAndParameter(object value) => value switch + { + Guid g => ("guid_value", g), + string => ("string_value", value), + decimal d => ("number_value", d), + DateTimeOffset dto => ("datetime_value", dto), + DateTime dt => ("datetime_value", new DateTimeOffset(dt.ToUniversalTime())), + bool b => ("boolean_value", b), + IEnumerable => ("", value), // For IN operator, return the collection as-is + _ => throw new NotSupportedException($"Value type {value.GetType().Name} is not supported.") + }; + + private string AddParameter(string name, object value) + { + var paramName = $"@{name}"; + dialect.AddParameter(command, paramName, value); + return paramName; + } + + private sealed record ArrayItemCondition(string FieldPath, string Operator, object Value); + } +} diff --git a/storage/src/Storage/Internal/Querying/SystemFields.cs b/storage/src/Storage/Internal/Querying/SystemFields.cs new file mode 100644 index 000000000..dfa9f3774 --- /dev/null +++ b/storage/src/Storage/Internal/Querying/SystemFields.cs @@ -0,0 +1,65 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.Internal.Querying; + +/// +/// Constants for system field paths that map to entity-level columns +/// rather than EAV search values. +/// +public static class SystemFields +{ + /// + /// The internal (upper-case) field path for the entity creation timestamp. + /// + public const string Created = "CREATEDAT"; + + /// + /// The internal (upper-case) field path for the entity last-updated timestamp. + /// + public const string LastUpdated = "LASTUPDATEDAT"; + + /// + /// The public attribute name for the entity creation timestamp (lowercase, used in SCIM filters and sort). + /// + public const string CreatedAttributeName = "created_at"; + + /// + /// The public attribute name for the entity last-updated timestamp (lowercase, used in SCIM filters and sort). + /// + public const string LastUpdatedAttributeName = "last_updated_at"; + + /// + /// Ready-made for querying/sorting by creation timestamp. + /// + public static readonly DateTimeField CreatedAtField = new(Created); + + /// + /// Ready-made for querying/sorting by last-updated timestamp. + /// + public static readonly DateTimeField LastUpdatedAtField = new(LastUpdated); + + /// + /// Returns true if the given field path is a system field (case-insensitive). + /// Matches both internal paths (CREATEDAT, LASTUPDATEDAT) and public attribute names (created_at, last_updated_at). + /// + public static bool IsSystemField(string fieldPath) => + string.Equals(fieldPath, Created, StringComparison.OrdinalIgnoreCase) || + string.Equals(fieldPath, LastUpdated, StringComparison.OrdinalIgnoreCase) || + string.Equals(fieldPath, CreatedAttributeName, StringComparison.OrdinalIgnoreCase) || + string.Equals(fieldPath, LastUpdatedAttributeName, StringComparison.OrdinalIgnoreCase); + + /// + /// Returns true if the given attribute name is a reserved system attribute. + /// Checks both public aliases (created_at, last_updated_at) and internal paths (CREATEDAT, LASTUPDATEDAT) + /// using case-insensitive comparison to prevent user-defined attributes from colliding + /// with system fields after upper-case normalization. + /// + public static bool IsReservedAttributeName(string attributeName) => + string.Equals(attributeName, CreatedAttributeName, StringComparison.OrdinalIgnoreCase) || + string.Equals(attributeName, LastUpdatedAttributeName, StringComparison.OrdinalIgnoreCase) || + string.Equals(attributeName, Created, StringComparison.OrdinalIgnoreCase) || + string.Equals(attributeName, LastUpdated, StringComparison.OrdinalIgnoreCase); +} diff --git a/storage/src/Storage/Internal/StorageBuilderExtensions.cs b/storage/src/Storage/Internal/StorageBuilderExtensions.cs new file mode 100644 index 000000000..15319f3cc --- /dev/null +++ b/storage/src/Storage/Internal/StorageBuilderExtensions.cs @@ -0,0 +1,28 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.Internal; + +public static class StorageBuilderExtensions +{ + extension(IServiceCollection services) + { + /// + /// Configures storage services using the provided builder callback. + /// + public IServiceCollection AddStorageInternal(Action configure) + { + ArgumentNullException.ThrowIfNull(configure); + var builder = new StorageBuilder(services); + configure(builder); + return services; + } + } + + private sealed class StorageBuilder(IServiceCollection services) : IStorageBuilder + { + public IServiceCollection Services { get; } = services; + } +} diff --git a/storage/src/Storage/Internal/StorageConstants.cs b/storage/src/Storage/Internal/StorageConstants.cs new file mode 100644 index 000000000..4a92ad95d --- /dev/null +++ b/storage/src/Storage/Internal/StorageConstants.cs @@ -0,0 +1,12 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +internal static class StorageConstants +{ + /// + /// Maximum number of expired entities that can be purged in a single batch. + /// + internal const int TtlCleanupMaxBatchSize = 1000; +} diff --git a/storage/src/Storage/Internal/StoreBase.cs b/storage/src/Storage/Internal/StoreBase.cs new file mode 100644 index 000000000..2b050dd26 --- /dev/null +++ b/storage/src/Storage/Internal/StoreBase.cs @@ -0,0 +1,21 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +/// +/// Base class for store implementations that provides the space identifier. +/// Stores derive from this to receive the space context at construction time +/// rather than reading it from an ambient ISpaceContextAccessor. +/// +internal abstract class StoreBase +{ + /// + /// The space identifier that scopes all operations performed by this store instance. + /// + protected PoolId PoolId { get; private set; } = -1; + + public void SetPoolId(PoolId poolId) => PoolId = poolId; +} + + diff --git a/storage/src/Storage/Internal/StoreServiceCollectionExtensions.cs b/storage/src/Storage/Internal/StoreServiceCollectionExtensions.cs new file mode 100644 index 000000000..8b2e99afa --- /dev/null +++ b/storage/src/Storage/Internal/StoreServiceCollectionExtensions.cs @@ -0,0 +1,52 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Builder; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; + +namespace Duende.Storage.Internal; + +internal static class StoreServiceCollectionExtensions +{ + extension(IServiceCollection services) + { + /// + /// Registers store services with a specific service key for multi-store scenarios. + /// + internal IServiceCollection AddStore(object serviceKey) + where TStoreBase : IStore, IDatabaseSchema + { + services.TryAddSingleton(_ => TimeProvider.System); + services.TryAddSingleton(); + _ = services.AddKeyedTransient(serviceKey, + (provider, _) => new PooledStore(provider, serviceKey)); + _ = services.AddKeyedSingleton(serviceKey); + + _ = services.AddKeyedTransient(serviceKey, + (sp, _) => sp.GetRequiredKeyedService(serviceKey)); + _ = services.AddKeyedTransient(serviceKey, + (sp, _) => sp.GetRequiredKeyedService(serviceKey)); + return services; + } + + /// + /// Registers store services without a service key for single-store scenarios. + /// + internal IServiceCollection AddStore() + where TStoreBase : IStore, IDatabaseSchema + { + services.TryAddSingleton(_ => TimeProvider.System); + services.TryAddSingleton(); + _ = services.AddTransient(provider => + new PooledStore(provider, null)); + _ = services.AddSingleton(); + + _ = services.AddTransient(sp => + sp.GetRequiredService()); + _ = services.AddTransient(sp => + sp.GetRequiredService()); + return services; + } + } +} diff --git a/storage/src/Storage/Internal/Telemetry/StorageMetrics.cs b/storage/src/Storage/Internal/Telemetry/StorageMetrics.cs new file mode 100644 index 000000000..53a17b659 --- /dev/null +++ b/storage/src/Storage/Internal/Telemetry/StorageMetrics.cs @@ -0,0 +1,89 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Diagnostics.Metrics; + +namespace Duende.Storage.Internal.Telemetry; + +/// +/// Records storage metrics using System.Diagnostics.Metrics. +/// +public sealed class StorageMetrics : IDisposable +{ + private readonly Meter _meter; + private readonly Counter _operationCounter; + private readonly Histogram _operationDuration; + + public StorageMetrics() + { + _meter = new Meter(StorageTelemetryConstants.MeterName, StorageTracing.ServiceVersion); + _operationCounter = _meter.CreateCounter( + StorageTelemetryConstants.Instruments.OperationCount, + description: "Number of storage operations executed."); + _operationDuration = _meter.CreateHistogram( + StorageTelemetryConstants.Instruments.OperationDuration, + unit: "s", + description: "Duration of storage operations in seconds."); + } + + public void RecordSuccess(string operation, string dbSystem, string? entityType) + { + if (entityType != null) + { + _operationCounter.Add(1, + new(StorageTelemetryConstants.Tags.Operation, operation), + new(StorageTelemetryConstants.Tags.DbSystem, dbSystem), + new(StorageTelemetryConstants.Tags.EntityType, entityType), + new(StorageTelemetryConstants.Tags.Result, StorageTelemetryConstants.TagValues.Success)); + } + else + { + _operationCounter.Add(1, + new(StorageTelemetryConstants.Tags.Operation, operation), + new(StorageTelemetryConstants.Tags.DbSystem, dbSystem), + new(StorageTelemetryConstants.Tags.Result, StorageTelemetryConstants.TagValues.Success)); + } + } + + public void RecordError(string operation, string dbSystem, Exception ex, string? entityType) + { + if (entityType != null) + { + _operationCounter.Add(1, + new(StorageTelemetryConstants.Tags.Operation, operation), + new(StorageTelemetryConstants.Tags.DbSystem, dbSystem), + new(StorageTelemetryConstants.Tags.EntityType, entityType), + new(StorageTelemetryConstants.Tags.Result, StorageTelemetryConstants.TagValues.Error), + new(StorageTelemetryConstants.Tags.ErrorType, ex.GetType().Name)); + } + else + { + _operationCounter.Add(1, + new(StorageTelemetryConstants.Tags.Operation, operation), + new(StorageTelemetryConstants.Tags.DbSystem, dbSystem), + new(StorageTelemetryConstants.Tags.Result, StorageTelemetryConstants.TagValues.Error), + new(StorageTelemetryConstants.Tags.ErrorType, ex.GetType().Name)); + } + } + + public void RecordDuration(string operation, double durationSeconds, string dbSystem, string result, string? entityType) + { + if (entityType != null) + { + _operationDuration.Record(durationSeconds, + new(StorageTelemetryConstants.Tags.Operation, operation), + new(StorageTelemetryConstants.Tags.DbSystem, dbSystem), + new(StorageTelemetryConstants.Tags.EntityType, entityType), + new(StorageTelemetryConstants.Tags.Result, result)); + } + else + { + _operationDuration.Record(durationSeconds, + new(StorageTelemetryConstants.Tags.Operation, operation), + new(StorageTelemetryConstants.Tags.DbSystem, dbSystem), + new(StorageTelemetryConstants.Tags.Result, result)); + } + } + + public void Dispose() => _meter.Dispose(); +} diff --git a/storage/src/Storage/Internal/Telemetry/StorageTelemetryConstants.cs b/storage/src/Storage/Internal/Telemetry/StorageTelemetryConstants.cs new file mode 100644 index 000000000..f23b09480 --- /dev/null +++ b/storage/src/Storage/Internal/Telemetry/StorageTelemetryConstants.cs @@ -0,0 +1,63 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal.Telemetry; + +/// +/// Constants for storage telemetry tag keys, values, and instrument names. +/// +public static class StorageTelemetryConstants +{ + public const string MeterName = "Duende.Storage"; + + public static class Instruments + { + public const string OperationCount = "duende.storage.operation.count"; + public const string OperationDuration = "duende.storage.operation.duration"; + } + + public static class Tags + { + public const string Operation = "duende.storage.operation"; + public const string DbSystem = "db.system"; + public const string EntityType = "duende.storage.entity_type"; + public const string Result = "duende.storage.result"; + public const string ErrorType = "error.type"; + } + + public static class TagValues + { + public const string Success = "success"; + public const string Error = "error"; + public const string Unknown = "unknown"; + } + + public static class DatabaseProviders + { + public const string MsSql = "mssql"; + public const string PostgreSql = "postgresql"; + public const string Sqlite = "sqlite"; + public const string InMemory = "in_memory"; + } + +#pragma warning disable CA1724 // Type name conflicts with namespace + public static class Operations +#pragma warning restore CA1724 + { + public const string Create = "create"; + public const string Read = "read"; + public const string ReadMany = "read_many"; + public const string Update = "update"; + public const string Delete = "delete"; + public const string Link = "link"; + public const string Unlink = "unlink"; + public const string PurgeExpired = "purge_expired"; + public const string Batch = "batch"; + public const string OutboxGet = "outbox_get"; + public const string OutboxDelete = "outbox_delete"; + public const string Query = "query"; + public const string QueryFields = "query_fields"; + public const string QueryLinks = "query_links"; + public const string Count = "count"; + } +} diff --git a/storage/src/Storage/Internal/Telemetry/StorageTracing.cs b/storage/src/Storage/Internal/Telemetry/StorageTracing.cs new file mode 100644 index 000000000..71bcfcd5a --- /dev/null +++ b/storage/src/Storage/Internal/Telemetry/StorageTracing.cs @@ -0,0 +1,22 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Diagnostics; + +namespace Duende.Storage.Internal.Telemetry; + +/// +/// Constants and ActivitySources for tracing storage operations. +/// +internal static class StorageTracing +{ + private static readonly Version? AssemblyVersion = typeof(StorageTracing).Assembly.GetName().Version; + + public static string ServiceVersion { get; } = AssemblyVersion is { } v + ? $"{v.Major}.{v.Minor}.{v.Build}" + : "0.0.0"; + + public static ActivitySource ActivitySource { get; } = new(SourceName, ServiceVersion); + + public const string SourceName = "Duende.Storage"; +} diff --git a/storage/src/Storage/Internal/TypedDso.cs b/storage/src/Storage/Internal/TypedDso.cs new file mode 100644 index 000000000..382659369 --- /dev/null +++ b/storage/src/Storage/Internal/TypedDso.cs @@ -0,0 +1,18 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +public sealed record TypedDso +{ + public static TypedDso For(TDso value) where TDso : IDataStorageObject => new TypedDso + { + Value = value, + EntityType = TDso.DsoVersion.EntityType, + Version = TDso.DsoVersion + }; + + public required IDataStorageObject Value { get; init; } + public required EntityType EntityType { get; init; } + public required DataStorageObjectVersion Version { get; init; } +} diff --git a/storage/src/Storage/Internal/UnlinkResult.cs b/storage/src/Storage/Internal/UnlinkResult.cs new file mode 100644 index 000000000..c1d08943a --- /dev/null +++ b/storage/src/Storage/Internal/UnlinkResult.cs @@ -0,0 +1,13 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Internal; + +/// +/// The result of an Unlink operation on . +/// +public enum UnlinkResult +{ + /// The link was removed, or did not exist (idempotent). + Success, +} diff --git a/storage/src/Storage/Internal/ValueObjects/CharsetExtensions.g.cs b/storage/src/Storage/Internal/ValueObjects/CharsetExtensions.g.cs new file mode 100644 index 000000000..8b845f53a --- /dev/null +++ b/storage/src/Storage/Internal/ValueObjects/CharsetExtensions.g.cs @@ -0,0 +1,101 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System; +using System.Text; + +namespace Duende.Storage; + +/// +/// Extension methods for CharSet validation +/// +internal static class CharSetExtensions +{ + /// + /// Defines the set of symbols for the 'Symbols' flag. + /// + private const string SymbolSet = "!@#$%^&*()_+-=[]{}|;:',.<>/?"; + + /// + /// Gets the combined string of all allowed characters for a given CharSet. + /// + internal static string GetAllowedCharacters(this CharSet charset) + { + var sb = new StringBuilder(); + + if (charset.HasFlag(CharSet.LowercaseLatin)) + { + sb.Append("abcdefghijklmnopqrstuvwxyz"); + } + if (charset.HasFlag(CharSet.UppercaseLatin)) + { + sb.Append("ABCDEFGHIJKLMNOPQRSTUVWXYZ"); + } + if (charset.HasFlag(CharSet.Digits)) + { + sb.Append("0123456789"); + } + if (charset.HasFlag(CharSet.Symbols)) + { + sb.Append(SymbolSet); + } + + return sb.ToString(); + } + + /// + /// Checks if a string contains ONLY characters defined by the CharSet. + /// + /// The charset to validate against. + /// The string to check. + /// True if the string is null, empty, or contains only allowed characters. False otherwise. + internal static bool IsMatch(this CharSet charset, string input) + { + // An empty or null string contains no *invalid* characters. + if (string.IsNullOrEmpty(input)) + { + return true; + } + + // Cache the flags for quick lookups inside the loop + bool allowLower = charset.HasFlag(CharSet.LowercaseLatin); + bool allowUpper = charset.HasFlag(CharSet.UppercaseLatin); + bool allowDigits = charset.HasFlag(CharSet.Digits); + bool allowSymbols = charset.HasFlag(CharSet.Symbols); + + foreach (char c in input) + { + // We use a series of 'continue' statements. + // If any condition is met, the character is valid, and we check the next. + + // Using BCL methods is faster than c >= 'a' && c <= 'z' + if (allowLower && char.IsAsciiLetterLower(c)) + { + continue; + } + if (allowUpper && char.IsAsciiLetterUpper(c)) + { + continue; + } + if (allowDigits && char.IsAsciiDigit(c)) + { + continue; + } + + // For symbols, checking against the string is the clearest way + if (allowSymbols && SymbolSet.Contains(c)) + { + continue; + } + + // If we get here, the character 'c' did not match any + // allowed set, so the string is not a match. + return false; + } + + // If the loop finishes, all characters were valid. + return true; + } +} diff --git a/storage/src/Storage/Internal/ValueObjects/IValueOf.g.cs b/storage/src/Storage/Internal/ValueObjects/IValueOf.g.cs new file mode 100644 index 000000000..6d930e499 --- /dev/null +++ b/storage/src/Storage/Internal/ValueObjects/IValueOf.g.cs @@ -0,0 +1,43 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage; + +/// +/// Provides type-safe access to the underlying value of a value object. +/// +/// The type of the underlying value. +internal interface IValueOf +{ + /// + /// Gets the underlying value. + /// + T Value { get; } +} + +/// +/// Provides type-safe access to the underlying value of a value object, +/// along with creation capabilities. +/// +/// The value object type itself. +/// The type of the underlying value. +internal interface IValueOf : IValueOf where TSelf : IValueOf +{ + /// + /// Creates a value object from a string representation. + /// + static abstract TSelf Create(string s); + + /// + /// Tries to create a value object from a string representation. + /// + static abstract bool TryCreate(string? s, [NotNullWhen(true)] out TSelf? result); +} + +internal interface IStringValue : IValueOf where TSelf : IStringValue +{ +} diff --git a/storage/src/Storage/Internal/ValueObjects/ValueOfTypeConverter.g.cs b/storage/src/Storage/Internal/ValueObjects/ValueOfTypeConverter.g.cs new file mode 100644 index 000000000..a29f1ae60 --- /dev/null +++ b/storage/src/Storage/Internal/ValueObjects/ValueOfTypeConverter.g.cs @@ -0,0 +1,51 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System; +using System.ComponentModel; +using System.Globalization; + +namespace Duende.Storage; + +/// +/// Generic type converter for value objects. +/// Enables ASP.NET Core model binding and IConfiguration support. +/// +internal class ValueOfTypeConverter : TypeConverter where TValue : IValueOf +{ + public override bool CanConvertFrom(ITypeDescriptorContext? context, Type sourceType) + { + return sourceType == typeof(string) || base.CanConvertFrom(context, sourceType); + } + + public override object? ConvertFrom(ITypeDescriptorContext? context, CultureInfo? culture, object value) + { + if (value is string stringValue) + { + if (TValue.TryCreate(stringValue, out var result)) + { + return result; + } + throw new FormatException($"'{stringValue}' is not a valid {typeof(TValue).Name}."); + } + return base.ConvertFrom(context, culture!, value); + } + + public override bool CanConvertTo(ITypeDescriptorContext? context, Type? destinationType) + { + return destinationType == typeof(string) || base.CanConvertTo(context, destinationType); + } + + public override object? ConvertTo(ITypeDescriptorContext? context, CultureInfo? culture, object? value, Type destinationType) + { + if (destinationType == typeof(string) && value is TValue typedValue) + { + return (typedValue.Value is IFormattable formattable) + ? formattable.ToString(null, CultureInfo.InvariantCulture) + : typedValue.Value?.ToString(); + } + return base.ConvertTo(context, culture, value, destinationType); + } +} diff --git a/storage/src/Storage/Pagination/ContinuationToken.cs b/storage/src/Storage/Pagination/ContinuationToken.cs new file mode 100644 index 000000000..6bf392bf4 --- /dev/null +++ b/storage/src/Storage/Pagination/ContinuationToken.cs @@ -0,0 +1,16 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Querying; + +namespace Duende.Storage.Pagination; + +/// +/// An opaque continuation token used for cursor-based pagination. +/// Returned in and consumed by . +/// +[StringValue] +public partial record ContinuationToken +{ + public const string Beginning = "Beginning"; +} diff --git a/storage/src/Storage/Pagination/ContinuationToken.g.cs b/storage/src/Storage/Pagination/ContinuationToken.g.cs new file mode 100644 index 000000000..1c1c99a85 --- /dev/null +++ b/storage/src/Storage/Pagination/ContinuationToken.g.cs @@ -0,0 +1,62 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Collections.Generic; +using Duende.Storage; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.Pagination; + +[System.ComponentModel.TypeConverter(typeof(ValueOfTypeConverter))] +partial record ContinuationToken : IStringValue +{ + // Constructor for controlled creation + private ContinuationToken(string value) => Value = value; + + public string Value { get; } + + public static ContinuationToken Create(string s) + { + if (!TryCreate(s, out var result, out var errors)) + { + throw new FormatException($"The value '{s}' is not a valid ContinuationToken. {string.Join(" ", errors)}"); + } + return result; + } + + public static bool TryCreate(string? s, [NotNullWhen(true)] out ContinuationToken? result) + => TryCreate(s, out result, out _); + + public static bool TryCreate(string? s, [NotNullWhen(true)] out ContinuationToken? result, [NotNullWhen(false)] out IReadOnlyList? errors) + { + result = null; + errors = null; + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["A value is required."]; + return false; + } + + result = new ContinuationToken(s); + return true; + } + + public static implicit operator ContinuationToken(string value) => Create(value); + + public override string ToString() => Value; + + public static ContinuationToken? CreateOrDefault(string? input) + { + if (string.IsNullOrEmpty(input)) + { + return null; + } + + return Create(input); + } + + internal static ContinuationToken Load(string value) => new ContinuationToken(value); + +} diff --git a/storage/src/Storage/Pagination/ContinuationTokenDataRange.cs b/storage/src/Storage/Pagination/ContinuationTokenDataRange.cs new file mode 100644 index 000000000..b9f6bd407 --- /dev/null +++ b/storage/src/Storage/Pagination/ContinuationTokenDataRange.cs @@ -0,0 +1,29 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Pagination; + +public sealed record ContinuationTokenDataRange +{ + /// + /// Represents cursor-based pagination: a continuation token and page size. + /// + /// The continuation token to resume from, or for the first page. + /// The number of items per page. + public ContinuationTokenDataRange(ContinuationToken? start, DataRangeSize? size) + { + Start = start ?? ContinuationToken.Beginning; + Size = size ?? DataRangeSize.Default; + } + + /// + /// Creates a starting from the first page. + /// + public static ContinuationTokenDataRange Beginning(DataRangeSize? size = null) => new(ContinuationToken.Beginning, size); + + /// The continuation token indicating where to resume. + public ContinuationToken Start { get; init; } + + /// The number of items per page. + public DataRangeSize Size { get; init; } +} diff --git a/storage/src/Storage/Pagination/DataRange.cs b/storage/src/Storage/Pagination/DataRange.cs new file mode 100644 index 000000000..3278364c2 --- /dev/null +++ b/storage/src/Storage/Pagination/DataRange.cs @@ -0,0 +1,116 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Pagination; + +#pragma warning disable CA2225 // Allow implicit conversions from related types for ease of use. + +/// +/// Specifies where to begin fetching results in a query. +/// Can be created from a , +/// a , or an . +/// +public sealed class DataRange +{ + private DataRange() + { + } + + public ContinuationTokenDataRange? TokenValue { get; private init; } + + public OffsetDataRange? OffsetValue { get; private init; } + + public PagedDataRange? PageValue { get; private init; } + + /// + /// Implicitly converts a to a . + /// + public static implicit operator DataRange(ContinuationTokenDataRange? token) => new() + { + TokenValue = token + }; + + /// + /// Implicitly converts a to a . + /// + public static implicit operator DataRange(PagedDataRange page) => new() + { + PageValue = page + }; + + /// + /// Implicitly converts an to a . + /// + public static implicit operator DataRange(OffsetDataRange offsetDataRange) => new() + { + OffsetValue = offsetDataRange + }; + + + /// + /// Creates a from page-number-based pagination. + /// + public static DataRange FromPage(PageNumber page) => new() + { + PageValue = new PagedDataRange(page, null) + }; + + + /// + /// Creates a from page-number-based pagination. + /// + public static DataRange FromPage(PageNumber? page, DataRangeSize? size) => new() + { + PageValue = new PagedDataRange(page, size) + }; + + /// + /// Creates a from page-number-based pagination. + /// + public static DataRange FromPage(PagedDataRange page) => new() + { + PageValue = page + }; + + public static DataRange FromOffset(OffsetSkip? skip) => new() + { + OffsetValue = new OffsetDataRange(skip, null) + }; + + public static DataRange FromOffset(OffsetSkip? skip, DataRangeSize? size) => new() + { + OffsetValue = new OffsetDataRange(skip, size) + }; + + /// + /// Creates a from offset-based pagination. + /// + public static DataRange FromOffset(OffsetDataRange offsetDataRange) => new() + { + OffsetValue = offsetDataRange + }; + + /// + /// Creates a from a continuation token. + /// + public static DataRange FromContinuationToken(ContinuationTokenDataRange? token) => new() + { + TokenValue = token + }; + + /// + /// Creates a from a continuation token. + /// + public static DataRange FromContinuationToken(ContinuationToken? token) => new() + { + TokenValue = new ContinuationTokenDataRange(token, null) + }; + + /// + /// Creates a from a continuation token. + /// + public static DataRange FromContinuationToken(ContinuationToken? token, DataRangeSize? size) => new() + { + TokenValue = new ContinuationTokenDataRange(token, size) + }; +} diff --git a/storage/src/Storage/Pagination/DataRangeSize.cs b/storage/src/Storage/Pagination/DataRangeSize.cs new file mode 100644 index 000000000..d4a792a65 --- /dev/null +++ b/storage/src/Storage/Pagination/DataRangeSize.cs @@ -0,0 +1,35 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Pagination; + +/// +/// The number of items per page or batch. Must be between 1 and (1000). +/// +[ValueOf] +public partial record DataRangeSize +{ + /// The default page size (25). + public static readonly DataRangeSize Default = 25; + + /// The maximum allowed page size. + public const int MaxValue = 1000; + + internal static bool TryValidate(int value, out IReadOnlyList? errors) + { + errors = null; + if (value < 1) + { + errors = ["Page size must be at least 1."]; + return false; + } + + if (value > MaxValue) + { + errors = [$"Page size must not exceed {MaxValue}."]; + return false; + } + + return true; + } +} diff --git a/storage/src/Storage/Pagination/DataRangeSize.g.cs b/storage/src/Storage/Pagination/DataRangeSize.g.cs new file mode 100644 index 000000000..738c85c05 --- /dev/null +++ b/storage/src/Storage/Pagination/DataRangeSize.g.cs @@ -0,0 +1,82 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Collections.Generic; +using Duende.Storage; +using System.Globalization; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.Pagination; + +[System.ComponentModel.TypeConverter(typeof(ValueOfTypeConverter))] +partial record DataRangeSize : IValueOf +{ + // Constructor for controlled creation + private DataRangeSize(global::System.Int32 value) => Value = value; + + public global::System.Int32 Value { get; } + + public static DataRangeSize Create(string s) + { + if (!TryCreate(s, out var result, out var errors)) + { + throw new FormatException($"The value '{s}' is not a valid '{nameof(DataRangeSize)}'. {string.Join(" ", errors)}"); + } + return result; + } + + public static bool TryCreate(string? s, [NotNullWhen(true)] out DataRangeSize? result) + => TryCreate(s, out result, out _); + + public static bool TryCreate(string? s, [NotNullWhen(true)] out DataRangeSize? result, [NotNullWhen(false)] out IReadOnlyList? errors) + { + result = null; + errors = null; + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["A value is required."]; + return false; + } + + if (global::System.Int32.TryParse(s, CultureInfo.InvariantCulture, out var value)) + { + if (!TryValidate(value, out var tryValidateErrors)) + { + errors = tryValidateErrors is { Count: > 0 } ? tryValidateErrors : [$"The value is not a valid '{nameof(DataRangeSize)}'."]; + return false; + } + var instance = new DataRangeSize(value); + result = instance; + return true; + } + + errors = ["The value could not be parsed."]; + return false; + } + + public static implicit operator DataRangeSize(global::System.Int32 value) + { + if (!TryValidate(value, out var errors)) + { + var errorMessage = $"The value '{value}' is not a valid '{nameof(DataRangeSize)}'. {string.Join(" ", errors ?? [])}"; + throw new FormatException(errorMessage); + } + return new DataRangeSize(value); + } + + public override string ToString() => Value.ToString(null, CultureInfo.InvariantCulture); + + public static DataRangeSize? CreateOrDefault(string? input) + { + if (string.IsNullOrEmpty(input)) + { + return null; + } + + return Create(input); + } + + internal static DataRangeSize Load(global::System.Int32 value) => new DataRangeSize(value); +} diff --git a/storage/src/Storage/Pagination/OffsetDataRange.cs b/storage/src/Storage/Pagination/OffsetDataRange.cs new file mode 100644 index 000000000..2e332c1f5 --- /dev/null +++ b/storage/src/Storage/Pagination/OffsetDataRange.cs @@ -0,0 +1,32 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Pagination; + +/// +/// Represents offset-based pagination: a 0-based row offset and batch size. +/// +public sealed record OffsetDataRange +{ + /// + /// Represents offset-based pagination: a 0-based row offset and batch size. + /// + /// The 0-based row offset. + /// The number of items to return. + public OffsetDataRange(OffsetSkip? skip, DataRangeSize? take) + { + Skip = skip ?? 0; + Take = take ?? DataRangeSize.Default; + } + + /// + /// Creates an starting from the first row. + /// + public static OffsetDataRange First(DataRangeSize? pageSize = null) => new(0L, pageSize); + + /// The 0-based row offset. + public OffsetSkip Skip { get; init; } + + /// The number of items to return. + public DataRangeSize Take { get; init; } +} diff --git a/storage/src/Storage/Pagination/OffsetSkip.cs b/storage/src/Storage/Pagination/OffsetSkip.cs new file mode 100644 index 000000000..0c245b732 --- /dev/null +++ b/storage/src/Storage/Pagination/OffsetSkip.cs @@ -0,0 +1,24 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Pagination; + +/// +/// A starting position for pagination. Represents a 0-based row offset or a 1-based page number, +/// depending on context. +/// +[ValueOf] +public partial record OffsetSkip +{ + internal static bool TryValidate(long value, out IReadOnlyList? errors) + { + errors = null; + if (value < 0) + { + errors = ["Start position must be at least 0."]; + return false; + } + + return true; + } +} diff --git a/storage/src/Storage/Pagination/OffsetSkip.g.cs b/storage/src/Storage/Pagination/OffsetSkip.g.cs new file mode 100644 index 000000000..b635e3e03 --- /dev/null +++ b/storage/src/Storage/Pagination/OffsetSkip.g.cs @@ -0,0 +1,82 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Collections.Generic; +using Duende.Storage; +using System.Globalization; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.Pagination; + +[System.ComponentModel.TypeConverter(typeof(ValueOfTypeConverter))] +partial record OffsetSkip : IValueOf +{ + // Constructor for controlled creation + private OffsetSkip(global::System.Int64 value) => Value = value; + + public global::System.Int64 Value { get; } + + public static OffsetSkip Create(string s) + { + if (!TryCreate(s, out var result, out var errors)) + { + throw new FormatException($"The value '{s}' is not a valid '{nameof(OffsetSkip)}'. {string.Join(" ", errors)}"); + } + return result; + } + + public static bool TryCreate(string? s, [NotNullWhen(true)] out OffsetSkip? result) + => TryCreate(s, out result, out _); + + public static bool TryCreate(string? s, [NotNullWhen(true)] out OffsetSkip? result, [NotNullWhen(false)] out IReadOnlyList? errors) + { + result = null; + errors = null; + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["A value is required."]; + return false; + } + + if (global::System.Int64.TryParse(s, CultureInfo.InvariantCulture, out var value)) + { + if (!TryValidate(value, out var tryValidateErrors)) + { + errors = tryValidateErrors is { Count: > 0 } ? tryValidateErrors : [$"The value is not a valid '{nameof(OffsetSkip)}'."]; + return false; + } + var instance = new OffsetSkip(value); + result = instance; + return true; + } + + errors = ["The value could not be parsed."]; + return false; + } + + public static implicit operator OffsetSkip(global::System.Int64 value) + { + if (!TryValidate(value, out var errors)) + { + var errorMessage = $"The value '{value}' is not a valid '{nameof(OffsetSkip)}'. {string.Join(" ", errors ?? [])}"; + throw new FormatException(errorMessage); + } + return new OffsetSkip(value); + } + + public override string ToString() => Value.ToString(null, CultureInfo.InvariantCulture); + + public static OffsetSkip? CreateOrDefault(string? input) + { + if (string.IsNullOrEmpty(input)) + { + return null; + } + + return Create(input); + } + + internal static OffsetSkip Load(global::System.Int64 value) => new OffsetSkip(value); +} diff --git a/storage/src/Storage/Pagination/PageNumber.cs b/storage/src/Storage/Pagination/PageNumber.cs new file mode 100644 index 000000000..328f9bbed --- /dev/null +++ b/storage/src/Storage/Pagination/PageNumber.cs @@ -0,0 +1,24 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Pagination; + +/// +/// A starting position for pagination. Represents a 0-based row offset or a 1-based page number, +/// depending on context. +/// +[ValueOf] +public partial record PageNumber +{ + internal static bool TryValidate(long value, out IReadOnlyList? errors) + { + errors = null; + if (value < 1) + { + errors = ["PageNumber must be at least 1."]; + return false; + } + + return true; + } +} diff --git a/storage/src/Storage/Pagination/PageNumber.g.cs b/storage/src/Storage/Pagination/PageNumber.g.cs new file mode 100644 index 000000000..27764a6de --- /dev/null +++ b/storage/src/Storage/Pagination/PageNumber.g.cs @@ -0,0 +1,82 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Collections.Generic; +using Duende.Storage; +using System.Globalization; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage.Pagination; + +[System.ComponentModel.TypeConverter(typeof(ValueOfTypeConverter))] +partial record PageNumber : IValueOf +{ + // Constructor for controlled creation + private PageNumber(global::System.Int32 value) => Value = value; + + public global::System.Int32 Value { get; } + + public static PageNumber Create(string s) + { + if (!TryCreate(s, out var result, out var errors)) + { + throw new FormatException($"The value '{s}' is not a valid '{nameof(PageNumber)}'. {string.Join(" ", errors)}"); + } + return result; + } + + public static bool TryCreate(string? s, [NotNullWhen(true)] out PageNumber? result) + => TryCreate(s, out result, out _); + + public static bool TryCreate(string? s, [NotNullWhen(true)] out PageNumber? result, [NotNullWhen(false)] out IReadOnlyList? errors) + { + result = null; + errors = null; + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["A value is required."]; + return false; + } + + if (global::System.Int32.TryParse(s, CultureInfo.InvariantCulture, out var value)) + { + if (!TryValidate(value, out var tryValidateErrors)) + { + errors = tryValidateErrors is { Count: > 0 } ? tryValidateErrors : [$"The value is not a valid '{nameof(PageNumber)}'."]; + return false; + } + var instance = new PageNumber(value); + result = instance; + return true; + } + + errors = ["The value could not be parsed."]; + return false; + } + + public static implicit operator PageNumber(global::System.Int32 value) + { + if (!TryValidate(value, out var errors)) + { + var errorMessage = $"The value '{value}' is not a valid '{nameof(PageNumber)}'. {string.Join(" ", errors ?? [])}"; + throw new FormatException(errorMessage); + } + return new PageNumber(value); + } + + public override string ToString() => Value.ToString(null, CultureInfo.InvariantCulture); + + public static PageNumber? CreateOrDefault(string? input) + { + if (string.IsNullOrEmpty(input)) + { + return null; + } + + return Create(input); + } + + internal static PageNumber Load(global::System.Int32 value) => new PageNumber(value); +} diff --git a/storage/src/Storage/Pagination/PagedDataRange.cs b/storage/src/Storage/Pagination/PagedDataRange.cs new file mode 100644 index 000000000..e7cc12efe --- /dev/null +++ b/storage/src/Storage/Pagination/PagedDataRange.cs @@ -0,0 +1,37 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Pagination; + +/// +/// Represents page-number-based pagination: a 1-based page number and page size. +/// +public sealed record PagedDataRange +{ + /// + /// Represents page-number-based pagination: a 1-based page number and page size. + /// + /// The 1-based page number. + /// The number of items per page. + public PagedDataRange(PageNumber? page, DataRangeSize? pageSize) + { + Page = page ?? 1; + PageSize = pageSize ?? DataRangeSize.Default; + } + + /// + /// Creates a starting from the first page. + /// + public static PagedDataRange First(DataRangeSize pageSize) => new((PageNumber)1, pageSize); + + /// + /// Creates a starting from the first page. + /// + public static PagedDataRange First() => new((PageNumber)1, DataRangeSize.Default); + + /// The 1-based page number. + public PageNumber Page { get; init; } + + /// The number of items per page. + public DataRangeSize PageSize { get; init; } +} diff --git a/storage/src/Storage/Querying/FilterBy.cs b/storage/src/Storage/Querying/FilterBy.cs new file mode 100644 index 000000000..43e2a996a --- /dev/null +++ b/storage/src/Storage/Querying/FilterBy.cs @@ -0,0 +1,82 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Querying; + +/// +/// Specifies search/filter criteria for a query. +/// Accepts a via implicit conversion. +/// Use for typed filter support. +/// +public sealed class FilterBy +{ + private FilterBy() + { + } + + public SearchExpression? SearchExpressionValue { get; init; } + + /// + /// Implicitly converts a to a . + /// + public static implicit operator FilterBy(SearchExpression searchExpression) => FromSearchExpression(searchExpression); + + /// + /// Creates a from a . + /// + public static FilterBy FromSearchExpression(SearchExpression searchExpression) => new() + { + SearchExpressionValue = searchExpression + }; + + /// + /// Creates a from a typed filter object. + /// + public static FilterBy Filter(T filterValue) => FilterBy.FromFilter(filterValue); +} + +/// +/// Specifies search/filter criteria for a query, supporting both +/// and a typed filter . +/// +/// The typed filter type (e.g., ClientFilter, RoleFilter). +#pragma warning disable CA1000 // Do not declare static members on generic types — intentional: implicit operators require static members +public sealed class FilterBy +{ + private FilterBy() + { + } + + internal SearchExpression? SearchExpressionValue { get; init; } + + public T? FilterValue { get; init; } + + /// + /// Implicitly converts a to a . + /// Named alternate: . + /// +#pragma warning disable CA2225 // Named alternate is FilterBy.Filter() on the non-generic class + public static implicit operator FilterBy(SearchExpression searchExpression) => new() + { + SearchExpressionValue = searchExpression + }; + + /// + /// Implicitly converts a typed filter to a . + /// Named alternate: . + /// + public static implicit operator FilterBy(T filterValue) => new() + { + FilterValue = filterValue + }; +#pragma warning restore CA2225 +#pragma warning restore CA1000 + + /// + /// Creates a from a typed filter value. + /// + internal static FilterBy FromFilter(T filterValue) => new() + { + FilterValue = filterValue + }; +} diff --git a/storage/src/Storage/Querying/FilterParseException.cs b/storage/src/Storage/Querying/FilterParseException.cs new file mode 100644 index 000000000..b5be3642c --- /dev/null +++ b/storage/src/Storage/Querying/FilterParseException.cs @@ -0,0 +1,27 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Querying; + +/// +/// Exception thrown when a filter expression cannot be parsed. +/// +public sealed class FilterParseException : Exception +{ + /// + /// Initializes a new instance of . + /// + private FilterParseException() { } + + /// + /// Initializes a new instance of with a message. + /// + public FilterParseException(string message) + : base(message) { } + + /// + /// Initializes a new instance of with a message and inner exception. + /// + public FilterParseException(string message, Exception innerException) + : base(message, innerException) { } +} diff --git a/storage/src/Storage/Querying/QueryRequest.cs b/storage/src/Storage/Querying/QueryRequest.cs new file mode 100644 index 000000000..0139a4f53 --- /dev/null +++ b/storage/src/Storage/Querying/QueryRequest.cs @@ -0,0 +1,170 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Pagination; + +namespace Duende.Storage.Querying; + +/// +/// Encapsulates filter, sort, and pagination parameters for a query operation. +/// +public sealed record QueryRequest +{ + /// A query request with no filter, sort, or range. + public static QueryRequest Empty { get; } = new(); + + /// Optional search/filter criteria. + public FilterBy? Filter { get; init; } + + /// Optional sort criteria. + public SortBy? Sort { get; init; } + + /// Optional pagination parameters. + public DataRange? Range { get; init; } + + public static QueryRequest Create() => Empty; + + public static QueryRequest Create(FilterBy filter) => new() + { + Filter = filter + }; + + public static QueryRequest Create(SortBy sort) => new() + { + Sort = sort + }; + + public static QueryRequest Create(DataRange range) => new() + { + Range = range + }; + + public static QueryRequest Create(FilterBy filter, SortBy sort) => new() + { + Filter = filter, + Sort = sort + }; + + public static QueryRequest Create(FilterBy filter, DataRange range) => new() + { + Filter = filter, + Range = range + }; + + public static QueryRequest Create(SortBy sort, DataRange range) => new() + { + Sort = sort, + Range = range + }; + + public static QueryRequest Create(FilterBy filter, SortBy sort, DataRange range) => new() + { + Filter = filter, + Sort = sort, + Range = range + }; + + public static QueryRequest Create() + where TSort : struct, Enum => new(); + + public static QueryRequest Create(TFilter? filter) + where TSort : struct, Enum => new() + { + Filter = CreateFilter(filter) + }; + + public static QueryRequest Create(FilterBy filter) + where TSort : struct, Enum => new() + { + Filter = filter + }; + + public static QueryRequest Create(SortBy.SortByField sort) + where TSort : struct, Enum => new() + { + Sort = sort + }; + + public static QueryRequest Create(DataRange range) + where TSort : struct, Enum => new() + { + Range = range + }; + + public static QueryRequest Create(TFilter? filter, SortBy.SortByField? sort) + where TSort : struct, Enum => new() + { + Filter = CreateFilter(filter), + Sort = sort + }; + + public static QueryRequest Create(FilterBy filter, SortBy.SortByField sort) + where TSort : struct, Enum => new() + { + Filter = filter, + Sort = sort + }; + + public static QueryRequest Create(TFilter? filter, DataRange? range) + where TSort : struct, Enum => new() + { + Filter = CreateFilter(filter), + Range = range + }; + + public static QueryRequest Create(FilterBy filter, DataRange range) + where TSort : struct, Enum => new() + { + Filter = filter, + Range = range + }; + + public static QueryRequest Create(SortBy.SortByField? sort, DataRange? range) + where TSort : struct, Enum => new() + { + Sort = sort, + Range = range + }; + + public static QueryRequest Create( + TFilter? filter, + SortBy.SortByField? sort, + DataRange? range) + where TSort : struct, Enum => new() + { + Filter = CreateFilter(filter), + Sort = sort, + Range = range + }; + + public static QueryRequest Create( + FilterBy filter, + SortBy.SortByField sort, + DataRange range) + where TSort : struct, Enum => new() + { + Filter = filter, + Sort = sort, + Range = range + }; + + private static FilterBy? CreateFilter(TFilter? filter) => + filter is null ? null : FilterBy.Filter(filter); +} + +/// +/// Encapsulates typed filter, typed sort, and pagination parameters for a query operation. +/// +/// The typed filter type. +/// The typed sort field enum. +public sealed record QueryRequest where TSort : struct, Enum +{ + /// Optional typed filter criteria. + public FilterBy? Filter { get; init; } + + /// Optional typed sort criteria. + public SortBy.SortByField? Sort { get; init; } + + /// Optional pagination parameters. + public DataRange? Range { get; init; } +} diff --git a/storage/src/Storage/Querying/QueryResult.cs b/storage/src/Storage/Querying/QueryResult.cs new file mode 100644 index 000000000..79af27fbf --- /dev/null +++ b/storage/src/Storage/Querying/QueryResult.cs @@ -0,0 +1,68 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Collections; +using Duende.Storage.Pagination; + +namespace Duende.Storage.Querying; + +/// +/// The unified result of a query operation. Works for all pagination modes. +/// Implements so results can be iterated directly. +/// +/// The type of items in the result. +public sealed record QueryResult : IReadOnlyList +{ + /// The items in the current page. + public required IReadOnlyList Items { get; init; } + + /// + /// The continuation token to retrieve the previous page of results. + /// + public ContinuationToken? PreviousToken { get; init; } + + /// + /// The continuation token to retrieve the next page of results. + /// + public ContinuationToken? NextToken { get; init; } + + /// + /// If available, the total number of items. This may be null if the total count + /// is not known or expensive to compute. + /// + public int? TotalCount { get; init; } + + /// + /// If available, the total number of pages. This may be null if the total count + /// is not known or expensive to compute. + /// + public int? TotalPages { get; init; } + + /// + /// Indicates whether there is more data available. + /// + public bool HasMoreData { get; init; } + + /// + public IEnumerator GetEnumerator() => Items.GetEnumerator(); + + /// + IEnumerator IEnumerable.GetEnumerator() => ((IEnumerable)Items).GetEnumerator(); + + /// + int IReadOnlyCollection.Count => Items.Count; + + /// + public TItem this[int index] => Items[index]; + + public QueryResult ConvertTo(Func convert) => + new() + { + Items = Items.Select(convert).ToArray(), + TotalPages = TotalPages, + TotalCount = TotalCount, + HasMoreData = HasMoreData, + NextToken = NextToken, + PreviousToken = PreviousToken + }; +} diff --git a/storage/src/Storage/Querying/SortBy.cs b/storage/src/Storage/Querying/SortBy.cs new file mode 100644 index 000000000..743c36c03 --- /dev/null +++ b/storage/src/Storage/Querying/SortBy.cs @@ -0,0 +1,50 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.EntityAttributeValue; + +namespace Duende.Storage.Querying; + +/// +/// Specifies sort criteria for a query. +/// +public abstract record SortBy +{ + /// The sort direction. + public SortDirection Direction { get; } + + private SortBy(SortDirection direction) => Direction = direction; + + /// + /// Sort by a named attribute (string-based, for dynamic/EAV entities). + /// + public static SortByAttributeCode Attribute(AttributeCode code, SortDirection direction = SortDirection.Ascending) => new(code, direction); + + /// + /// Sort by a typed sort field enum value (for fixed-schema entities). + /// + public static SortByField Field(TField field, SortDirection direction = SortDirection.Ascending) + where TField : struct, Enum => new(field, direction); + + /// Sort by a named attribute code. + public sealed record SortByAttributeCode : SortBy + { + /// The attribute code to sort by. + public AttributeCode Code { get; } + + internal SortByAttributeCode(AttributeCode code, SortDirection direction) : base(direction) + { + ArgumentException.ThrowIfNullOrEmpty(code.Value); + Code = code; + } + } + + /// Sort by a typed enum field. + public sealed record SortByField : SortBy where TField : struct, Enum + { + /// The sort field. + public TField Field { get; } + + internal SortByField(TField field, SortDirection direction) : base(direction) => Field = field; + } +} diff --git a/storage/src/Storage/Querying/SortDirection.cs b/storage/src/Storage/Querying/SortDirection.cs new file mode 100644 index 000000000..c22f37bad --- /dev/null +++ b/storage/src/Storage/Querying/SortDirection.cs @@ -0,0 +1,20 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.Querying; + +/// +/// Specifies the direction for sorting query results. +/// +public enum SortDirection +{ + /// + /// Sort in ascending order (A-Z, 0-9, oldest to newest). + /// + Ascending, + + /// + /// Sort in descending order (Z-A, 9-0, newest to oldest). + /// + Descending +} diff --git a/storage/src/Storage/SchemaVerificationError.cs b/storage/src/Storage/SchemaVerificationError.cs new file mode 100644 index 000000000..cf788bed4 --- /dev/null +++ b/storage/src/Storage/SchemaVerificationError.cs @@ -0,0 +1,21 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage; + +public enum SchemaVerificationErrorKind +{ + MissingTable, + MissingColumn, + WrongType, + MissingIndex, + MissingForeignKey, + MissingUserDefinedType, + Other +} + +public sealed record SchemaVerificationError( + string Table, + string? Column, + string ErrorMessage, + SchemaVerificationErrorKind Kind); diff --git a/storage/src/Storage/SchemaVerificationResult.cs b/storage/src/Storage/SchemaVerificationResult.cs new file mode 100644 index 000000000..171907edc --- /dev/null +++ b/storage/src/Storage/SchemaVerificationResult.cs @@ -0,0 +1,9 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage; + +public sealed record SchemaVerificationResult(IReadOnlyList Errors) +{ + public bool IsValid => Errors.Count == 0; +} diff --git a/storage/src/Storage/SearchExpression.cs b/storage/src/Storage/SearchExpression.cs new file mode 100644 index 000000000..6b43f1e23 --- /dev/null +++ b/storage/src/Storage/SearchExpression.cs @@ -0,0 +1,15 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage; + +/// +/// A SCIM-like search expression string (e.g., displayName eq "Engineers"). +/// Used to filter query results using SCIM filter syntax (RFC 7644 §3.4.2.2). +/// +[StringValue] +public partial record SearchExpression +{ + /// The maximum allowed length of a search expression. + public const int MaxLength = 4000; +} diff --git a/storage/src/Storage/SearchExpression.g.cs b/storage/src/Storage/SearchExpression.g.cs new file mode 100644 index 000000000..699d6b368 --- /dev/null +++ b/storage/src/Storage/SearchExpression.g.cs @@ -0,0 +1,71 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage; + +[System.ComponentModel.TypeConverter(typeof(ValueOfTypeConverter))] +partial record SearchExpression : IStringValue +{ + // Constructor for controlled creation + private SearchExpression(string value) => Value = value; + + public string Value { get; } + + public static SearchExpression Create(string s) + { + if (!TryCreate(s, out var result, out var errors)) + { + throw new FormatException($"The value '{s}' is not a valid SearchExpression. {string.Join(" ", errors)}"); + } + return result; + } + + public static bool TryCreate(string? s, [NotNullWhen(true)] out SearchExpression? result) + => TryCreate(s, out result, out _); + + public static bool TryCreate(string? s, [NotNullWhen(true)] out SearchExpression? result, [NotNullWhen(false)] out IReadOnlyList? errors) + { + result = null; + errors = null; + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["A value is required."]; + return false; + } + + var validationErrors = new List(); + if (s.Length > MaxLength) + { + validationErrors.Add($"Must not exceed {MaxLength} characters."); + } + if (validationErrors.Count > 0) + { + errors = validationErrors; + return false; + } + result = new SearchExpression(s); + return true; + } + + public static implicit operator SearchExpression(string value) => Create(value); + + public override string ToString() => Value; + + public static SearchExpression? CreateOrDefault(string? input) + { + if (string.IsNullOrEmpty(input)) + { + return null; + } + + return Create(input); + } + + internal static SearchExpression Load(string value) => new SearchExpression(value); + +} diff --git a/storage/src/Storage/Storage.csproj b/storage/src/Storage/Storage.csproj new file mode 100644 index 000000000..14da6463e --- /dev/null +++ b/storage/src/Storage/Storage.csproj @@ -0,0 +1,42 @@ + + + + Duende.Storage + Duende.Storage + Duende.Labs.Storage + Duende.Storage + $(NoWarn);CA1062 + + + + + + + + + + + + + + + + + + + + + + + + $([System.String]::Copy('%(Filename)').Replace('.g', '.cs')) + + + $([System.String]::Copy('%(Filename)').Replace('.g', '.cs')) + + + $([System.String]::Copy('%(Filename)').Replace('.g', '.cs')) + + + + diff --git a/storage/src/Storage/UuidV7.cs b/storage/src/Storage/UuidV7.cs new file mode 100644 index 000000000..0880d2143 --- /dev/null +++ b/storage/src/Storage/UuidV7.cs @@ -0,0 +1,41 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage; + +[ValueOf] +public partial record UuidV7 +{ + /// + /// Creates a new UuidV7. + /// + public static UuidV7 New() => new(Guid.CreateVersion7()); + + public static UuidV7 From(Guid value) => value; + + public static bool TryValidate(Guid? input, out IReadOnlyList? errors) + { + errors = null; + + if (input == null) + { + errors = ["No UUID value provided"]; + return false; + } + + var value = input.Value; + if (value.Variant is < 0x8 or > 0xB) + { + errors = [$"Not a valid UUIDV7 Guid: Invalid variant: {value.Variant:X}"]; + return false; + } + + if (value.Version != 7) + { + errors = [$"Not a valid UUIDV7 Guid: Version is {value.Version} but should be 7."]; + return false; + } + + return true; + } +} diff --git a/storage/src/Storage/UuidV7.g.cs b/storage/src/Storage/UuidV7.g.cs new file mode 100644 index 000000000..a2854bbcf --- /dev/null +++ b/storage/src/Storage/UuidV7.g.cs @@ -0,0 +1,81 @@ +// +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. +#nullable enable + +using System.Collections.Generic; +using System.Globalization; +using System.Diagnostics.CodeAnalysis; + +namespace Duende.Storage; + +[System.ComponentModel.TypeConverter(typeof(ValueOfTypeConverter))] +partial record UuidV7 : IValueOf +{ + // Constructor for controlled creation + private UuidV7(global::System.Guid value) => Value = value; + + public global::System.Guid Value { get; } + + public static UuidV7 Create(string s) + { + if (!TryCreate(s, out var result, out var errors)) + { + throw new FormatException($"The value '{s}' is not a valid '{nameof(UuidV7)}'. {string.Join(" ", errors)}"); + } + return result; + } + + public static bool TryCreate(string? s, [NotNullWhen(true)] out UuidV7? result) + => TryCreate(s, out result, out _); + + public static bool TryCreate(string? s, [NotNullWhen(true)] out UuidV7? result, [NotNullWhen(false)] out IReadOnlyList? errors) + { + result = null; + errors = null; + if (string.IsNullOrWhiteSpace(s)) + { + errors = ["A value is required."]; + return false; + } + + if (global::System.Guid.TryParse(s, CultureInfo.InvariantCulture, out var value)) + { + if (!TryValidate(value, out var tryValidateErrors)) + { + errors = tryValidateErrors is { Count: > 0 } ? tryValidateErrors : [$"The value is not a valid '{nameof(UuidV7)}'."]; + return false; + } + var instance = new UuidV7(value); + result = instance; + return true; + } + + errors = ["The value could not be parsed."]; + return false; + } + + public static implicit operator UuidV7(global::System.Guid value) + { + if (!TryValidate(value, out var errors)) + { + var errorMessage = $"The value '{value}' is not a valid '{nameof(UuidV7)}'. {string.Join(" ", errors ?? [])}"; + throw new FormatException(errorMessage); + } + return new UuidV7(value); + } + + public override string ToString() => Value.ToString(null, CultureInfo.InvariantCulture); + + public static UuidV7? CreateOrDefault(string? input) + { + if (string.IsNullOrEmpty(input)) + { + return null; + } + + return Create(input); + } + + internal static UuidV7 Load(global::System.Guid value) => new UuidV7(value); +} diff --git a/storage/src/Storage/ValueOfSourceGeneratorAttributes.cs b/storage/src/Storage/ValueOfSourceGeneratorAttributes.cs new file mode 100644 index 000000000..50b6a14ce --- /dev/null +++ b/storage/src/Storage/ValueOfSourceGeneratorAttributes.cs @@ -0,0 +1,54 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +// These attributes are used by the source generator to generate value objects. +// They must be replicated in each assembly that uses the source generator. +// in the 'root' namespace for that assembly. + +namespace Duende.Storage; + +/// +/// Marks a record as a value object. The source generator will create strongly-typed +/// value object members including parsing, validation, and serialization support. +/// +[AttributeUsage(AttributeTargets.Class)] +internal sealed class StringValue : Attribute +{ + /// + /// Gets or sets whether to generate a ToString() method. Default is true. + /// + public bool GenerateToString { get; set; } = true; +} + +/// +/// Marks a record as a value object. The source generator will create strongly-typed +/// value object members including parsing, validation, and serialization support. +/// +/// The underlying value type. +[AttributeUsage(AttributeTargets.Class)] +internal sealed class ValueOfAttribute : Attribute + where T : struct +{ + /// + /// Gets or sets whether to generate a ToString() method. Default is true. + /// + public bool GenerateToString { get; set; } = true; +} + +/// +/// Enum for the pre-defined character sets +/// +[Flags] +internal enum CharSet +{ + None = 0, + LowercaseLatin = 1, // a-z + UppercaseLatin = 2, // A-Z + Digits = 4, // 0-9 + Symbols = 8, // e.g., !@#$%^&*() + + // Combinations + LatinLetters = LowercaseLatin | UppercaseLatin, + AlphaNumeric = LatinLetters | Digits +} + diff --git a/storage/storage.slnf b/storage/storage.slnf new file mode 100644 index 000000000..2cc3ea175 --- /dev/null +++ b/storage/storage.slnf @@ -0,0 +1,17 @@ +{ + "solution": { + "path": "..\\products.slnx", + "projects": [ + "storage\\src\\Storage.CliPlugin\\Storage.CliPlugin.csproj", + "storage\\src\\Storage.MsSql\\Storage.MsSql.csproj", + "storage\\src\\Storage.PostgreSql\\Storage.PostgreSql.csproj", + "storage\\src\\Storage.Sqlite\\Storage.Sqlite.csproj", + "storage\\src\\Storage\\Storage.csproj", + "storage\\testing\\TestAppHost\\TestAppHost.csproj", + "storage\\test\\Storage.MsSql.Tests\\Storage.MsSql.Tests.csproj", + "storage\\test\\Storage.PostgreSql.Tests\\Storage.PostgreSql.Tests.csproj", + "storage\\test\\Storage.Sqlite.Tests\\Storage.Sqlite.Tests.csproj", + "storage\\test\\Storage.Tests\\Storage.Tests.csproj" + ] + } +} \ No newline at end of file diff --git a/storage/test/Directory.Build.props b/storage/test/Directory.Build.props new file mode 100644 index 000000000..03bcbe461 --- /dev/null +++ b/storage/test/Directory.Build.props @@ -0,0 +1,13 @@ + + + + + + + $(NoWarn);IDE1006 + + $(NoWarn);CA1859 + + $(NoWarn);IDE0130 + + diff --git a/storage/test/SharedIntegrationTests/FilterTranslatorIntegrationTests.cs b/storage/test/SharedIntegrationTests/FilterTranslatorIntegrationTests.cs new file mode 100644 index 000000000..2453b4d81 --- /dev/null +++ b/storage/test/SharedIntegrationTests/FilterTranslatorIntegrationTests.cs @@ -0,0 +1,393 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Filtering; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Internal.Querying.SearchFields; +using Duende.Storage.Internal.Querying.Sorting; +using Duende.Storage.Pagination; +using Duende.Storage.Querying; + +namespace Duende.Storage.IntegrationTests; + +public partial class FilterTranslatorIntegrationTests +{ + + + private readonly EntityType _testEntityType = new(3, "TestEntity"); + + private readonly Ct _ct = TestContext.Current.CancellationToken; + + private sealed class TestEntityAttributeResolver : IScimAttributeTypeResolver + { + public Field ResolveField(string attributePath) => attributePath.ToLowerInvariant() switch + { + "name" => new StringField("name"), + "score" => new NumberField("score"), + "price" => new NumberField("price"), + "recordedat" => new DateTimeField("recordedAt"), + "lastlogin" => new DateTimeField("lastLogin"), + "isactive" => new BooleanField("isActive"), + "status" => new StringField("status"), + _ => throw new NotSupportedException($"Unknown attribute: {attributePath}") + }; + } + + private async Task CreateProviderAsync() => + await FixtureFactory.CreateAsync(_ct, services => + { + services.AddDsoRegistration(); + services.AddDsoRegistration(); + services.AddDsoRegistration(); + }); + + private static async Task CreateTestEntityAsync( + IStore store, + string name, + int? score = null, + decimal? price = null, + DateTimeOffset? createdAt = null, + DateTimeOffset? lastLogin = null, + bool? isActive = null, + string? status = null, + Ct ct = default) + { + var id = UuidV7.New(); + var dso = new TestEntityDso + { + Name = name, + Score = score, + Price = price, + CreatedAt = createdAt, + LastLogin = lastLogin, + IsActive = isActive, + Status = status + }; + + var searchFieldsBuilder = new SearchFieldsBuilder(); + _ = searchFieldsBuilder.Add("name", name); + if (score.HasValue) + { + _ = searchFieldsBuilder.Add("score", score.Value); + } + + if (price.HasValue) + { + _ = searchFieldsBuilder.Add("price", price.Value); + } + + if (createdAt.HasValue) + { + _ = searchFieldsBuilder.Add("recordedAt", createdAt.Value); + } + + if (lastLogin.HasValue) + { + _ = searchFieldsBuilder.Add("lastLogin", lastLogin.Value); + } + + if (isActive.HasValue) + { + _ = searchFieldsBuilder.Add("isActive", isActive.Value); + } + + if (status != null) + { + _ = searchFieldsBuilder.Add("status", status); + } + + var searchFields = searchFieldsBuilder.Build(); + + var storeInterface = store; + var result = await storeInterface.CreateAsync(id, dso, Array.Empty(), searchFields, Expiration.NoExpiration, [], ct); + result.ShouldBe(CreateResult.Success); + return id; + } + + private static async Task>> TranslateAndQueryAsync( + IStore store, string filterString, EntityType entityType) + { + var resolver = new TestEntityAttributeResolver(); + var translator = new FilterTranslator(resolver); + var filter = translator.Translate(filterString)!; + var page = DataRange.FromPage(1, 10); + return await store.QueryAsync(entityType, filter, SortParameter.Empty, page, Ct.None); + } + + [Fact] + public async Task FilterAndQuerySimpleEqualityAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice"); + _ = await CreateTestEntityAsync(store, "Bob"); + _ = await CreateTestEntityAsync(store, "Charlie"); + + var result = await TranslateAndQueryAsync(store, "name eq \"Alice\"", _testEntityType); + + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Alice"); + } + + [Fact] + public async Task FilterAndQueryNotEqualAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice"); + _ = await CreateTestEntityAsync(store, "Bob"); + _ = await CreateTestEntityAsync(store, "Charlie"); + + var result = await TranslateAndQueryAsync(store, "name ne \"Alice\"", _testEntityType); + + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Bob"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie"); + } + + [Fact] + public async Task FilterAndQueryContainsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice Smith"); + _ = await CreateTestEntityAsync(store, "Bob Jones"); + _ = await CreateTestEntityAsync(store, "Charlie Smith"); + + var result = await TranslateAndQueryAsync(store, "name co \"Smith\"", _testEntityType); + + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice Smith"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie Smith"); + } + + [Fact] + public async Task FilterAndQueryStartsWithAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alpha"); + _ = await CreateTestEntityAsync(store, "Beta"); + _ = await CreateTestEntityAsync(store, "Alpha Centauri"); + + var result = await TranslateAndQueryAsync(store, "name sw \"Alpha\"", _testEntityType); + + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alpha"); + result.Items.ShouldContain(x => x.Value.Name == "Alpha Centauri"); + } + + [Fact] + public async Task FilterAndQueryEndsWithAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice Smith"); + _ = await CreateTestEntityAsync(store, "Bob Jones"); + _ = await CreateTestEntityAsync(store, "John Smith"); + + var result = await TranslateAndQueryAsync(store, "name ew \"Smith\"", _testEntityType); + + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice Smith"); + result.Items.ShouldContain(x => x.Value.Name == "John Smith"); + } + + [Fact] + public async Task FilterAndQueryPresentAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "WithScore1", score: 80); + _ = await CreateTestEntityAsync(store, "NoScore"); + _ = await CreateTestEntityAsync(store, "WithScore2", score: 50); + + var result = await TranslateAndQueryAsync(store, "score pr", _testEntityType); + + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "WithScore1"); + result.Items.ShouldContain(x => x.Value.Name == "WithScore2"); + } + + [Fact] + public async Task FilterAndQueryBooleanEqualityAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Active", isActive: true); + _ = await CreateTestEntityAsync(store, "Inactive", isActive: false); + _ = await CreateTestEntityAsync(store, "Unknown"); + + var result = await TranslateAndQueryAsync(store, "isActive eq true", _testEntityType); + + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Active"); + } + + [Fact] + public async Task FilterAndQueryNumberGreaterThanAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Low", score: 50); + _ = await CreateTestEntityAsync(store, "Boundary", score: 75); + _ = await CreateTestEntityAsync(store, "High", score: 100); + + var result = await TranslateAndQueryAsync(store, "score gt 75", _testEntityType); + + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("High"); + } + + [Fact] + public async Task FilterAndQueryNumberLessOrEqualAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Low", score: 25); + _ = await CreateTestEntityAsync(store, "Boundary", score: 50); + _ = await CreateTestEntityAsync(store, "High", score: 75); + + var result = await TranslateAndQueryAsync(store, "score le 50", _testEntityType); + + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Low"); + result.Items.ShouldContain(x => x.Value.Name == "Boundary"); + } + + [Fact] + public async Task FilterAndQueryNumberEqualDecimalAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Cheap", price: 19.99m); + _ = await CreateTestEntityAsync(store, "Expensive", price: 29.99m); + + var result = await TranslateAndQueryAsync(store, "price eq 19.99", _testEntityType); + + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Cheap"); + } + + [Fact] + public async Task FilterAndQueryDateTimeGreaterThanAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "May", createdAt: new DateTimeOffset(2024, 5, 1, 0, 0, 0, TimeSpan.Zero)); + _ = await CreateTestEntityAsync(store, "July", createdAt: new DateTimeOffset(2024, 7, 1, 0, 0, 0, TimeSpan.Zero)); + _ = await CreateTestEntityAsync(store, "December", createdAt: new DateTimeOffset(2024, 12, 1, 0, 0, 0, TimeSpan.Zero)); + + var result = await TranslateAndQueryAsync(store, "recordedAt gt \"2024-06-01T00:00:00Z\"", _testEntityType); + + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "July"); + result.Items.ShouldContain(x => x.Value.Name == "December"); + } + + [Fact] + public async Task FilterAndQueryAndCombinationAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", score: 30); + _ = await CreateTestEntityAsync(store, "User2", score: 80); + _ = await CreateTestEntityAsync(store, "Admin1", score: 90); + + var result = await TranslateAndQueryAsync(store, "name sw \"User\" and score gt 50", _testEntityType); + + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("User2"); + } + + [Fact] + public async Task FilterAndQueryOrCombinationAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "ActiveEntity", status: "active"); + _ = await CreateTestEntityAsync(store, "PendingEntity", status: "pending"); + _ = await CreateTestEntityAsync(store, "ArchivedEntity", status: "archived"); + + var result = await TranslateAndQueryAsync(store, "status eq \"active\" or status eq \"pending\"", _testEntityType); + + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "ActiveEntity"); + result.Items.ShouldContain(x => x.Value.Name == "PendingEntity"); + } + + [Fact] + public async Task FilterAndQueryNotExpressionAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice"); + _ = await CreateTestEntityAsync(store, "Bob"); + _ = await CreateTestEntityAsync(store, "Charlie"); + + var result = await TranslateAndQueryAsync(store, "not (name eq \"Alice\")", _testEntityType); + + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Bob"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie"); + } + + [Fact] + public async Task FilterAndQueryComplexNestedAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "A", score: 80, isActive: true, status: "standard"); + _ = await CreateTestEntityAsync(store, "B", score: 30, isActive: true, status: "premium"); + _ = await CreateTestEntityAsync(store, "C", score: 80, isActive: false, status: "standard"); + _ = await CreateTestEntityAsync(store, "D", score: 20, isActive: false, status: "basic"); + + var result = await TranslateAndQueryAsync( + store, + "(score gt 50 and isActive eq true) or status eq \"premium\"", + _testEntityType); + + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "A"); + result.Items.ShouldContain(x => x.Value.Name == "B"); + } + + [Fact] + public async Task FilterAndQueryPrecedenceAndBindsTighterThanOrAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "a", score: 10); + _ = await CreateTestEntityAsync(store, "b", score: 80); + _ = await CreateTestEntityAsync(store, "c", score: 90); + + var result = await TranslateAndQueryAsync( + store, + "name eq \"a\" or name eq \"b\" and score gt 50", + _testEntityType); + + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "a"); + result.Items.ShouldContain(x => x.Value.Name == "b"); + } +} diff --git a/storage/test/SharedIntegrationTests/IMigrationFixtureFactory.cs b/storage/test/SharedIntegrationTests/IMigrationFixtureFactory.cs new file mode 100644 index 000000000..fcaa5a53e --- /dev/null +++ b/storage/test/SharedIntegrationTests/IMigrationFixtureFactory.cs @@ -0,0 +1,30 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.IntegrationTests; + +/// +/// Abstraction for creating a migration test fixture in provider-agnostic tests. +/// Each provider implements this to wire up its own DI and database. +/// +public interface IMigrationFixtureFactory +{ + /// + /// Creates a fresh migration fixture backed by a new (empty) database. + /// + Task CreateAsync(CancellationToken ct); +} + +/// +/// A disposable fixture that exposes the and +/// a way to execute raw SQL against the same database. +/// +public interface IMigrationFixture : IAsyncDisposable +{ + IDatabaseSchema Schema { get; } + + /// + /// Executes raw SQL against the database backing this fixture. + /// + Task ExecuteSqlAsync(string sql, CancellationToken ct); +} diff --git a/storage/test/SharedIntegrationTests/IStoreFixtureFactory.cs b/storage/test/SharedIntegrationTests/IStoreFixtureFactory.cs new file mode 100644 index 000000000..5cb0b6440 --- /dev/null +++ b/storage/test/SharedIntegrationTests/IStoreFixtureFactory.cs @@ -0,0 +1,28 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.IntegrationTests; + +/// +/// Abstraction for creating a store fixture in provider-agnostic integration tests. +/// Each provider implements this to wire up its own DI and database. +/// +public interface IStoreFixtureFactory +{ + /// + /// Creates a fresh store fixture backed by a new database. + /// The returned fixture must be disposed to clean up the database. + /// + Task CreateAsync(CancellationToken ct, Action? configure = null); +} + +/// +/// A disposable store fixture that exposes the under test. +/// +public interface IStoreFixture : IAsyncDisposable +{ + IStore Store { get; } +} diff --git a/storage/test/SharedIntegrationTests/MigrationTests.cs b/storage/test/SharedIntegrationTests/MigrationTests.cs new file mode 100644 index 000000000..9536f8daf --- /dev/null +++ b/storage/test/SharedIntegrationTests/MigrationTests.cs @@ -0,0 +1,68 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.IntegrationTests; + +public partial class MigrationTests +{ + private readonly Ct _ct = TestContext.Current.CancellationToken; + + [Fact] + public async Task migrate_creates_schema() + { + await using var fixture = await MigrationFixtureFactory.CreateAsync(_ct); + + await fixture.Schema.MigrateAsync(_ct); + + var result = await fixture.Schema.VerifySchemaAsync(_ct); + result.IsValid.ShouldBeTrue(); + result.Errors.ShouldBeEmpty(); + } + + [Fact] + public async Task migrate_is_idempotent() + { + await using var fixture = await MigrationFixtureFactory.CreateAsync(_ct); + + await fixture.Schema.MigrateAsync(_ct); + await fixture.Schema.MigrateAsync(_ct); + + var result = await fixture.Schema.VerifySchemaAsync(_ct); + result.IsValid.ShouldBeTrue(); + result.Errors.ShouldBeEmpty(); + } + + [Fact] + public async Task build_migration_script_returns_executable_sql() + { + await using var fixture = await MigrationFixtureFactory.CreateAsync(_ct); + + var script = fixture.Schema.BuildMigrationScript(DatabaseSchemaVersion.Zero); + script.ShouldNotBeNullOrWhiteSpace(); + + await fixture.ExecuteSqlAsync(script, _ct); + + var result = await fixture.Schema.VerifySchemaAsync(_ct); + result.IsValid.ShouldBeTrue(); + result.Errors.ShouldBeEmpty(); + } + + [Fact] + public async Task migration_script_is_idempotent() + { + // All providers' scripts are self-contained and safe to execute twice: + // MsSql/PostgreSql use version gates; SQLite uses IF NOT EXISTS. + await using var fixture = await MigrationFixtureFactory.CreateAsync(_ct); + + var script = fixture.Schema.BuildMigrationScript(DatabaseSchemaVersion.Zero); + + await fixture.ExecuteSqlAsync(script, _ct); + await fixture.ExecuteSqlAsync(script, _ct); + + var result = await fixture.Schema.VerifySchemaAsync(_ct); + result.IsValid.ShouldBeTrue(); + result.Errors.ShouldBeEmpty(); + } + + +} diff --git a/storage/test/SharedIntegrationTests/PurgeExpiredTests.cs b/storage/test/SharedIntegrationTests/PurgeExpiredTests.cs new file mode 100644 index 000000000..420be217d --- /dev/null +++ b/storage/test/SharedIntegrationTests/PurgeExpiredTests.cs @@ -0,0 +1,498 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Outbox; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Pagination; +using Microsoft.Extensions.DependencyInjection; +using OutboxEventName = Duende.Storage.Internal.Outbox.OutboxEventName; +using SubscriberName = Duende.Storage.Internal.Outbox.SubscriberName; + +namespace Duende.Storage.IntegrationTests; + +public partial class PurgeExpiredTests +{ + + + private readonly Ct _ct = TestContext.Current.CancellationToken; + + private static readonly SubscriberName WildcardSubscriberName = + SubscriberName.Create("test-subscriber"); + + [Fact] + public async Task PurgeExpired_should_remove_expired_entities() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + + var entityType = TestDso.DsoVersion.EntityType; + var ids = new List(); + + // Create 5 entities that will expire + for (var i = 0; i < 5; i++) + { + var id = UuidV7.New(); + ids.Add(id); + (await store.CreateAsync(id, new TestDso($"purge-{i}-{Guid.NewGuid()}"), [], [], + Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)).ShouldBe(CreateResult.Success); + } + + // Create 2 that won't expire + var persistId1 = UuidV7.New(); + var persistId2 = UuidV7.New(); + (await store.CreateAsync(persistId1, new TestDso($"persist-1-{Guid.NewGuid()}"), [], [], + Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + (await store.CreateAsync(persistId2, new TestDso($"persist-2-{Guid.NewGuid()}"), [], [], + Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + + // Advance time past expiration + tp.Advance(TimeSpan.FromHours(2)); + + var purged = await store.PurgeExpiredAsync(batchSize: 100, _ct); + + purged.ShouldBeGreaterThanOrEqualTo(5); + + // Expired entities should be gone + foreach (var id in ids) + { + (await store.TryReadAsync(entityType, id, _ct)).Found.ShouldBeFalse(); + } + + // Persistent entities should remain + (await store.TryReadAsync(entityType, persistId1, _ct)).Found.ShouldBeTrue(); + (await store.TryReadAsync(entityType, persistId2, _ct)).Found.ShouldBeTrue(); + } + + [Fact] + public async Task PurgeExpired_with_no_expired_records_should_return_zero() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + + // Create an entity that won't expire + (await store.CreateAsync(UuidV7.New(), new TestDso($"alive-{Guid.NewGuid()}"), [], [], + Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + + var purged = await store.PurgeExpiredAsync(batchSize: 100, _ct); + + purged.ShouldBe(0); + } + + [Fact] + public async Task PurgeExpired_single_batch_should_respect_size() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + + var entityType = TestDso.DsoVersion.EntityType; + + // Create 10 entities that will expire + var ids = new List(); + for (var i = 0; i < 10; i++) + { + var id = UuidV7.New(); + ids.Add(id); + (await store.CreateAsync(id, new TestDso($"multi-batch-{i}-{Guid.NewGuid()}"), [], [], + Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)).ShouldBe(CreateResult.Success); + } + + tp.Advance(TimeSpan.FromHours(2)); + + // PurgeExpired only processes a single batch — with batchSize 3, it should purge at most 3 + var purged = await store.PurgeExpiredAsync(batchSize: 3, _ct); + purged.ShouldBeLessThanOrEqualTo(3); + purged.ShouldBeGreaterThan(0); + + // Iterate until all expired entities are purged (simulating what the job does) + var totalPurged = purged; + while (purged > 0) + { + purged = await store.PurgeExpiredAsync(batchSize: 3, _ct); + totalPurged += purged; + } + + totalPurged.ShouldBeGreaterThanOrEqualTo(10); + + // All expired entities should be gone + foreach (var id in ids) + { + (await store.TryReadAsync(entityType, id, _ct)).Found.ShouldBeFalse(); + } + } + + [Fact] + public async Task PurgeExpired_should_also_remove_links() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var queryStore = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + var testLink = new LinkDefinition + { + Left = TestDso.DsoVersion.EntityType, + Right = TestDso2.DsoVersion.EntityType, + Link = LinkTypeRegistry.MembershipRole + }; + + // Create two entities and link them; left expires in 1 hour + (await store.CreateAsync(leftId, new TestDso($"left-{Guid.NewGuid()}"), [], [], + Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)).ShouldBe(CreateResult.Success); + (await store.CreateAsync(rightId, new TestDso2($"right-{Guid.NewGuid()}"), [], [], + Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)).ShouldBe(CreateResult.Success); + (await store.LinkAsync(testLink, leftId, rightId, [], _ct)).ShouldBe(LinkResult.Success); + + // Verify the link exists + var query = LinkQuery.From(TestDso2.DsoVersion.EntityType) + .Join(testLink) + .Where(TestDso.DsoVersion.EntityType, leftId) + .Build(); + var before = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + before.Items.Count.ShouldBe(1); + + // Advance past expiration and purge + tp.Advance(TimeSpan.FromHours(2)); + var purged = await store.PurgeExpiredAsync(batchSize: 100, _ct); + purged.ShouldBeGreaterThanOrEqualTo(1); + + // Left entity gone + (await store.TryReadAsync(TestDso.DsoVersion.EntityType, leftId, _ct)).Found.ShouldBeFalse(); + + // Link should be gone too + var after = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + after.Items.ShouldBeEmpty(); + } + + [Fact] + public async Task PurgeExpired_should_not_delete_entity_whose_ttl_was_extended() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + + var id = UuidV7.New(); + + // Create entity that expires in 1 hour + (await store.CreateAsync(id, new TestDso($"ttl-race-{Guid.NewGuid()}"), [], [], + Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)).ShouldBe(CreateResult.Success); + + // Advance past expiry + tp.Advance(TimeSpan.FromHours(2)); + + // Extend the entity's TTL before purge runs + var entityType = TestDso.DsoVersion.EntityType; + var version = (await store.TryReadAsync(entityType, id, _ct)).Version.ShouldNotBeNull(); + (await store.UpdateAsync(id, new TestDso($"ttl-extended-{Guid.NewGuid()}"), version, [], [], + Expiration.InRelative(TimeSpan.FromHours(5)), [], _ct)).ShouldBe(UpdateResult.Success); + + // PurgeExpired should skip this entity because expires_at is now in the future + var purged = await store.PurgeExpiredAsync(batchSize: 100, _ct); + purged.ShouldBe(0); + + // Entity should still exist + (await store.TryReadAsync(entityType, id, _ct)).Found.ShouldBeTrue(); + } + + [Fact] + public async Task PurgeExpired_should_not_delete_links_for_entity_whose_ttl_was_extended() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var queryStore = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + var testLink = new LinkDefinition + { + Left = TestDso.DsoVersion.EntityType, + Right = TestDso2.DsoVersion.EntityType, + Link = LinkTypeRegistry.MembershipRole + }; + + // Create two entities and link them; left expires in 1 hour + (await store.CreateAsync(leftId, new TestDso($"left-{Guid.NewGuid()}"), [], [], + Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)).ShouldBe(CreateResult.Success); + (await store.CreateAsync(rightId, new TestDso2($"right-{Guid.NewGuid()}"), [], [], + Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + (await store.LinkAsync(testLink, leftId, rightId, [], _ct)).ShouldBe(LinkResult.Success); + + // Advance past expiry + tp.Advance(TimeSpan.FromHours(2)); + + // Extend the left entity's TTL before purge runs + var version = (await store.TryReadAsync(TestDso.DsoVersion.EntityType, leftId, _ct)).Version.ShouldNotBeNull(); + (await store.UpdateAsync(leftId, new TestDso($"extended-{Guid.NewGuid()}"), version, [], [], + Expiration.InRelative(TimeSpan.FromHours(5)), [], _ct)).ShouldBe(UpdateResult.Success); + + // PurgeExpired should skip this entity — TTL was extended + var purged = await store.PurgeExpiredAsync(batchSize: 100, _ct); + purged.ShouldBe(0); + + // Left entity should still exist + (await store.TryReadAsync(TestDso.DsoVersion.EntityType, leftId, _ct)).Found.ShouldBeTrue(); + + // Link should still exist + var query = LinkQuery.From(TestDso2.DsoVersion.EntityType) + .Join(testLink) + .Where(TestDso.DsoVersion.EntityType, leftId) + .Build(); + var after = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + after.Items.Count.ShouldBe(1); + } + + [Fact] + public async Task PurgeExpired_should_write_EntityExpired_outbox_events_when_subscriber_matches() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp, [new WildcardTestSubscriber()]); + var store = fixture.Store; + + var entityType = TestDso.DsoVersion.EntityType; + var ids = new List(); + + for (var i = 0; i < 3; i++) + { + var id = UuidV7.New(); + ids.Add(id); + (await store.CreateAsync(id, new TestDso($"outbox-{i}-{Guid.NewGuid()}"), [], [], + Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)).ShouldBe(CreateResult.Success); + } + + tp.Advance(TimeSpan.FromHours(2)); + var purged = await store.PurgeExpiredAsync(batchSize: 100, _ct); + purged.ShouldBeGreaterThanOrEqualTo(3); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 100, _ct); + page.Events.Count.ShouldBe(3); + + foreach (var id in ids) + { + var matching = page.Events.Where(e => e.SubjectId == UuidV7.From(id.Value)).ToList(); + matching.Count.ShouldBe(1); + var evt = matching[0]; + evt.EventName.ShouldBe(OutboxEventName.EntityExpired); + evt.EntityTypeId.ShouldBe((int)entityType.Id); + evt.Payload.ShouldNotBeNullOrEmpty(); + + // Entity should be deleted + (await store.TryReadAsync(entityType, id, _ct)).Found.ShouldBeFalse(); + } + } + + [Fact] + public async Task PurgeExpired_should_not_write_outbox_events_when_outbox_disabled() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + // Use default CreateServiceProvider (no outbox) — outbox is disabled by default + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + + for (var i = 0; i < 3; i++) + { + (await store.CreateAsync(UuidV7.New(), new TestDso($"no-outbox-{i}-{Guid.NewGuid()}"), [], [], + Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)).ShouldBe(CreateResult.Success); + } + + tp.Advance(TimeSpan.FromHours(2)); + var purged = await store.PurgeExpiredAsync(batchSize: 100, _ct); + purged.ShouldBeGreaterThanOrEqualTo(3); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 100, _ct); + page.Events.ShouldBeEmpty(); + } + + [Fact] + public async Task PurgeExpired_should_only_write_outbox_events_for_matching_entity_types() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + var testDsoTypeId = (int)TestDso.DsoVersion.EntityType.Id; + await using var fixture = await CreateProviderAsync(tp, + [new TypeFilteredTestSubscriber("typed-sub", [testDsoTypeId])]); + var store = fixture.Store; + + // Create entities of both types with expiration + var testDsoId = UuidV7.New(); + var testDso2Id = UuidV7.New(); + (await store.CreateAsync(testDsoId, new TestDso($"typed-{Guid.NewGuid()}"), [], [], + Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)).ShouldBe(CreateResult.Success); + (await store.CreateAsync(testDso2Id, new TestDso2($"typed2-{Guid.NewGuid()}"), [], [], + Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)).ShouldBe(CreateResult.Success); + + tp.Advance(TimeSpan.FromHours(2)); + var purged = await store.PurgeExpiredAsync(batchSize: 100, _ct); + purged.ShouldBeGreaterThanOrEqualTo(2); + + // Both entities should be deleted + (await store.TryReadAsync(TestDso.DsoVersion.EntityType, testDsoId, _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(TestDso2.DsoVersion.EntityType, testDso2Id, _ct)).Found.ShouldBeFalse(); + + // Outbox should only have events for TestDso, not TestDso2 + var page = await store.GetOutboxEventsForSubscriberAsync(SubscriberName.Create("typed-sub"), 100, _ct); + page.Events.Count.ShouldBe(1); + page.Events[0].EntityTypeId.ShouldBe(testDsoTypeId); + page.Events[0].SubjectId.ShouldBe(UuidV7.From(testDsoId.Value)); + } + + [Fact] + public async Task PurgeExpired_should_not_write_outbox_events_when_no_subscribers() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + // Outbox enabled but NO subscribers + await using var fixture = await CreateProviderAsync(tp, []); + var store = fixture.Store; + + for (var i = 0; i < 2; i++) + { + (await store.CreateAsync(UuidV7.New(), new TestDso($"no-sub-{i}-{Guid.NewGuid()}"), [], [], + Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)).ShouldBe(CreateResult.Success); + } + + tp.Advance(TimeSpan.FromHours(2)); + var purged = await store.PurgeExpiredAsync(batchSize: 100, _ct); + purged.ShouldBeGreaterThanOrEqualTo(2); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 100, _ct); + page.Events.ShouldBeEmpty(); + } + + [Fact] + public async Task PurgeExpired_should_fanout_to_multiple_subscribers() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp, + [new NamedTestSubscriber("sub-a"), new NamedTestSubscriber("sub-b")]); + var store = fixture.Store; + + for (var i = 0; i < 2; i++) + { + var id = UuidV7.New(); + (await store.CreateAsync(id, new TestDso($"fanout-{i}-{Guid.NewGuid()}"), [], [], + Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)).ShouldBe(CreateResult.Success); + } + + tp.Advance(TimeSpan.FromHours(2)); + var purged = await store.PurgeExpiredAsync(batchSize: 100, _ct); + purged.ShouldBeGreaterThanOrEqualTo(2); + + var pageA = await store.GetOutboxEventsForSubscriberAsync(SubscriberName.Create("sub-a"), 100, _ct); + var pageB = await store.GetOutboxEventsForSubscriberAsync(SubscriberName.Create("sub-b"), 100, _ct); + var allEvents = pageA.Events.Concat(pageB.Events).ToList(); + allEvents.Count.ShouldBe(4); // 2 entities × 2 subscribers + + // Each subscriber should appear exactly twice + var subACounts = allEvents.Count(e => e.SubscriberName == SubscriberName.Create("sub-a")); + var subBCounts = allEvents.Count(e => e.SubscriberName == SubscriberName.Create("sub-b")); + subACounts.ShouldBe(2); + subBCounts.ShouldBe(2); + + // Each message should have a unique MessageId + allEvents.Select(e => e.MessageId).Distinct().Count().ShouldBe(4); + } + + [Fact] + public async Task PurgeExpired_should_share_EventId_across_subscribers_for_same_entity() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp, + [new NamedTestSubscriber("sub-x"), new NamedTestSubscriber("sub-y")]); + var store = fixture.Store; + + var id1 = UuidV7.New(); + var id2 = UuidV7.New(); + (await store.CreateAsync(id1, new TestDso($"eid-{Guid.NewGuid()}"), [], [], + Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)).ShouldBe(CreateResult.Success); + (await store.CreateAsync(id2, new TestDso($"eid-{Guid.NewGuid()}"), [], [], + Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)).ShouldBe(CreateResult.Success); + + tp.Advance(TimeSpan.FromHours(2)); + var purged = await store.PurgeExpiredAsync(batchSize: 100, _ct); + purged.ShouldBeGreaterThanOrEqualTo(2); + + var pageX = await store.GetOutboxEventsForSubscriberAsync(SubscriberName.Create("sub-x"), 100, _ct); + var pageY = await store.GetOutboxEventsForSubscriberAsync(SubscriberName.Create("sub-y"), 100, _ct); + var allEvents = pageX.Events.Concat(pageY.Events).ToList(); + allEvents.Count.ShouldBe(4); // 2 entities × 2 subscribers + + // Group events by subject (entity) — each entity's events should share the same EventId + var bySubject = allEvents.GroupBy(e => e.SubjectId).ToList(); + bySubject.Count.ShouldBe(2); + foreach (var group in bySubject) + { + var eventIds = group.Select(e => e.EventId).Distinct().ToList(); + eventIds.Count.ShouldBe(1, $"All subscriber rows for subject {group.Key} should share the same EventId"); + } + + // The two entities should have different EventIds + var allEventIds = bySubject.Select(g => g.First().EventId).Distinct().ToList(); + allEventIds.Count.ShouldBe(2, "Different entities should have different EventIds"); + + // MessageIds must all be unique + allEvents.Select(e => e.MessageId).Distinct().Count().ShouldBe(4); + } + + private async Task CreateProviderAsync(FakeTimeProvider tp) => + await FixtureFactory.CreateAsync(_ct, services => + { + _ = services.AddSingleton(tp); + _ = services.AddSingleton(tp); + services.AddDsoRegistration(); + services.AddDsoRegistration(); + }); + + private async Task CreateProviderAsync(FakeTimeProvider tp, IOutboxSubscriber[] subscribers) => + await FixtureFactory.CreateAsync(_ct, services => + { + _ = services.AddSingleton(tp); + _ = services.AddSingleton(tp); + foreach (var subscriber in subscribers) + { + _ = services.AddSingleton(subscriber); + } + services.AddDsoRegistration(); + services.AddDsoRegistration(); + }); + + /// + /// Wildcard subscriber that matches all entity types and event names. + /// + private sealed class WildcardTestSubscriber : IOutboxSubscriber + { + public SubscriberName SubscriberName => SubscriberName.Create("test-subscriber"); + public bool IsEnabled => true; + public IReadOnlySet EventNames => new HashSet(); + public IReadOnlySet EntityTypeIds => new HashSet(); + } + + /// + /// Named wildcard subscriber for multi-subscriber fanout tests. + /// + private sealed class NamedTestSubscriber(string name) : IOutboxSubscriber + { + public SubscriberName SubscriberName => SubscriberName.Create(name); + public bool IsEnabled => true; + public IReadOnlySet EventNames => new HashSet(); + public IReadOnlySet EntityTypeIds => new HashSet(); + } + + /// + /// Subscriber filtered to specific entity type IDs. + /// + private sealed class TypeFilteredTestSubscriber(string name, int[] entityTypeIds) : IOutboxSubscriber + { + public SubscriberName SubscriberName => SubscriberName.Create(name); + public bool IsEnabled => true; + public IReadOnlySet EventNames => new HashSet(); + public IReadOnlySet EntityTypeIds => new HashSet(entityTypeIds); + } +} diff --git a/storage/test/SharedIntegrationTests/QueryStoreArrayFilterTests.cs b/storage/test/SharedIntegrationTests/QueryStoreArrayFilterTests.cs new file mode 100644 index 000000000..65dabc114 --- /dev/null +++ b/storage/test/SharedIntegrationTests/QueryStoreArrayFilterTests.cs @@ -0,0 +1,1123 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Internal.Querying.SearchFields; +using Duende.Storage.Pagination; +using SortParameter = Duende.Storage.Internal.Querying.Sorting.SortParameter; + +namespace Duende.Storage.IntegrationTests; + +/// +/// Tests for array filter expressions across all store implementations. +/// Tests SCIM2-compatible array filtering where all conditions must match within the same array item. +/// +/// Test Coverage for OR Expressions in Array Filters: +/// - EqualExpression: Tested with multiple OR branches (strings, numbers, datetimes) +/// - ContainsExpression: Tested in OR conditions +/// - StartsWithExpression: Tested in OR conditions +/// - InExpression: Tested in OR conditions and combined with other expressions +/// - Mixed expression types: Tested with combinations of different expression types in OR +/// - Complex multi-branch OR: Tested with 3+ OR branches +/// +/// Note: Numeric comparison expressions (GreaterThan, LessThan, GreaterOrEqual, LessOrEqual, BetweenExpression) +/// follow the same code path as EqualExpression in OR conditions (see SqlWhereClauseBuilder.cs:587-596, IsLeafExpression). +/// The BuildOrFilter and CollectConditions methods handle all expression types uniformly. +/// +public partial class QueryStoreArrayFilterTests +{ + private readonly EntityType _testEntityType = new(2, "UserEntity"); + + private readonly Ct _ct = TestContext.Current.CancellationToken; + + private static readonly string[] WorkBusinessTypes = ["work", "business"]; + + private async Task CreateProviderAsync() => + await FixtureFactory.CreateAsync(_ct, services => + { + services.AddDsoRegistration(); + services.AddDsoRegistration(); + services.AddDsoRegistration(); + }); + + private static async Task CreateUserWithEmailsAsync( + IStore store, + string name, + EmailAddress[] emails, + Ct ct) + { + var id = UuidV7.New(); + var dso = new TestUserDso + { + Name = name, + Emails = emails + }; + + var searchFieldsBuilder = new SearchFieldsBuilder(); + _ = searchFieldsBuilder.Add("name", name); + + for (var i = 0; i < emails.Length; i++) + { + _ = searchFieldsBuilder.Add("emails.type", i, emails[i].Type); + _ = searchFieldsBuilder.Add("emails.value", i, emails[i].Value); + if (emails[i].CreatedAt is { } createdAt) + { + _ = searchFieldsBuilder.Add("emails.recordedAt", i, createdAt); + } + if (emails[i].Priority is { } priority) + { + _ = searchFieldsBuilder.Add("emails.priority", i, priority); + } + } + + var searchFields = searchFieldsBuilder.Build(); + // IStore extends IStore, so we can cast + var storeInterface = store; + var result = await storeInterface.CreateAsync(id, dso, Array.Empty(), searchFields, Expiration.NoExpiration, [], ct); + result.ShouldBe(CreateResult.Success); + return id; + } + + [Fact] + public async Task QueryArrayFilterSingleConditionShouldReturnMatchingEntitiesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@work.com" }, + new EmailAddress { Type = "personal", Value = "alice@home.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "personal", Value = "bob@home.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Charlie", [ + new EmailAddress { Type = "work", Value = "charlie@work.com" } + ], Ct.None); + + var filter = Query.ArrayFilter("emails", new StringField("type").Equals("work")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie"); + } + + [Fact] + public async Task QueryArrayFilterMultipleConditionsShouldMatchSameArrayItemAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@work.com" }, + new EmailAddress { Type = "personal", Value = "alice@example.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "work", Value = "bob@home.com" }, + new EmailAddress { Type = "personal", Value = "bob@example.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Charlie", [ + new EmailAddress { Type = "work", Value = "charlie@example.com" } + ], Ct.None); + + // Filter: emails where type="work" AND value contains "example" + // Should match: Charlie (has work email containing "example") + // Should NOT match: Alice (has work email but doesn't contain "example", has example email but not work type) + var filter = Query.ArrayFilter("emails", + new StringField("type").Equals("work") + .And(new StringField("value").Contains("example"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Charlie"); + } + + [Fact] + public async Task QueryArrayFilterWithOrConditionShouldMatchSameArrayItemAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@work.com" }, + new EmailAddress { Type = "personal", Value = "alice@home.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "business", Value = "bob@company.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Charlie", [ + new EmailAddress { Type = "other", Value = "charlie@test.com" } + ], Ct.None); + + // Filter: emails where type="work" OR type="business" + var filter = Query.ArrayFilter("emails", + new StringField("type").Equals("work") + .Or(new StringField("type").Equals("business"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); + result.Items.ShouldContain(x => x.Value.Name == "Bob"); + } + + [Fact] + public async Task QueryArrayFilterCombinedWithOtherFiltersShouldWorkAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@work.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "work", Value = "bob@work.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Charlie", [ + new EmailAddress { Type = "personal", Value = "charlie@home.com" } + ], Ct.None); + + // Filter: name starts with "A" AND has a work email + var filter = new StringField("name").StartsWith("A") + .And(Query.ArrayFilter("emails", new StringField("type").Equals("work"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Alice"); + } + + [Fact] + public async Task QueryArrayFilterNoMatchingArrayItemShouldReturnNoResultsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@work.com" }, + new EmailAddress { Type = "personal", Value = "alice@home.com" } + ], Ct.None); + + // Filter: emails where type="work" AND value contains "home" + // No single array item has both conditions true + var filter = Query.ArrayFilter("emails", + new StringField("type").Equals("work") + .And(new StringField("value").Contains("home"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(0); + } + + [Fact] + public async Task QueryArrayFilterEmptyArrayShouldReturnNoMatchAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [], Ct.None); + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "work", Value = "bob@work.com" } + ], Ct.None); + + var filter = Query.ArrayFilter("emails", new StringField("type").Equals("work")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Bob"); + } + + [Fact] + public async Task QueryArrayFilterComplexScimLikeScenarioShouldWorkAsync() + { + // Arrange - Simulating SCIM2 user schema + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "User1", [ + new EmailAddress { Type = "work", Value = "user1@example.com" }, + new EmailAddress { Type = "home", Value = "user1@personal.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "User2", [ + new EmailAddress { Type = "work", Value = "user2@test.com" }, + new EmailAddress { Type = "other", Value = "user2@example.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "User3", [ + new EmailAddress { Type = "home", Value = "user3@example.com" } + ], Ct.None); + + // SCIM filter: emails[type eq "work" and value co "@example.com"] + var filter = Query.ArrayFilter("emails", + new StringField("type").Equals("work") + .And(new StringField("value").Contains("@example.com"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("User1"); + } + + [Fact] + public async Task QueryArrayFilterOrWithContainsExpressionShouldMatchAnyArrayItemAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@company.com" }, + new EmailAddress { Type = "personal", Value = "alice@home.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "work", Value = "bob@example.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Charlie", [ + new EmailAddress { Type = "personal", Value = "charlie@other.com" } + ], Ct.None); + + // Filter: emails where value contains "company" OR value contains "example" + var filter = Query.ArrayFilter("emails", + new StringField("value").Contains("company") + .Or(new StringField("value").Contains("example"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); + result.Items.ShouldContain(x => x.Value.Name == "Bob"); + } + + [Fact] + public async Task QueryArrayFilterOrWithStartsWithExpressionShouldMatchAnyArrayItemAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@work.com" }, + new EmailAddress { Type = "personal", Value = "alice@home.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "work", Value = "bob@company.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Charlie", [ + new EmailAddress { Type = "personal", Value = "charlie@personal.com" } + ], Ct.None); + + // Filter: emails where value starts with "alice@" OR value starts with "charlie@" + var filter = Query.ArrayFilter("emails", + new StringField("value").StartsWith("alice@") + .Or(new StringField("value").StartsWith("charlie@"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie"); + } + + [Fact] + public async Task QueryArrayFilterOrWithInExpressionShouldMatchAnyArrayItemAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@work.com" }, + new EmailAddress { Type = "personal", Value = "alice@home.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "business", Value = "bob@company.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Charlie", [ + new EmailAddress { Type = "other", Value = "charlie@test.com" } + ], Ct.None); + + // Filter: emails where type IN ("work", "business") OR type = "other" + var filter = Query.ArrayFilter("emails", + new StringField("type").In(WorkBusinessTypes) + .Or(new StringField("type").Equals("other"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(3); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); + result.Items.ShouldContain(x => x.Value.Name == "Bob"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie"); + } + + [Fact] + public async Task QueryArrayFilterOrStringDatetimeoffsetShouldMatchAnyArrayItemAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var date1 = new DateTimeOffset(2024, 1, 15, 10, 30, 0, TimeSpan.Zero); + var date2 = new DateTimeOffset(2024, 6, 15, 14, 45, 0, TimeSpan.Zero); + var date3 = new DateTimeOffset(2024, 12, 20, 16, 0, 0, TimeSpan.Zero); + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@work.com", CreatedAt = date1 }, + new EmailAddress { Type = "personal", Value = "alice@home.com", CreatedAt = date2 } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "business", Value = "bob@company.com", CreatedAt = date2 } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Charlie", [ + new EmailAddress { Type = "other", Value = "charlie@test.com", CreatedAt = date3 } + ], Ct.None); + + // Filter: emails where type == work OR recordedAt = date3 + var filter = Query.ArrayFilter("emails", + new StringField("type").Equals("work") + .Or(new DateTimeField("recordedAt").Equals(date3))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie"); + } + + [Fact] + public async Task QueryArrayFilterOrStringNumberShouldMatchAnyArrayItemAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@work.com", Priority = 1 }, + new EmailAddress { Type = "personal", Value = "alice@home.com", Priority = 2 } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "business", Value = "bob@company.com", Priority = 2 } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Charlie", [ + new EmailAddress { Type = "other", Value = "charlie@test.com", Priority = 5 } + ], Ct.None); + + // Filter: emails where type == "work" OR priority = 5 + var filter = Query.ArrayFilter("emails", + new StringField("type").Equals("work") + .Or(new NumberField("priority").Equals(5))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie"); + } + + [Fact] + public async Task QueryArrayFilterOrWithMixedExpressionTypesShouldWorkAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@work.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "personal", Value = "bob@example.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Charlie", [ + new EmailAddress { Type = "business", Value = "charlie@test.com" } + ], Ct.None); + + // Filter: emails where (type = "work") OR (value contains "@example.com") + // Should match: Alice (has work), Bob (has @example.com) + var filter = Query.ArrayFilter("emails", + new StringField("type").Equals("work") + .Or(new StringField("value").Contains("@example.com"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); + result.Items.ShouldContain(x => x.Value.Name == "Bob"); + } + + [Fact] + public async Task QueryArrayFilterOrWithMultipleContainsExpressionsShouldWorkAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "User1", [ + new EmailAddress { Type = "work", Value = "user1@company.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "User2", [ + new EmailAddress { Type = "personal", Value = "user2@example.org" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "User3", [ + new EmailAddress { Type = "other", Value = "user3@test.net" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "User4", [ + new EmailAddress { Type = "business", Value = "user4@other.com" } + ], Ct.None); + + // Filter: emails where value contains ".com" OR value contains ".org" + var filter = Query.ArrayFilter("emails", + new StringField("value").Contains(".com") + .Or(new StringField("value").Contains(".org"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(3); + result.Items.ShouldContain(x => x.Value.Name == "User1"); + result.Items.ShouldContain(x => x.Value.Name == "User2"); + result.Items.ShouldContain(x => x.Value.Name == "User4"); + } + + [Fact] + public async Task QueryArrayFilterOrWithMultipleStartsWithExpressionsShouldWorkAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "AdminUser", [ + new EmailAddress { Type = "admin", Value = "admin@system.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "SupportUser", [ + new EmailAddress { Type = "support", Value = "support@system.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "RegularUser", [ + new EmailAddress { Type = "user", Value = "user@system.com" } + ], Ct.None); + + // Filter: emails where value starts with "admin@" OR value starts with "support@" + var filter = Query.ArrayFilter("emails", + new StringField("value").StartsWith("admin@") + .Or(new StringField("value").StartsWith("support@"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "AdminUser"); + result.Items.ShouldContain(x => x.Value.Name == "SupportUser"); + } + + [Fact] + public async Task QueryArrayFilterComplexOrWithThreeBranchesShouldWorkAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "User1", [ + new EmailAddress { Type = "work", Value = "user1@company.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "User2", [ + new EmailAddress { Type = "personal", Value = "user2@home.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "User3", [ + new EmailAddress { Type = "business", Value = "user3@biz.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "User4", [ + new EmailAddress { Type = "other", Value = "user4@test.com" } + ], Ct.None); + + // Filter: emails where type="work" OR type="personal" OR type="business" + var filter = Query.ArrayFilter("emails", + new StringField("type").Equals("work") + .Or(new StringField("type").Equals("personal")) + .Or(new StringField("type").Equals("business"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(3); + result.Items.ShouldContain(x => x.Value.Name == "User1"); + result.Items.ShouldContain(x => x.Value.Name == "User2"); + result.Items.ShouldContain(x => x.Value.Name == "User3"); + } + + [Fact] + public async Task QueryArrayFilterOrCombinedWithInExpressionShouldWorkAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "TeamA", [ + new EmailAddress { Type = "work", Value = "teama@company.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "TeamB", [ + new EmailAddress { Type = "business", Value = "teamb@company.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "TeamC", [ + new EmailAddress { Type = "contractor", Value = "teamc@contractor.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "External", [ + new EmailAddress { Type = "external", Value = "ext@external.com" } + ], Ct.None); + + // Filter: emails where type IN ("work", "business") OR value contains "contractor" + var filter = Query.ArrayFilter("emails", + new StringField("type").In(WorkBusinessTypes) + .Or(new StringField("value").Contains("contractor"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(3); + result.Items.ShouldContain(x => x.Value.Name == "TeamA"); + result.Items.ShouldContain(x => x.Value.Name == "TeamB"); + result.Items.ShouldContain(x => x.Value.Name == "TeamC"); + } + + [Fact] + public async Task QueryArrayFilterWithPagingShouldHandlePageBreaksAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 15 users with work emails + for (var i = 1; i <= 15; i++) + { + _ = await CreateUserWithEmailsAsync(store, $"User{i:D2}", [ + new EmailAddress { Type = "work", Value = $"user{i}@work.com" }, + new EmailAddress { Type = "personal", Value = $"user{i}@home.com" } + ], Ct.None); + } + + // Add 5 users without work emails + for (var i = 16; i <= 20; i++) + { + _ = await CreateUserWithEmailsAsync(store, $"User{i:D2}", [ + new EmailAddress { Type = "personal", Value = $"user{i}@home.com" } + ], Ct.None); + } + + var filter = Query.ArrayFilter("emails", new StringField("type").Equals("work")); + var sort = new SortParameter(new StringField("name")); + + // Act - Get pages with size 5 (should be 3 pages: 5+5+5) + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 5), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 5), Ct.None); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 5), Ct.None); + var page4 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(4, 5), Ct.None); + + // Assert + page1.Items.Count.ShouldBe(5); + page1.TotalCount.ShouldBe(15); + page1.HasMoreData.ShouldBeTrue(); + page1.Items[0].Value.Name.ShouldBe("User01"); + page1.Items[4].Value.Name.ShouldBe("User05"); + + page2.Items.Count.ShouldBe(5); + page2.HasMoreData.ShouldBeTrue(); + page2.Items[0].Value.Name.ShouldBe("User06"); + page2.Items[4].Value.Name.ShouldBe("User10"); + + page3.Items.Count.ShouldBe(5); + page3.HasMoreData.ShouldBeFalse(); + page3.Items[0].Value.Name.ShouldBe("User11"); + page3.Items[4].Value.Name.ShouldBe("User15"); + + page4.Items.Count.ShouldBe(0); + } + + [Fact] + public async Task QueryArrayFilterComplexConditionPartialPageAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 14 users with various email patterns + for (var i = 1; i <= 14; i++) + { + var emails = new List + { + new EmailAddress { Type = "work", Value = $"user{i}@company.com" } + }; + + // Every 3rd user gets an @example.com work email + if (i % 3 == 0) + { + emails.Add(new EmailAddress { Type = "work", Value = $"user{i}@example.com" }); + } + + _ = await CreateUserWithEmailsAsync(store, $"User{i:D2}", emails.ToArray(), Ct.None); + } + + // Filter: work emails containing "@example.com" (should match users 3, 6, 9, 12) + var filter = Query.ArrayFilter("emails", + new StringField("type").Equals("work") + .And(new StringField("value").Contains("@example.com"))); + var sort = new SortParameter(new StringField("name")); + + // Act - Page size 3 creates 2 pages (3+1) + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 3), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 3), Ct.None); + + // Assert + page1.Items.Count.ShouldBe(3); + page1.TotalCount.ShouldBe(4); + page1.Items[0].Value.Name.ShouldBe("User03"); + page1.Items[1].Value.Name.ShouldBe("User06"); + page1.Items[2].Value.Name.ShouldBe("User09"); + + page2.Items.Count.ShouldBe(1); // Partial last page + page2.Items[0].Value.Name.ShouldBe("User12"); + page2.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QueryArrayFilterWithOtherFilterAndPagingShouldWorkAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create users A1-A10 and B1-B10, all with work emails + for (var i = 1; i <= 10; i++) + { + _ = await CreateUserWithEmailsAsync(store, $"A{i:D2}", [ + new EmailAddress { Type = "work", Value = $"a{i}@work.com" } + ], Ct.None); + _ = await CreateUserWithEmailsAsync(store, $"B{i:D2}", [ + new EmailAddress { Type = "work", Value = $"b{i}@work.com" } + ], Ct.None); + } + + // Filter: name starts with "A" AND has work email (10 results) + var filter = new StringField("name").StartsWith("A") + .And(Query.ArrayFilter("emails", new StringField("type").Equals("work"))); + var sort = new SortParameter(new StringField("name")); + + // Act - Page size 4 creates 3 pages (4+4+2) + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 4), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 4), Ct.None); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 4), Ct.None); + + // Assert + page1.TotalCount.ShouldBe(10); + page1.Items.Count.ShouldBe(4); + page1.Items.ShouldAllBe(u => u.Value.Name.StartsWith('A')); + + page2.Items.Count.ShouldBe(4); + page2.Items.ShouldAllBe(u => u.Value.Name.StartsWith('A')); + + page3.Items.Count.ShouldBe(2); + page3.Items.ShouldAllBe(u => u.Value.Name.StartsWith('A')); + } + + [Fact] + public async Task QueryArrayFilterOrConditionAcrossPagesShouldMaintainConsistencyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 8 users with work emails, 8 with business emails + for (var i = 1; i <= 8; i++) + { + _ = await CreateUserWithEmailsAsync(store, $"WorkUser{i:D2}", [ + new EmailAddress { Type = "work", Value = $"work{i}@company.com" } + ], Ct.None); + _ = await CreateUserWithEmailsAsync(store, $"BizUser{i:D2}", [ + new EmailAddress { Type = "business", Value = $"biz{i}@company.com" } + ], Ct.None); + } + + // Filter: type = "work" OR type = "business" (16 results) + var filter = Query.ArrayFilter("emails", + new StringField("type").Equals("work") + .Or(new StringField("type").Equals("business"))); + var sort = new SortParameter(new StringField("name")); + + // Act - Page size 6 creates 3 pages (6+6+4) + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 6), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 6), Ct.None); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 6), Ct.None); + + // Assert + page1.TotalCount.ShouldBe(16); + page1.Items.Count.ShouldBe(6); + + page2.Items.Count.ShouldBe(6); + + page3.Items.Count.ShouldBe(4); + page3.HasMoreData.ShouldBeFalse(); + + // Verify all results have either work or business email + var allItems = page1.Items.Concat(page2.Items).Concat(page3.Items).ToList(); + allItems.Count.ShouldBe(16); + } + + [Fact] + public async Task QueryArrayFilterNoMatchesShouldReturnEmptyPagesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "User1", [ + new EmailAddress { Type = "personal", Value = "user1@home.com" } + ], Ct.None); + _ = await CreateUserWithEmailsAsync(store, "User2", [ + new EmailAddress { Type = "personal", Value = "user2@home.com" } + ], Ct.None); + + // Filter for work emails (no matches) + var filter = Query.ArrayFilter("emails", new StringField("type").Equals("work")); + + // Act + var page1 = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, DataRange.FromPage(1, 10), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, DataRange.FromPage(2, 10), Ct.None); + + // Assert + page1.Items.Count.ShouldBe(0); + page1.TotalCount.ShouldBe(0); + page1.HasMoreData.ShouldBeFalse(); + + page2.Items.Count.ShouldBe(0); + } + + [Fact] + public async Task QueryArrayFilterExactPageBoundaryShouldWorkAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create exactly 12 users with work emails (exactly 3 pages of 4) + for (var i = 1; i <= 12; i++) + { + _ = await CreateUserWithEmailsAsync(store, $"User{i:D2}", [ + new EmailAddress { Type = "work", Value = $"user{i}@work.com" } + ], Ct.None); + } + + var filter = Query.ArrayFilter("emails", new StringField("type").Equals("work")); + var sort = new SortParameter(new StringField("name")); + + // Act + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 4), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 4), Ct.None); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 4), Ct.None); + var page4 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(4, 4), Ct.None); + + // Assert + page1.Items.Count.ShouldBe(4); + page1.Items[0].Value.Name.ShouldBe("User01"); + + page2.Items.Count.ShouldBe(4); + page2.Items[0].Value.Name.ShouldBe("User05"); + + page3.Items.Count.ShouldBe(4); // Full last page (not partial) + page3.Items[0].Value.Name.ShouldBe("User09"); + page3.Items[3].Value.Name.ShouldBe("User12"); + page3.HasMoreData.ShouldBeFalse(); + + // Page 4 should be empty + page4.Items.Count.ShouldBe(0); + } + + [Fact] + public async Task QueryArrayFilterSmallPagesShouldIterateCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 5; i++) + { + _ = await CreateUserWithEmailsAsync(store, $"User{i}", [ + new EmailAddress { Type = "work", Value = $"user{i}@work.com" } + ], Ct.None); + } + + var filter = Query.ArrayFilter("emails", new StringField("type").Equals("work")); + var sort = new SortParameter(new StringField("name")); + + // Act - Page size of 2 creates 3 pages (2+2+1) + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 2), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 2), Ct.None); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 2), Ct.None); + + // Assert + page1.Items.Count.ShouldBe(2); + page1.Items[0].Value.Name.ShouldBe("User1"); + page1.Items[1].Value.Name.ShouldBe("User2"); + + page2.Items.Count.ShouldBe(2); + page2.Items[0].Value.Name.ShouldBe("User3"); + page2.Items[1].Value.Name.ShouldBe("User4"); + + page3.Items.Count.ShouldBe(1); + page3.Items[0].Value.Name.ShouldBe("User5"); + page3.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QueryArrayFilterWithEndsWithShouldMatchArrayItemAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@work.com" }, + new EmailAddress { Type = "personal", Value = "alice@home.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "work", Value = "bob@company.org" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Charlie", [ + new EmailAddress { Type = "personal", Value = "charlie@work.com" } + ], Ct.None); + + // Filter: emails where value ends with "@work.com" + var filter = Query.ArrayFilter("emails", new StringField("value").EndsWith("@work.com")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Alice (alice@work.com) and Charlie (charlie@work.com) match + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie"); + } + + [Fact] + public async Task QueryArrayFilterEndsWithWithAndConditionShouldMatchSameArrayItemAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@company.com" }, + new EmailAddress { Type = "personal", Value = "alice@home.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "personal", Value = "bob@company.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Charlie", [ + new EmailAddress { Type = "work", Value = "charlie@other.org" } + ], Ct.None); + + // Filter: emails where type="work" AND value ends with "@company.com" + // Only Alice has a work email ending with @company.com + var filter = Query.ArrayFilter("emails", + new StringField("type").Equals("work") + .And(new StringField("value").EndsWith("@company.com"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Alice"); + } + + [Fact] + public async Task QueryArrayFilterWithNotEqualShouldExcludeMatchingItemsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@work.com" }, + new EmailAddress { Type = "personal", Value = "alice@home.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "personal", Value = "bob@home.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Charlie", [ + new EmailAddress { Type = "personal", Value = "charlie@home.com" } + ], Ct.None); + + // Filter: emails where NOT(type = "personal") + // Alice has a non-personal email (work), Bob and Charlie only have personal emails + var filter = Query.ArrayFilter("emails", Query.Not(new StringField("type").Equals("personal"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Only Alice has an email where type != "personal" + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Alice"); + } + + [Fact] + public async Task QueryArrayFilterNotCombinedWithTopLevelFilterShouldWorkAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@work.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "work", Value = "bob@work.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Charlie", [ + new EmailAddress { Type = "personal", Value = "charlie@home.com" } + ], Ct.None); + + // NOT(emails where type = "work") - excludes users who have a work email + var arrayFilter = Query.ArrayFilter("emails", new StringField("type").Equals("work")); + var filter = Query.Not(arrayFilter); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Only Charlie has no work email + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Charlie"); + } + + [Fact] + public async Task QueryArrayFilterPresentShouldMatchEntitiesWithArrayItemsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateUserWithEmailsAsync(store, "Alice", [ + new EmailAddress { Type = "work", Value = "alice@work.com" } + ], Ct.None); + + _ = await CreateUserWithEmailsAsync(store, "Bob", [ + new EmailAddress { Type = "personal", Value = "bob@home.com" } + ], Ct.None); + + // Charlie has no emails + _ = await CreateUserWithEmailsAsync(store, "Charlie", [], Ct.None); + + // Filter: emails where value is present (any email value exists) + var filter = Query.ArrayFilter("emails", new StringField("value").Present()); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Alice and Bob have email entries, Charlie has none + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); + result.Items.ShouldContain(x => x.Value.Name == "Bob"); + } + +} diff --git a/storage/test/SharedIntegrationTests/QueryStoreBasicExpressionTests.cs b/storage/test/SharedIntegrationTests/QueryStoreBasicExpressionTests.cs new file mode 100644 index 000000000..037b109c1 --- /dev/null +++ b/storage/test/SharedIntegrationTests/QueryStoreBasicExpressionTests.cs @@ -0,0 +1,2291 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Internal.Querying.SearchFields; +using Duende.Storage.Pagination; +using Duende.Storage.Querying; +using SortParameter = Duende.Storage.Internal.Querying.Sorting.SortParameter; + +namespace Duende.Storage.IntegrationTests; + +/// +/// Tests for basic query expressions across all store implementations. +/// Focuses on testing various field types and operations, with special emphasis on range queries +/// for datetime and number fields which are critical for filtering data. +/// +public partial class QueryStoreBasicExpressionTests +{ + + + private readonly EntityType _testEntityType = new(3, "TestEntity"); + + private readonly Ct _ct = TestContext.Current.CancellationToken; + + private async Task CreateProviderAsync() => + await FixtureFactory.CreateAsync(_ct, services => + { + services.AddDsoRegistration(); + services.AddDsoRegistration(); + services.AddDsoRegistration(); + }); + + private static async Task CreateEntityAsync( + IStore store, + string name, + int? score = null, + decimal? price = null, + DateTimeOffset? createdAt = null, + DateTimeOffset? lastLogin = null, + bool? isActive = null, + string? status = null, + Ct ct = default) => await CreateTestEntityAsync(store, name, score, price, createdAt, lastLogin, isActive, status, ct); + + private static async Task CreateTestEntityAsync( + IStore store, + string name, + int? score = null, + decimal? price = null, + DateTimeOffset? createdAt = null, + DateTimeOffset? lastLogin = null, + bool? isActive = null, + string? status = null, + Ct ct = default) + { + var id = UuidV7.New(); + var dso = new TestEntityDso + { + Name = name, + Score = score, + Price = price, + CreatedAt = createdAt, + LastLogin = lastLogin, + IsActive = isActive, + Status = status + }; + + var searchFieldsBuilder = new SearchFieldsBuilder(); + _ = searchFieldsBuilder.Add("name", name); + if (score.HasValue) + { + _ = searchFieldsBuilder.Add("score", score.Value); + } + + if (price.HasValue) + { + _ = searchFieldsBuilder.Add("price", price.Value); + } + + if (createdAt.HasValue) + { + _ = searchFieldsBuilder.Add("recordedAt", createdAt.Value); + } + + if (lastLogin.HasValue) + { + _ = searchFieldsBuilder.Add("lastLogin", lastLogin.Value); + } + + if (isActive.HasValue) + { + _ = searchFieldsBuilder.Add("isActive", isActive.Value); + } + + if (status != null) + { + _ = searchFieldsBuilder.Add("status", status); + } + + var searchFields = searchFieldsBuilder.Build(); + + var storeInterface = store; + var result = await storeInterface.CreateAsync(id, dso, Array.Empty(), searchFields, Expiration.NoExpiration, [], ct); + result.ShouldBe(CreateResult.Success); + return id; + } + + [Fact] + public async Task QueryStringFieldEqualsShouldReturnExactMatchesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice"); + _ = await CreateTestEntityAsync(store, "alice"); // Different case - stored as same uppercase value + _ = await CreateTestEntityAsync(store, "Bob"); + + var filter = new StringField("name").Equals("Alice"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Both "Alice" and "alice" are stored as "ALICE" in search fields, + // so querying for "Alice" (uppercased to "ALICE") matches both entries. + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); + result.Items.ShouldContain(x => x.Value.Name == "alice"); + } + + [Fact] + public async Task QueryStringFieldContainsShouldReturnPartialMatchesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice Smith"); + _ = await CreateTestEntityAsync(store, "Bob Jones"); + _ = await CreateTestEntityAsync(store, "Charlie Smith"); + + var filter = new StringField("name").Contains("Smith"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice Smith"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie Smith"); + } + + [Fact] + public async Task QueryStringFieldStartsWithShouldReturnPrefixMatchesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alpha"); + _ = await CreateTestEntityAsync(store, "Beta"); + _ = await CreateTestEntityAsync(store, "Alpha Centauri"); + + var filter = new StringField("name").StartsWith("Alpha"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alpha"); + result.Items.ShouldContain(x => x.Value.Name == "Alpha Centauri"); + } + + [Fact] + public async Task QueryStringFieldInShouldReturnMatchingValuesAsync() + { + // Arrange + + string[] activePendingStatuses = ["active", "pending"]; + + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice", status: "active"); + _ = await CreateTestEntityAsync(store, "Bob", status: "pending"); + _ = await CreateTestEntityAsync(store, "Charlie", status: "inactive"); + _ = await CreateTestEntityAsync(store, "David", status: "active"); + + var filter = new StringField("status").In(activePendingStatuses); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(3); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); + result.Items.ShouldContain(x => x.Value.Name == "Bob"); + result.Items.ShouldContain(x => x.Value.Name == "David"); + } + + [Fact] + public async Task QueryNumberFieldGreaterThanShouldReturnLargerValuesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", score: 50); + _ = await CreateTestEntityAsync(store, "User2", score: 75); + _ = await CreateTestEntityAsync(store, "User3", score: 100); + _ = await CreateTestEntityAsync(store, "User4", score: 125); + + var filter = new NumberField("score").GreaterThan(75); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "User3"); + result.Items.ShouldContain(x => x.Value.Name == "User4"); + } + + [Fact] + public async Task NumberField_GreaterThan_should_exclude_the_boundary_value_async() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Age25", score: 25); + _ = await CreateTestEntityAsync(store, "Age30", score: 30); + _ = await CreateTestEntityAsync(store, "Age35", score: 35); + + var filter = new NumberField("score").GreaterThan(25); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert — GreaterThan(25) must use > not >=, so Age25 must NOT appear + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Age30"); + result.Items.ShouldContain(x => x.Value.Name == "Age35"); + result.Items.ShouldNotContain(x => x.Value.Name == "Age25"); + } + + [Fact] + public async Task NumberField_Equal_should_return_only_the_exact_boundary_value_async() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Age25", score: 25); + _ = await CreateTestEntityAsync(store, "Age30", score: 30); + _ = await CreateTestEntityAsync(store, "Age35", score: 35); + + var filter = new NumberField("score").Equals(25); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert — Equal(25) must use = so only Age25 matches + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Age25"); + } + + [Fact] + public async Task NumberField_GreaterThan_and_Equal_should_return_different_result_sets_async() + { + // Arrange — verifies GreaterThan and Equal produce distinct SQL operators + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Age25", score: 25); + _ = await CreateTestEntityAsync(store, "Age30", score: 30); + _ = await CreateTestEntityAsync(store, "Age35", score: 35); + + var greaterThanFilter = new NumberField("score").GreaterThan(25); + var equalFilter = new NumberField("score").Equals(25); + var page = DataRange.FromPage(1, 10); + + // Act + var greaterThanResult = await store.QueryAsync(_testEntityType, greaterThanFilter, SortParameter.Empty, page, _ct); + var equalResult = await store.QueryAsync(_testEntityType, equalFilter, SortParameter.Empty, page, _ct); + + // Assert — the two filters must produce non-overlapping results + greaterThanResult.Items.Count.ShouldBe(2); + equalResult.Items.Count.ShouldBe(1); + + // The entity returned by Equal must NOT appear in GreaterThan results + var equalName = equalResult.Items[0].Value.Name; + greaterThanResult.Items.ShouldNotContain(x => x.Value.Name == equalName); + } + + [Fact] + public async Task DateTimeField_GreaterThan_should_exclude_the_boundary_date_async() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var boundary = new DateTimeOffset(2024, 6, 1, 0, 0, 0, TimeSpan.Zero); + var after1 = new DateTimeOffset(2024, 9, 1, 0, 0, 0, TimeSpan.Zero); + var after2 = new DateTimeOffset(2024, 12, 1, 0, 0, 0, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Boundary", createdAt: boundary); + _ = await CreateTestEntityAsync(store, "After1", createdAt: after1); + _ = await CreateTestEntityAsync(store, "After2", createdAt: after2); + + var cutoff = new DateTime(2024, 6, 1, 0, 0, 0, DateTimeKind.Utc); + var filter = new DateTimeField("recordedAt").GreaterThan(cutoff); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert — GreaterThan must use > so the boundary date is excluded + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "After1"); + result.Items.ShouldContain(x => x.Value.Name == "After2"); + result.Items.ShouldNotContain(x => x.Value.Name == "Boundary"); + } + + [Fact] + public async Task DateTimeField_Equal_should_return_only_the_exact_boundary_date_async() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var boundary = new DateTimeOffset(2024, 6, 1, 0, 0, 0, TimeSpan.Zero); + var other = new DateTimeOffset(2024, 9, 1, 0, 0, 0, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Boundary", createdAt: boundary); + _ = await CreateTestEntityAsync(store, "Other", createdAt: other); + + var cutoff = new DateTime(2024, 6, 1, 0, 0, 0, DateTimeKind.Utc); + var filter = new DateTimeField("recordedAt").Equals(cutoff); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert — Equal must use = so only the exact date matches + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Boundary"); + } + + [Fact] + public async Task QueryNumberFieldGreaterOrEqualShouldReturnEqualAndLargerValuesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", score: 50); + _ = await CreateTestEntityAsync(store, "User2", score: 75); + _ = await CreateTestEntityAsync(store, "User3", score: 100); + + var filter = new NumberField("score").GreaterOrEqual(75); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "User2"); + result.Items.ShouldContain(x => x.Value.Name == "User3"); + } + + [Fact] + public async Task QueryNumberFieldLessThanShouldReturnSmallerValuesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", score: 25); + _ = await CreateTestEntityAsync(store, "User2", score: 50); + _ = await CreateTestEntityAsync(store, "User3", score: 75); + _ = await CreateTestEntityAsync(store, "User4", score: 100); + + var filter = new NumberField("score").LessThan(60); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "User1"); + result.Items.ShouldContain(x => x.Value.Name == "User2"); + } + + [Fact] + public async Task QueryNumberFieldLessOrEqualShouldReturnEqualAndSmallerValuesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", score: 25); + _ = await CreateTestEntityAsync(store, "User2", score: 50); + _ = await CreateTestEntityAsync(store, "User3", score: 75); + + var filter = new NumberField("score").LessOrEqual(50); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "User1"); + result.Items.ShouldContain(x => x.Value.Name == "User2"); + } + + [Fact] + public async Task QueryNumberFieldBetweenShouldReturnValuesInRangeAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", score: 10); + _ = await CreateTestEntityAsync(store, "User2", score: 25); + _ = await CreateTestEntityAsync(store, "User3", score: 50); + _ = await CreateTestEntityAsync(store, "User4", score: 75); + _ = await CreateTestEntityAsync(store, "User5", score: 100); + + var filter = new NumberField("score").Between(25, 75); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Should include boundaries (25 and 75) + result.Items.Count.ShouldBe(3); + result.Items.ShouldContain(x => x.Value.Name == "User2"); + result.Items.ShouldContain(x => x.Value.Name == "User3"); + result.Items.ShouldContain(x => x.Value.Name == "User4"); + } + + [Fact] + public async Task QueryNumberFieldBetweenWithDecimalValuesShouldReturnValuesInRangeAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Product1", price: 9.99m); + _ = await CreateTestEntityAsync(store, "Product2", price: 19.99m); + _ = await CreateTestEntityAsync(store, "Product3", price: 29.99m); + _ = await CreateTestEntityAsync(store, "Product4", price: 39.99m); + _ = await CreateTestEntityAsync(store, "Product5", price: 49.99m); + + var filter = new NumberField("price").Between(15.0m, 35.0m); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Product2"); + result.Items.ShouldContain(x => x.Value.Name == "Product3"); + } + + [Fact] + public async Task QueryNumberFieldInShouldReturnMatchingValuesAsync() + { + // Arrange + decimal[] testNumberValues = [10.0m, 30.0m, 50.0m]; + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", score: 10); + _ = await CreateTestEntityAsync(store, "User2", score: 20); + _ = await CreateTestEntityAsync(store, "User3", score: 30); + _ = await CreateTestEntityAsync(store, "User4", score: 40); + _ = await CreateTestEntityAsync(store, "User5", score: 50); + + var filter = new NumberField("score").In(testNumberValues); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(3); + result.Items.ShouldContain(x => x.Value.Name == "User1"); + result.Items.ShouldContain(x => x.Value.Name == "User3"); + result.Items.ShouldContain(x => x.Value.Name == "User5"); + } + + [Fact] + public async Task QueryDateTimeFieldGreaterThanShouldReturnLaterDatesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var date1 = new DateTimeOffset(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + var date2 = new DateTimeOffset(2024, 6, 1, 0, 0, 0, TimeSpan.Zero); + var date3 = new DateTimeOffset(2024, 9, 1, 0, 0, 0, TimeSpan.Zero); + var date4 = new DateTimeOffset(2024, 12, 1, 0, 0, 0, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Event1", createdAt: date1); + _ = await CreateTestEntityAsync(store, "Event2", createdAt: date2); + _ = await CreateTestEntityAsync(store, "Event3", createdAt: date3); + _ = await CreateTestEntityAsync(store, "Event4", createdAt: date4); + + var cutoffDate = new DateTime(2024, 6, 1, 0, 0, 0, DateTimeKind.Utc); + var filter = new DateTimeField("recordedAt").GreaterThan(cutoffDate); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Event3"); + result.Items.ShouldContain(x => x.Value.Name == "Event4"); + } + + [Fact] + public async Task QueryDateTimeFieldGreaterOrEqualShouldReturnEqualAndLaterDatesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var date1 = new DateTimeOffset(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + var date2 = new DateTimeOffset(2024, 6, 1, 0, 0, 0, TimeSpan.Zero); + var date3 = new DateTimeOffset(2024, 12, 1, 0, 0, 0, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Event1", createdAt: date1); + _ = await CreateTestEntityAsync(store, "Event2", createdAt: date2); + _ = await CreateTestEntityAsync(store, "Event3", createdAt: date3); + + var cutoffDate = new DateTime(2024, 6, 1, 0, 0, 0, DateTimeKind.Utc); + var filter = new DateTimeField("recordedAt").GreaterOrEqual(cutoffDate); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Event2"); + result.Items.ShouldContain(x => x.Value.Name == "Event3"); + } + + [Fact] + public async Task QueryDateTimeFieldLessThanShouldReturnEarlierDatesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var date1 = new DateTimeOffset(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + var date2 = new DateTimeOffset(2024, 3, 1, 0, 0, 0, TimeSpan.Zero); + var date3 = new DateTimeOffset(2024, 6, 1, 0, 0, 0, TimeSpan.Zero); + var date4 = new DateTimeOffset(2024, 9, 1, 0, 0, 0, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Event1", createdAt: date1); + _ = await CreateTestEntityAsync(store, "Event2", createdAt: date2); + _ = await CreateTestEntityAsync(store, "Event3", createdAt: date3); + _ = await CreateTestEntityAsync(store, "Event4", createdAt: date4); + + var cutoffDate = new DateTime(2024, 6, 1, 0, 0, 0, DateTimeKind.Utc); + var filter = new DateTimeField("recordedAt").LessThan(cutoffDate); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Event1"); + result.Items.ShouldContain(x => x.Value.Name == "Event2"); + } + + [Fact] + public async Task QueryDateTimeFieldLessOrEqualShouldReturnEqualAndEarlierDatesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var date1 = new DateTimeOffset(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + var date2 = new DateTimeOffset(2024, 6, 1, 0, 0, 0, TimeSpan.Zero); + var date3 = new DateTimeOffset(2024, 12, 1, 0, 0, 0, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Event1", createdAt: date1); + _ = await CreateTestEntityAsync(store, "Event2", createdAt: date2); + _ = await CreateTestEntityAsync(store, "Event3", createdAt: date3); + + var cutoffDate = new DateTime(2024, 6, 1, 0, 0, 0, DateTimeKind.Utc); + var filter = new DateTimeField("recordedAt").LessOrEqual(cutoffDate); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Event1"); + result.Items.ShouldContain(x => x.Value.Name == "Event2"); + } + + [Fact] + public async Task QueryDateTimeFieldBetweenShouldReturnDatesInRangeAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var date1 = new DateTimeOffset(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + var date2 = new DateTimeOffset(2024, 3, 1, 0, 0, 0, TimeSpan.Zero); + var date3 = new DateTimeOffset(2024, 6, 1, 0, 0, 0, TimeSpan.Zero); + var date4 = new DateTimeOffset(2024, 9, 1, 0, 0, 0, TimeSpan.Zero); + var date5 = new DateTimeOffset(2024, 12, 1, 0, 0, 0, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Event1", createdAt: date1); + _ = await CreateTestEntityAsync(store, "Event2", createdAt: date2); + _ = await CreateTestEntityAsync(store, "Event3", createdAt: date3); + _ = await CreateTestEntityAsync(store, "Event4", createdAt: date4); + _ = await CreateTestEntityAsync(store, "Event5", createdAt: date5); + + var startDate = new DateTime(2024, 3, 1, 0, 0, 0, DateTimeKind.Utc); + var endDate = new DateTime(2024, 9, 1, 0, 0, 0, DateTimeKind.Utc); + var filter = new DateTimeField("recordedAt").Between(startDate, endDate); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Should include boundaries (March 1 and September 1) + result.Items.Count.ShouldBe(3); + result.Items.ShouldContain(x => x.Value.Name == "Event2"); + result.Items.ShouldContain(x => x.Value.Name == "Event3"); + result.Items.ShouldContain(x => x.Value.Name == "Event4"); + } + + [Fact] + public async Task QueryDateTimeFieldBetweenWithTimeShouldRespectTimeComponentAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var date1 = new DateTimeOffset(2024, 6, 1, 8, 0, 0, TimeSpan.Zero); + var date2 = new DateTimeOffset(2024, 6, 1, 12, 0, 0, TimeSpan.Zero); + var date3 = new DateTimeOffset(2024, 6, 1, 16, 0, 0, TimeSpan.Zero); + var date4 = new DateTimeOffset(2024, 6, 1, 20, 0, 0, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Event1", lastLogin: date1); + _ = await CreateTestEntityAsync(store, "Event2", lastLogin: date2); + _ = await CreateTestEntityAsync(store, "Event3", lastLogin: date3); + _ = await CreateTestEntityAsync(store, "Event4", lastLogin: date4); + + var startTime = new DateTime(2024, 6, 1, 10, 0, 0, DateTimeKind.Utc); + var endTime = new DateTime(2024, 6, 1, 18, 0, 0, DateTimeKind.Utc); + var filter = new DateTimeField("lastLogin").Between(startTime, endTime); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Event2"); + result.Items.ShouldContain(x => x.Value.Name == "Event3"); + } + + [Fact] + public async Task QueryDateTimeFieldEqualsShouldReturnExactMatchAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var date1 = new DateTimeOffset(2024, 6, 1, 12, 0, 0, TimeSpan.Zero); + var date2 = new DateTimeOffset(2024, 6, 1, 13, 0, 0, TimeSpan.Zero); + var date3 = new DateTimeOffset(2024, 6, 2, 12, 0, 0, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Event1", createdAt: date1); + _ = await CreateTestEntityAsync(store, "Event2", createdAt: date2); + _ = await CreateTestEntityAsync(store, "Event3", createdAt: date3); + + var targetDate = new DateTime(2024, 6, 1, 12, 0, 0, DateTimeKind.Utc); + var filter = new DateTimeField("recordedAt").Equals(targetDate); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Event1"); + } + + [Fact] + public async Task QueryBooleanFieldIsTrueShouldReturnTrueValuesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", isActive: true); + _ = await CreateTestEntityAsync(store, "User2", isActive: false); + _ = await CreateTestEntityAsync(store, "User3", isActive: true); + + var filter = new BooleanField("isActive").IsTrue(); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "User1"); + result.Items.ShouldContain(x => x.Value.Name == "User3"); + } + + [Fact] + public async Task QueryBooleanFieldIsFalseShouldReturnFalseValuesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", isActive: true); + _ = await CreateTestEntityAsync(store, "User2", isActive: false); + _ = await CreateTestEntityAsync(store, "User3", isActive: true); + + var filter = new BooleanField("isActive").IsFalse(); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("User2"); + } + + [Fact] + public async Task QueryBooleanFieldEqualsShouldReturnMatchingValuesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", isActive: true); + _ = await CreateTestEntityAsync(store, "User2", isActive: false); + _ = await CreateTestEntityAsync(store, "User3", isActive: true); + + var filterTrue = new BooleanField("isActive").Equals(true); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filterTrue, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "User1"); + result.Items.ShouldContain(x => x.Value.Name == "User3"); + } + + [Fact] + public async Task QueryNumberFieldEqualsWithIntegerShouldReturnExactMatchesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", score: 50); + _ = await CreateTestEntityAsync(store, "User2", score: 75); + _ = await CreateTestEntityAsync(store, "User3", score: 50); + + var filter = new NumberField("score").Equals(50); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "User1"); + result.Items.ShouldContain(x => x.Value.Name == "User3"); + } + + [Fact] + public async Task QueryNumberFieldEqualsWithDecimalShouldReturnExactMatchesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Product1", price: 19.99m); + _ = await CreateTestEntityAsync(store, "Product2", price: 29.99m); + _ = await CreateTestEntityAsync(store, "Product3", price: 19.99m); + + var filter = new NumberField("price").Equals(19.99m); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Product1"); + result.Items.ShouldContain(x => x.Value.Name == "Product3"); + } + + [Fact] + public async Task QueryNumberFieldWithNegativeNumbersShouldWorkCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Account1", score: -100); + _ = await CreateTestEntityAsync(store, "Account2", score: -50); + _ = await CreateTestEntityAsync(store, "Account3", score: 0); + _ = await CreateTestEntityAsync(store, "Account4", score: 50); + + var filter = new NumberField("score").LessThan(0); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Account1"); + result.Items.ShouldContain(x => x.Value.Name == "Account2"); + } + + [Fact] + public async Task QueryNumberFieldBetweenWithNegativeRangeShouldReturnCorrectResultsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Item1", score: -100); + _ = await CreateTestEntityAsync(store, "Item2", score: -75); + _ = await CreateTestEntityAsync(store, "Item3", score: -50); + _ = await CreateTestEntityAsync(store, "Item4", score: -25); + _ = await CreateTestEntityAsync(store, "Item5", score: 0); + + var filter = new NumberField("score").Between(-80, -40); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Item2"); + result.Items.ShouldContain(x => x.Value.Name == "Item3"); + } + + [Fact] + public async Task QueryNumberFieldWithZeroShouldWorkCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Zero1", score: 0); + _ = await CreateTestEntityAsync(store, "Positive", score: 10); + _ = await CreateTestEntityAsync(store, "Zero2", score: 0); + _ = await CreateTestEntityAsync(store, "Negative", score: -10); + + var filter = new NumberField("score").Equals(0); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Zero1"); + result.Items.ShouldContain(x => x.Value.Name == "Zero2"); + } + + [Fact] + public async Task QueryDateTimeFieldMultipleRangesUsingOrShouldReturnUnionAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var jan = new DateTimeOffset(2024, 1, 15, 0, 0, 0, TimeSpan.Zero); + var mar = new DateTimeOffset(2024, 3, 15, 0, 0, 0, TimeSpan.Zero); + var jun = new DateTimeOffset(2024, 6, 15, 0, 0, 0, TimeSpan.Zero); + var sep = new DateTimeOffset(2024, 9, 15, 0, 0, 0, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Event1", createdAt: jan); + _ = await CreateTestEntityAsync(store, "Event2", createdAt: mar); + _ = await CreateTestEntityAsync(store, "Event3", createdAt: jun); + _ = await CreateTestEntityAsync(store, "Event4", createdAt: sep); + + // Match dates in January OR June (using Equals with OR) + var jan1 = new DateTime(2024, 1, 15, 0, 0, 0, DateTimeKind.Utc); + var jun1 = new DateTime(2024, 6, 15, 0, 0, 0, DateTimeKind.Utc); + + var filter = new DateTimeField("recordedAt").Equals(jan1) + .Or(new DateTimeField("recordedAt").Equals(jun1)); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Event1"); + result.Items.ShouldContain(x => x.Value.Name == "Event3"); + } + + [Fact] + public async Task QueryDateTimeFieldWithMillisecondPrecisionShouldMatchExactlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var date1 = new DateTimeOffset(2024, 6, 1, 12, 30, 45, 123, TimeSpan.Zero); + var date2 = new DateTimeOffset(2024, 6, 1, 12, 30, 45, 456, TimeSpan.Zero); + var date3 = new DateTimeOffset(2024, 6, 1, 12, 30, 45, 789, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Event1", lastLogin: date1); + _ = await CreateTestEntityAsync(store, "Event2", lastLogin: date2); + _ = await CreateTestEntityAsync(store, "Event3", lastLogin: date3); + + var targetDate = new DateTime(2024, 6, 1, 12, 30, 45, 456, DateTimeKind.Utc); + var filter = new DateTimeField("lastLogin").Equals(targetDate); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Event2"); + } + + [Fact] + public async Task QueryStringFieldEqualsWithSpecialCharactersShouldMatchAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "user@example.com"); + _ = await CreateTestEntityAsync(store, "user_123"); + _ = await CreateTestEntityAsync(store, "user-test"); + _ = await CreateTestEntityAsync(store, "user.name"); + + var filter = new StringField("name").Equals("user@example.com"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("user@example.com"); + } + + [Fact] + public async Task QueryStringFieldInWithEmptyCollectionShouldReturnNoResultsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", status: "active"); + _ = await CreateTestEntityAsync(store, "User2", status: "pending"); + + string[] emptyStatuses = []; + var filter = new StringField("status").In(emptyStatuses); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(0); + } + + [Fact] + public async Task QueryStringFieldContainsWithPercentWildcardShouldTreatAsLiteralAsync() + { + // Arrange - Test that % is treated as a literal character, not a wildcard + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "100% Complete"); + _ = await CreateTestEntityAsync(store, "50% Done"); + _ = await CreateTestEntityAsync(store, "100 Complete"); + _ = await CreateTestEntityAsync(store, "100X Complete"); + + var filter = new StringField("name").Contains("100%"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Should only match "100% Complete", not "100 Complete" or "100X Complete" + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("100% Complete"); + } + + [Fact] + public async Task QueryStringFieldContainsWithUnderscoreWildcardShouldTreatAsLiteralAsync() + { + // Arrange - Test that _ is treated as a literal character, not a single-char wildcard + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "user_name"); + _ = await CreateTestEntityAsync(store, "user-name"); + _ = await CreateTestEntityAsync(store, "username"); + _ = await CreateTestEntityAsync(store, "userXname"); + + var filter = new StringField("name").Contains("user_"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Should only match "user_name", not "user-name", "username", or "userXname" + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("user_name"); + } + + [Fact] + public async Task QueryStringFieldStartsWithWithPercentWildcardShouldTreatAsLiteralAsync() + { + // Arrange - Test that % at the start is treated as a literal character + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "%discount_code"); + _ = await CreateTestEntityAsync(store, "Xdiscount_code"); + _ = await CreateTestEntityAsync(store, "discount_code"); + + var filter = new StringField("name").StartsWith("%discount"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Should only match "%discount_code" + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("%discount_code"); + } + + [Fact] + public async Task QueryStringFieldStartsWithWithUnderscoreWildcardShouldTreatAsLiteralAsync() + { + // Arrange - Test that _ at the start is treated as a literal character + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "_private_method"); + _ = await CreateTestEntityAsync(store, "Xprivate_method"); + _ = await CreateTestEntityAsync(store, "private_method"); + + var filter = new StringField("name").StartsWith("_private"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Should only match "_private_method" + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("_private_method"); + } + + [Fact] + public async Task QueryStringFieldContainsWithBackslashCharacterShouldTreatAsLiteralAsync() + { + // Arrange - Test that backslash is treated as a literal character + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, @"C:\Windows\System32"); + _ = await CreateTestEntityAsync(store, @"C:/Windows/System32"); + _ = await CreateTestEntityAsync(store, @"Windows\System32"); + + var filter = new StringField("name").Contains(@"Windows\System"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Should match paths with backslashes, not forward slashes + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == @"C:\Windows\System32"); + result.Items.ShouldContain(x => x.Value.Name == @"Windows\System32"); + } + + [Fact] + public async Task QueryStringFieldContainsWithMultipleWildcardsShouldEscapeAllAsync() + { + // Arrange - Test that multiple wildcard characters are all escaped + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "discount_50%_off"); + _ = await CreateTestEntityAsync(store, "discountX50XX off"); + _ = await CreateTestEntityAsync(store, "discount-50%-off"); + + var filter = new StringField("name").Contains("discount_50%"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Should only match "discount_50%_off" + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("discount_50%_off"); + } + + [Fact] + public async Task QueryStringFieldStartsWithWithCombinedWildcardsShouldEscapeAllAsync() + { + // Arrange - Test StartsWith with multiple wildcard characters + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "_%special_prefix%_test"); + _ = await CreateTestEntityAsync(store, "XXspecial_prefixXX test"); + _ = await CreateTestEntityAsync(store, "special_prefix_test"); + + var filter = new StringField("name").StartsWith("_%special"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Should only match "_%special_prefix%_test" + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("_%special_prefix%_test"); + } + + [Fact] + public async Task QueryNumberFieldInWithSingleValueShouldMatchAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", score: 10); + _ = await CreateTestEntityAsync(store, "User2", score: 20); + _ = await CreateTestEntityAsync(store, "User3", score: 30); + + decimal[] singleValue = [20.0m]; + var filter = new NumberField("score").In(singleValue); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("User2"); + } + + [Fact] + public async Task QueryNumberFieldInWithEmptyCollectionShouldReturnNoResultsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", score: 10); + _ = await CreateTestEntityAsync(store, "User2", score: 20); + + decimal[] emptyValues = []; + var filter = new NumberField("score").In(emptyValues); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(0); + } + + [Fact] + public async Task QueryDateTimeFieldEdgeOfDayShouldDistinguishAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var startOfDay = new DateTimeOffset(2024, 6, 1, 0, 0, 0, TimeSpan.Zero); + var endOfDay = new DateTimeOffset(2024, 6, 1, 23, 59, 59, 999, TimeSpan.Zero); + var nextDay = new DateTimeOffset(2024, 6, 2, 0, 0, 0, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "StartOfDay", createdAt: startOfDay); + _ = await CreateTestEntityAsync(store, "EndOfDay", createdAt: endOfDay); + _ = await CreateTestEntityAsync(store, "NextDay", createdAt: nextDay); + + // Filter for everything before end of June 1st + var cutoff = new DateTime(2024, 6, 2, 0, 0, 0, DateTimeKind.Utc); + var filter = new DateTimeField("recordedAt").LessThan(cutoff); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "StartOfDay"); + result.Items.ShouldContain(x => x.Value.Name == "EndOfDay"); + } + + [Fact] + public async Task QueryNumberFieldWithVeryLargeNumbersShouldWorkCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Item1", price: 999999999.99m); + _ = await CreateTestEntityAsync(store, "Item2", price: 1000000000.00m); + _ = await CreateTestEntityAsync(store, "Item3", price: 1000000000.01m); + + var filter = new NumberField("price").GreaterThan(1000000000.00m); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Item3"); + } + + [Fact] + public async Task QueryNumberFieldWithVerySmallDecimalsShouldWorkCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Item1", price: 0.001m); + _ = await CreateTestEntityAsync(store, "Item2", price: 0.002m); + _ = await CreateTestEntityAsync(store, "Item3", price: 0.003m); + + var filter = new NumberField("price").Between(0.0015m, 0.0025m); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Item2"); + } + + [Fact] + public async Task QueryStringFieldEndsWithUsingContainsShouldMatchAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "john.smith@example.com"); + _ = await CreateTestEntityAsync(store, "jane.doe@example.com"); + _ = await CreateTestEntityAsync(store, "bob@test.com"); + + var filter = new StringField("name").Contains("@example.com"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "john.smith@example.com"); + result.Items.ShouldContain(x => x.Value.Name == "jane.doe@example.com"); + } + + [Fact] + public async Task QueryDateTimeFieldAtMidnightShouldMatchExactlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var midnight = new DateTimeOffset(2024, 6, 1, 0, 0, 0, TimeSpan.Zero); + var oneSecondAfter = new DateTimeOffset(2024, 6, 1, 0, 0, 1, TimeSpan.Zero); + var oneSecondBefore = new DateTimeOffset(2024, 5, 31, 23, 59, 59, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Before", createdAt: oneSecondBefore); + _ = await CreateTestEntityAsync(store, "Midnight", createdAt: midnight); + _ = await CreateTestEntityAsync(store, "After", createdAt: oneSecondAfter); + + var filter = new DateTimeField("recordedAt").Equals(new DateTime(2024, 6, 1, 0, 0, 0, DateTimeKind.Utc)); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Midnight"); + } + + [Fact] + public async Task QueryCombinedFiltersNotOperatorSimulationUsingOrAsync() + { + // Arrange - Simulate NOT by testing values outside a range + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Low", score: 10); + _ = await CreateTestEntityAsync(store, "MidLow", score: 40); + _ = await CreateTestEntityAsync(store, "MidHigh", score: 60); + _ = await CreateTestEntityAsync(store, "High", score: 90); + + // Get all values NOT between 30 and 70 (i.e., less than 30 OR greater than 70) + var filter = new NumberField("score").LessThan(30) + .Or(new NumberField("score").GreaterThan(70)); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Low"); + result.Items.ShouldContain(x => x.Value.Name == "High"); + } + + [Fact] + public async Task QueryMixedTypesAllFieldTypesTogetherAsync() + { + // Arrange - Test all four field types in one complex query + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var date1 = new DateTimeOffset(2024, 1, 15, 10, 30, 0, TimeSpan.Zero); + var date2 = new DateTimeOffset(2024, 6, 15, 14, 45, 0, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Match1", score: 75, price: 25.50m, createdAt: date1, isActive: true, status: "premium"); + _ = await CreateTestEntityAsync(store, "NoMatch1", score: 45, price: 25.50m, createdAt: date1, isActive: true, status: "basic"); + _ = await CreateTestEntityAsync(store, "NoMatch2", score: 75, price: 15.00m, createdAt: date1, isActive: false, status: "premium"); + _ = await CreateTestEntityAsync(store, "Match2", score: 85, price: 30.00m, createdAt: date2, isActive: true, status: "premium"); + + // Complex filter: score >= 70 AND price > 20 AND isActive = true AND status = "premium" + var filter = new NumberField("score").GreaterOrEqual(70) + .And(new NumberField("price").GreaterThan(20m)) + .And(new BooleanField("isActive").IsTrue()) + .And(new StringField("status").Equals("premium")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Match1"); + result.Items.ShouldContain(x => x.Value.Name == "Match2"); + } + + [Fact] + public async Task QueryCombineNumberRangesWithAndShouldReturnIntersectionAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1", score: 50, price: 10.0m); + _ = await CreateTestEntityAsync(store, "User2", score: 75, price: 20.0m); + _ = await CreateTestEntityAsync(store, "User3", score: 100, price: 30.0m); + _ = await CreateTestEntityAsync(store, "User4", score: 125, price: 40.0m); + + // Score between 60-110 AND price between 15-35 + var filter = new NumberField("score").Between(60, 110) + .And(new NumberField("price").Between(15.0m, 35.0m)); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "User2"); + result.Items.ShouldContain(x => x.Value.Name == "User3"); + } + + [Fact] + public async Task QueryCombineDateTimeRangesWithOrShouldReturnUnionAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var jan = new DateTimeOffset(2024, 1, 15, 0, 0, 0, TimeSpan.Zero); + var apr = new DateTimeOffset(2024, 4, 15, 0, 0, 0, TimeSpan.Zero); + var jul = new DateTimeOffset(2024, 7, 15, 0, 0, 0, TimeSpan.Zero); + var oct = new DateTimeOffset(2024, 10, 15, 0, 0, 0, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Event1", createdAt: jan); + _ = await CreateTestEntityAsync(store, "Event2", createdAt: apr); + _ = await CreateTestEntityAsync(store, "Event3", createdAt: jul); + _ = await CreateTestEntityAsync(store, "Event4", createdAt: oct); + + // Created in Q1 (Jan-Mar) OR Q4 (Oct-Dec) + var q1Start = new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc); + var q1End = new DateTime(2024, 3, 31, 23, 59, 59, DateTimeKind.Utc); + var q4Start = new DateTime(2024, 10, 1, 0, 0, 0, DateTimeKind.Utc); + var q4End = new DateTime(2024, 12, 31, 23, 59, 59, DateTimeKind.Utc); + + var filter = new DateTimeField("recordedAt").Between(q1Start, q1End) + .Or(new DateTimeField("recordedAt").Between(q4Start, q4End)); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Event1"); + result.Items.ShouldContain(x => x.Value.Name == "Event4"); + } + + [Fact] + public async Task QueryMixedFieldTypesComplexExpressionShouldWorkAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var date1 = new DateTimeOffset(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + var date2 = new DateTimeOffset(2024, 6, 1, 0, 0, 0, TimeSpan.Zero); + var date3 = new DateTimeOffset(2024, 12, 1, 0, 0, 0, TimeSpan.Zero); + + _ = await CreateTestEntityAsync(store, "Alice", score: 85, createdAt: date1, isActive: true, status: "premium"); + _ = await CreateTestEntityAsync(store, "Bob", score: 45, createdAt: date2, isActive: false, status: "basic"); + _ = await CreateTestEntityAsync(store, "Charlie", score: 95, createdAt: date3, isActive: true, status: "premium"); + _ = await CreateTestEntityAsync(store, "David", score: 75, createdAt: date2, isActive: true, status: "basic"); + + // Complex filter: (score >= 80 AND isActive) OR (createdAt after June 1 AND status = premium) + var juneFirst = new DateTime(2024, 6, 1, 0, 0, 0, DateTimeKind.Utc); + var filter = new NumberField("score").GreaterOrEqual(80) + .And(new BooleanField("isActive").IsTrue()) + .Or(new DateTimeField("recordedAt").GreaterThan(juneFirst) + .And(new StringField("status").Equals("premium"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); // Matches first part: score >= 80 AND active + result.Items.ShouldContain(x => x.Value.Name == "Charlie"); // Matches second part: after June AND premium + } + + [Fact] + public async Task QueryWithNumberRangeFilterShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 25 entities with scores from 10 to 250 (step of 10) + for (var i = 1; i <= 25; i++) + { + _ = await CreateEntityAsync(store, $"User{i:D2}", score: i * 10); + } + + // Filter: score between 50 and 200 (should match 16 items: 50, 60, 70...200) + var filter = new NumberField("score").Between(50, 200); + var sort = new SortParameter(new NumberField("score")); + + // Act - Get page 1 + var page1Request = DataRange.FromPage(1, 5); + var page1 = await store.QueryAsync(_testEntityType, filter, sort, page1Request, Ct.None); + + // Act - Get page 2 + var page2Request = DataRange.FromPage(2, 5); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, page2Request, Ct.None); + + // Act - Get page 3 + var page3Request = DataRange.FromPage(3, 5); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, page3Request, Ct.None); + + // Act - Get page 4 (last page with 1 item) + var page4Request = DataRange.FromPage(4, 5); + var page4 = await store.QueryAsync(_testEntityType, filter, sort, page4Request, Ct.None); + + // Assert - Page 1 + page1.Items.Count.ShouldBe(5); + page1.TotalCount.ShouldBe(16); + page1.HasMoreData.ShouldBeTrue(); + page1.Items[0].Value.Score.ShouldBe(50); + page1.Items[4].Value.Score.ShouldBe(90); + + // Assert - Page 2 + page2.Items.Count.ShouldBe(5); + page2.TotalCount.ShouldBe(16); + page2.HasMoreData.ShouldBeTrue(); + page2.Items[0].Value.Score.ShouldBe(100); + page2.Items[4].Value.Score.ShouldBe(140); + + // Assert - Page 3 + page3.Items.Count.ShouldBe(5); + page3.TotalCount.ShouldBe(16); + page3.HasMoreData.ShouldBeTrue(); + page3.Items[0].Value.Score.ShouldBe(150); + page3.Items[4].Value.Score.ShouldBe(190); + + // Assert - Page 4 (partial page) + page4.Items.Count.ShouldBe(1); + page4.TotalCount.ShouldBe(16); + page4.HasMoreData.ShouldBeFalse(); + page4.Items[0].Value.Score.ShouldBe(200); + } + + [Fact] + public async Task QueryWithDateTimeRangeFilterShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 30 entities spread across 2024 + for (var month = 1; month <= 12; month++) + { + for (var day = 1; day <= 15; day += 7) // Day 1, 8, 15 + { + var date = new DateTimeOffset(2024, month, day, 12, 0, 0, TimeSpan.Zero); + _ = await CreateEntityAsync(store, $"Event{month:D2}_{day:D2}", createdAt: date); + } + } + + // Filter: Q2 and Q3 (April 1 to September 30) - should match 18 items + var startDate = new DateTime(2024, 4, 1, 0, 0, 0, DateTimeKind.Utc); + var endDate = new DateTime(2024, 9, 30, 23, 59, 59, DateTimeKind.Utc); + var filter = new DateTimeField("recordedAt").Between(startDate, endDate); + var sort = new SortParameter(new DateTimeField("recordedAt")); + + // Act - Get first page + var page1Request = DataRange.FromPage(1, 7); + var page1 = await store.QueryAsync(_testEntityType, filter, sort, page1Request, Ct.None); + + // Act - Get second page + var page2Request = DataRange.FromPage(2, 7); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, page2Request, Ct.None); + + // Act - Get third page (partial) + var page3Request = DataRange.FromPage(3, 7); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, page3Request, Ct.None); + + // Assert + page1.Items.Count.ShouldBe(7); + page1.TotalCount.ShouldBe(18); + page1.HasMoreData.ShouldBeTrue(); + + page2.Items.Count.ShouldBe(7); + page2.HasMoreData.ShouldBeTrue(); + + page3.Items.Count.ShouldBe(4); // Last 4 items + page3.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QueryWithComplexFilterShouldPageCorrectlyAroundBreaksAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 50 entities + for (var i = 1; i <= 50; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", + score: i, + isActive: i % 2 == 0, // Even numbers are active + status: i % 3 == 0 ? "premium" : "basic"); + } + + // Filter: (score > 20 AND score < 45) AND isActive = true + // Should match: 22, 24, 26, 28, 30, 32, 34, 36, 38, 40, 42, 44 (12 items) + var filter = new NumberField("score").GreaterThan(20) + .And(new NumberField("score").LessThan(45)) + .And(new BooleanField("isActive").IsTrue()); + var sort = new SortParameter(new NumberField("score")); + + // Test exact page break: 12 items with page size 4 = exactly 3 pages + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 4), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 4), Ct.None); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 4), Ct.None); + var page4 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(4, 4), Ct.None); + + // Assert - Verify exact page break + page1.Items.Count.ShouldBe(4); + page1.TotalCount.ShouldBe(12); + page1.Items[0].Value.Score.ShouldBe(22); + page1.Items[3].Value.Score.ShouldBe(28); + + page2.Items.Count.ShouldBe(4); + page2.Items[0].Value.Score.ShouldBe(30); + page2.Items[3].Value.Score.ShouldBe(36); + + page3.Items.Count.ShouldBe(4); + page3.Items[0].Value.Score.ShouldBe(38); + page3.Items[3].Value.Score.ShouldBe(44); + page3.HasMoreData.ShouldBeFalse(); + + // Page 4 should be empty + page4.Items.Count.ShouldBe(0); + page4.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QuerySmallPageSizeShouldHandleManyPagesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create exactly 10 items + for (var i = 1; i <= 10; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", score: i * 10); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("score")); + + // Act - Use page size of 3 to create 4 pages (3+3+3+1) + var pages = new List>>(); + for (var pageNum = 1; pageNum <= 5; pageNum++) + { + var page = await store.QueryAsync(_testEntityType, filter, sort, + DataRange.FromPage(pageNum, 3), Ct.None); + pages.Add(page); + } + + // Assert + pages[0].Items.Count.ShouldBe(3); + pages[0].Items[0].Value.Score.ShouldBe(10); + pages[0].Items[2].Value.Score.ShouldBe(30); + + pages[1].Items.Count.ShouldBe(3); + pages[1].Items[0].Value.Score.ShouldBe(40); + pages[1].Items[2].Value.Score.ShouldBe(60); + + pages[2].Items.Count.ShouldBe(3); + pages[2].Items[0].Value.Score.ShouldBe(70); + pages[2].Items[2].Value.Score.ShouldBe(90); + + pages[3].Items.Count.ShouldBe(1); + pages[3].Items[0].Value.Score.ShouldBe(100); + pages[3].HasMoreData.ShouldBeFalse(); + + pages[4].Items.Count.ShouldBe(0); // Beyond last page + } + + [Fact] + public async Task QueryExactPageBoundaryShouldHandleCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create exactly 20 items (will be exactly 4 pages of 5) + for (var i = 1; i <= 20; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", score: i); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("score")); + + // Act + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 5), Ct.None); + var page4 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(4, 5), Ct.None); + var page5 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(5, 5), Ct.None); + + // Assert + // Last page should be full (not partial) + page4.Items.Count.ShouldBe(5); + page4.Items[0].Value.Score.ShouldBe(16); + page4.Items[4].Value.Score.ShouldBe(20); + page4.HasMoreData.ShouldBeFalse(); + + // Page beyond should be empty + page5.Items.Count.ShouldBe(0); + } + + [Fact] + public async Task QuerySingleItemResultShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 10; i++) + { + _ = await CreateEntityAsync(store, $"Item{i}", score: i * 10); + } + + // Filter that matches only one item + var filter = new NumberField("score").Equals(50); + var sort = new SortParameter(new NumberField("score")); + + // Act + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 5), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 5), Ct.None); + + // Assert + page1.Items.Count.ShouldBe(1); + page1.TotalCount.ShouldBe(1); + page1.HasMoreData.ShouldBeFalse(); + + page2.Items.Count.ShouldBe(0); + page2.TotalCount.ShouldBe(1); + } + + [Fact] + public async Task QueryStringFieldEndsWithShouldReturnSuffixMatchesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice Smith"); + _ = await CreateTestEntityAsync(store, "Charlie Smith"); + _ = await CreateTestEntityAsync(store, "Bob Jones"); + + var filter = new StringField("name").EndsWith("Smith"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice Smith"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie Smith"); + } + + [Fact] + public async Task QueryStringFieldEndsWithCaseInsensitiveShouldMatchAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice Smith"); + _ = await CreateTestEntityAsync(store, "Charlie Smith"); + _ = await CreateTestEntityAsync(store, "Bob Jones"); + + // Lowercase suffix - should match case-insensitively + var filter = new StringField("name").EndsWith("smith"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice Smith"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie Smith"); + } + + [Fact] + public async Task QueryStringFieldEndsWithWithPercentCharacterShouldTreatAsLiteralAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "discount50%"); + _ = await CreateTestEntityAsync(store, "discount50"); + _ = await CreateTestEntityAsync(store, "no match"); + + // The '%' character should be treated as a literal, not a wildcard + var filter = new StringField("name").EndsWith("50%"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("discount50%"); + } + + [Fact] + public async Task QueryStringFieldEndsWithWithUnderscoreCharacterShouldTreatAsLiteralAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "user_name"); + _ = await CreateTestEntityAsync(store, "username"); + _ = await CreateTestEntityAsync(store, "no match"); + + // The '_' character should be treated as a literal, not a single-character wildcard + var filter = new StringField("name").EndsWith("_name"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("user_name"); + } + + [Fact] + public async Task QueryStringFieldEndsWithNoMatchShouldReturnEmptyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alpha"); + _ = await CreateTestEntityAsync(store, "Beta"); + _ = await CreateTestEntityAsync(store, "Gamma"); + + var filter = new StringField("name").EndsWith("xyz_nomatch"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(0); + } + + [Fact] + public async Task QueryNotEqualShouldReturnNonMatchingEntitiesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice"); + _ = await CreateTestEntityAsync(store, "Bob"); + _ = await CreateTestEntityAsync(store, "Charlie"); + + var filter = Query.Not(new StringField("name").Equals("Alice")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Bob"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie"); + } + + [Fact] + public async Task QueryNotContainsShouldExcludePartialMatchesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "alice@example.com"); + _ = await CreateTestEntityAsync(store, "bob@test.com"); + _ = await CreateTestEntityAsync(store, "charlie@example.com"); + + var filter = Query.Not(new StringField("name").Contains("example")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("bob@test.com"); + } + + [Fact] + public async Task QueryNotStartsWithShouldExcludePrefixMatchesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alpha"); + _ = await CreateTestEntityAsync(store, "Alpha Centauri"); + _ = await CreateTestEntityAsync(store, "Beta"); + + var filter = Query.Not(new StringField("name").StartsWith("Alpha")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Beta"); + } + + [Fact] + public async Task QueryNotEndsWithShouldExcludeSuffixMatchesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice Smith"); + _ = await CreateTestEntityAsync(store, "Charlie Smith"); + _ = await CreateTestEntityAsync(store, "Bob Jones"); + + var filter = Query.Not(new StringField("name").EndsWith("Smith")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Bob Jones"); + } + + [Fact] + public async Task QueryNotCombinedWithAndShouldApplyNegationToWholeExpressionAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice", score: 80, isActive: true); + _ = await CreateTestEntityAsync(store, "Bob", score: 90, isActive: true); + _ = await CreateTestEntityAsync(store, "Charlie", score: 70, isActive: false); + _ = await CreateTestEntityAsync(store, "David", score: 60, isActive: true); + + // NOT(score > 75 AND isActive = true) => excludes Alice and Bob + var filter = Query.Not(new NumberField("score").GreaterThan(75) + .And(new BooleanField("isActive").IsTrue())); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Charlie"); + result.Items.ShouldContain(x => x.Value.Name == "David"); + } + + [Fact] + public async Task QueryNotCombinedWithOrShouldApplyNegationToWholeExpressionAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice", status: "active"); + _ = await CreateTestEntityAsync(store, "Bob", status: "pending"); + _ = await CreateTestEntityAsync(store, "Charlie", status: "inactive"); + _ = await CreateTestEntityAsync(store, "David", status: "banned"); + + // NOT(status = "active" OR status = "pending") => only inactive and banned + var filter = Query.Not(new StringField("status").Equals("active") + .Or(new StringField("status").Equals("pending"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Charlie"); + result.Items.ShouldContain(x => x.Value.Name == "David"); + } + + [Fact] + public async Task QueryDoubleNotShouldReturnOriginalMatchesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice"); + _ = await CreateTestEntityAsync(store, "Bob"); + _ = await CreateTestEntityAsync(store, "Charlie"); + + // NOT(NOT(name = "Alice")) is equivalent to name = "Alice" + var filter = Query.Not(Query.Not(new StringField("name").Equals("Alice"))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Alice"); + } + + [Fact] + public async Task QueryNotBooleanShouldReturnOppositeValuesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Active1", isActive: true); + _ = await CreateTestEntityAsync(store, "Inactive1", isActive: false); + _ = await CreateTestEntityAsync(store, "Active2", isActive: true); + + var filter = Query.Not(new BooleanField("isActive").IsTrue()); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Inactive1"); + } + + [Fact] + public async Task QueryNotPresentShouldReturnEntitiesWithMissingFieldAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "WithScore", score: 42); + _ = await CreateTestEntityAsync(store, "WithoutScore"); + + var filter = Query.Not(new NumberField("score").Present()); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("WithoutScore"); + } + + [Fact] + public async Task QueryStringFieldPresentShouldReturnEntitiesWithFieldSetAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "WithStatus", status: "active"); + _ = await CreateTestEntityAsync(store, "WithoutStatus"); + _ = await CreateTestEntityAsync(store, "AlsoWithStatus", status: "pending"); + + var filter = new StringField("status").Present(); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "WithStatus"); + result.Items.ShouldContain(x => x.Value.Name == "AlsoWithStatus"); + } + + [Fact] + public async Task QueryNumberFieldPresentShouldReturnEntitiesWithFieldSetAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "WithScore", score: 42); + _ = await CreateTestEntityAsync(store, "WithoutScore"); + _ = await CreateTestEntityAsync(store, "AlsoWithScore", score: 99); + + var filter = new NumberField("score").Present(); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "WithScore"); + result.Items.ShouldContain(x => x.Value.Name == "AlsoWithScore"); + } + + [Fact] + public async Task QueryDateTimeFieldPresentShouldReturnEntitiesWithFieldSetAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var date = new DateTimeOffset(2024, 6, 15, 12, 0, 0, TimeSpan.Zero); + _ = await CreateTestEntityAsync(store, "WithDate", createdAt: date); + _ = await CreateTestEntityAsync(store, "WithoutDate"); + _ = await CreateTestEntityAsync(store, "AlsoWithDate", createdAt: date.AddDays(1)); + + var filter = new DateTimeField("recordedAt").Present(); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "WithDate"); + result.Items.ShouldContain(x => x.Value.Name == "AlsoWithDate"); + } + + [Fact] + public async Task QueryBooleanFieldPresentShouldReturnEntitiesWithFieldSetAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "WithFlag", isActive: true); + _ = await CreateTestEntityAsync(store, "WithFalseFlag", isActive: false); + _ = await CreateTestEntityAsync(store, "WithoutFlag"); + + var filter = new BooleanField("isActive").Present(); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "WithFlag"); + result.Items.ShouldContain(x => x.Value.Name == "WithFalseFlag"); + } + + [Fact] + public async Task QueryPresentWhenNoEntitiesHaveFieldShouldReturnEmptyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "User1"); + _ = await CreateTestEntityAsync(store, "User2"); + + var filter = new NumberField("score").Present(); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(0); + } + + [Fact] + public async Task QueryPresentCombinedWithOtherFilterShouldNarrowResultsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestEntityAsync(store, "Alice", score: 80, status: "active"); + _ = await CreateTestEntityAsync(store, "Bob", score: 50); // no status + _ = await CreateTestEntityAsync(store, "Charlie", status: "active"); // no score + + // Present(score) AND status = "active" + var filter = new NumberField("score").Present() + .And(new StringField("status").Equals("active")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Alice"); + } + + private static readonly string[] AdminUserTags = ["admin", "user"]; + private static readonly string[] UserOnlyTags = ["user"]; + private static readonly string[] AdminModeratorTags = ["admin", "moderator"]; + private static readonly string[] AdminOnlyTags = ["admin"]; + private static readonly EntityType UserEntityType = new(2, "UserEntity"); + + [Fact] + public async Task QueryArrayContainsShouldReturnEntitiesWithMatchingArrayElementAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestUserWithTagsAsync(store, "Alice", AdminUserTags); + _ = await CreateTestUserWithTagsAsync(store, "Bob", UserOnlyTags); + _ = await CreateTestUserWithTagsAsync(store, "Charlie", AdminModeratorTags); + + var filter = new StringArrayField("tags").Contains("admin"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(UserEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie"); + } + + [Fact] + public async Task QueryArrayContainsNoMatchShouldReturnEmptyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestUserWithTagsAsync(store, "Alice", AdminUserTags); + _ = await CreateTestUserWithTagsAsync(store, "Bob", UserOnlyTags); + + var filter = new StringArrayField("tags").Contains("superadmin"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(UserEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(0); + } + + [Fact] + public async Task QueryArrayContainsCombinedWithStringFilterShouldNarrowResultsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateTestUserWithTagsAsync(store, "Alice", AdminUserTags); + _ = await CreateTestUserWithTagsAsync(store, "AdminBob", AdminOnlyTags); + _ = await CreateTestUserWithTagsAsync(store, "Bob", UserOnlyTags); + + // tags contains "admin" AND name starts with "Alice" + var filter = new StringArrayField("tags").Contains("admin") + .And(new StringField("name").StartsWith("Alice")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(UserEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Alice"); + } + + private static async Task CreateTestUserWithTagsAsync( + IStore store, + string name, + string[] tags, + Ct ct = default) + { + var id = UuidV7.New(); + var dso = new TestUserDso { Name = name }; + var searchFieldsBuilder = new SearchFieldsBuilder(); + _ = searchFieldsBuilder.Add("name", name); + for (var i = 0; i < tags.Length; i++) + { + _ = searchFieldsBuilder.Add("tags", i, tags[i]); + } + + var searchFields = searchFieldsBuilder.Build(); + var storeInterface = store; + var result = await storeInterface.CreateAsync(id, dso, Array.Empty(), searchFields, Expiration.NoExpiration, [], ct); + result.ShouldBe(CreateResult.Success); + return id; + } + +} diff --git a/storage/test/SharedIntegrationTests/QueryStoreCountTests.cs b/storage/test/SharedIntegrationTests/QueryStoreCountTests.cs new file mode 100644 index 000000000..927d5dc87 --- /dev/null +++ b/storage/test/SharedIntegrationTests/QueryStoreCountTests.cs @@ -0,0 +1,150 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Querying.Expressions; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Internal.Querying.SearchFields; + +namespace Duende.Storage.IntegrationTests; + +/// +/// Tests for IStore.CountAsync across all store implementations. +/// +public partial class QueryStoreCountTests +{ + private readonly EntityType _testEntityType = new(3, "TestEntity"); + + private readonly Ct _ct = TestContext.Current.CancellationToken; + + private async Task CreateProviderAsync() => + await FixtureFactory.CreateAsync(_ct, services => services.AddDsoRegistration()); + + private static async Task CreateEntityAsync(IStore store, string name, int? score = null, Ct ct = default) + { + var id = UuidV7.New(); + var dso = new TestEntityDso { Name = name, Score = score }; + + var searchFieldsBuilder = new SearchFieldsBuilder(); + _ = searchFieldsBuilder.Add("name", name); + if (score.HasValue) + { + _ = searchFieldsBuilder.Add("score", score.Value); + } + + var searchFields = searchFieldsBuilder.Build(); + + var result = await store.CreateAsync(id, dso, Array.Empty(), searchFields, Expiration.NoExpiration, [], ct); + result.ShouldBe(CreateResult.Success); + } + + [Fact] + public async Task CountAsync_with_no_filter_should_return_total_count() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + await CreateEntityAsync(store, "Alice", ct: _ct); + await CreateEntityAsync(store, "Bob", ct: _ct); + await CreateEntityAsync(store, "Charlie", ct: _ct); + + // Act + var count = await store.CountAsync(_testEntityType, null, _ct); + + // Assert + count.ShouldBe(3); + } + + [Fact] + public async Task CountAsync_with_AllExpression_should_return_total_count() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + await CreateEntityAsync(store, "Alice", ct: _ct); + await CreateEntityAsync(store, "Bob", ct: _ct); + + // Act + var count = await store.CountAsync(_testEntityType, AllExpression.Instance, _ct); + + // Assert + count.ShouldBe(2); + } + + [Fact] + public async Task CountAsync_with_filter_should_return_matching_count() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + await CreateEntityAsync(store, "Alice", score: 100, ct: _ct); + await CreateEntityAsync(store, "Bob", score: 50, ct: _ct); + await CreateEntityAsync(store, "Charlie", score: 100, ct: _ct); + + var filter = new NumberField("score").Equals(100); + + // Act + var count = await store.CountAsync(_testEntityType, filter, _ct); + + // Assert + count.ShouldBe(2); + } + + [Fact] + public async Task CountAsync_with_no_matching_entities_should_return_zero() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + await CreateEntityAsync(store, "Alice", ct: _ct); + await CreateEntityAsync(store, "Bob", ct: _ct); + + var filter = new StringField("name").Equals("NonExistent"); + + // Act + var count = await store.CountAsync(_testEntityType, filter, _ct); + + // Assert + count.ShouldBe(0); + } + + [Fact] + public async Task CountAsync_with_empty_store_should_return_zero() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Act + var count = await store.CountAsync(_testEntityType, null, _ct); + + // Assert + count.ShouldBe(0); + } + + [Fact] + public async Task CountAsync_with_string_filter_should_count_matching_entities() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + await CreateEntityAsync(store, "Alice Smith", ct: _ct); + await CreateEntityAsync(store, "Bob Jones", ct: _ct); + await CreateEntityAsync(store, "Charlie Smith", ct: _ct); + + var filter = new StringField("name").Contains("Smith"); + + // Act + var count = await store.CountAsync(_testEntityType, filter, _ct); + + // Assert + count.ShouldBe(2); + } +} diff --git a/storage/test/SharedIntegrationTests/QueryStoreCursorPagingTests.cs b/storage/test/SharedIntegrationTests/QueryStoreCursorPagingTests.cs new file mode 100644 index 000000000..2cce53160 --- /dev/null +++ b/storage/test/SharedIntegrationTests/QueryStoreCursorPagingTests.cs @@ -0,0 +1,684 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Internal.Querying.SearchFields; +using Duende.Storage.Pagination; +using SortDirection = Duende.Storage.Querying.SortDirection; +using SortParameter = Duende.Storage.Internal.Querying.Sorting.SortParameter; + +namespace Duende.Storage.IntegrationTests; + +/// +/// Tests for cursor-based pagination functionality across all store implementations. +/// Covers first page, continuation, last page, and sort requirement verification. +/// +public partial class QueryStoreCursorPagingTests +{ + private readonly EntityType _testEntityType = TestCursorDso.DsoVersion.EntityType; + + private readonly Ct _ct = TestContext.Current.CancellationToken; + + private async Task CreateProviderAsync() => + await FixtureFactory.CreateAsync(_ct, services => + { + services.AddDsoRegistration(); + }); + + private static async Task CreateEntityAsync( + IStore store, + string name, + int rank, + DateTimeOffset? createdAt = null, + bool? isActive = null, + Ct ct = default) + { + var id = UuidV7.New(); + var dso = new TestCursorDso + { + Name = name, + Rank = rank, + CreatedAt = createdAt ?? DateTimeOffset.UtcNow, + IsActive = isActive ?? true + }; + + var searchFieldsBuilder = new SearchFieldsBuilder(); + _ = searchFieldsBuilder.Add("name", name); + _ = searchFieldsBuilder.Add("rank", rank); + _ = searchFieldsBuilder.Add("recordedAt", dso.CreatedAt); + _ = searchFieldsBuilder.Add("isActive", dso.IsActive); + + var searchFields = searchFieldsBuilder.Build(); + + var result = await store.CreateAsync(id, dso, Array.Empty(), searchFields, Expiration.NoExpiration, [], ct); + result.ShouldBe(CreateResult.Success); + return id; + } + + [Fact] + public async Task QueryCursorFirstPageNullTokenShouldReturnFirstPageWithNextTokenAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 10 items + for (var i = 1; i <= 10; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i * 10, ct: _ct); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 3); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, cursor, Ct.None); + + // Assert + result.Items.Count.ShouldBe(3); + result.Items[0].Value.Name.ShouldBe("Item01"); + result.Items[1].Value.Name.ShouldBe("Item02"); + result.Items[2].Value.Name.ShouldBe("Item03"); + _ = result.NextToken.ShouldNotBeNull(); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryFieldsCursorFirstPageNullTokenShouldReturnFirstPageWithNextTokenAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 8; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i * 5); + } + + var filter = Query.All(); + var sort = new SortParameter(new StringField("name")); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 4); + var fields = new List { new StringField("name"), new NumberField("rank") }; + + // Act + var result = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, cursor, Ct.None); + + // Assert + result.Items.Count.ShouldBe(4); + result.Items[0].Fields["NAME"].ShouldBe("ITEM01"); + result.Items[3].Fields["NAME"].ShouldBe("ITEM04"); + _ = result.NextToken.ShouldNotBeNull(); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryCursorContinuationWithNextTokenShouldReturnCorrectSubsequentPagesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 10 items + for (var i = 1; i <= 10; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 3); + + // Act - Page 1 + var page1 = await store.QueryAsync(_testEntityType, filter, sort, cursor, Ct.None); + + // Act - Page 2 using NextToken from page 1 + var cursor2 = DataRange.FromContinuationToken(page1.NextToken!.Value, 3); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, cursor2, Ct.None); + + // Act - Page 3 using NextToken from page 2 + var cursor3 = DataRange.FromContinuationToken(page2.NextToken!.Value, 3); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, cursor3, Ct.None); + + // Assert + page1.Items.Count.ShouldBe(3); + page1.Items[0].Value.Rank.ShouldBe(1); + page1.Items[2].Value.Rank.ShouldBe(3); + page1.HasMoreData.ShouldBeTrue(); + + page2.Items.Count.ShouldBe(3); + page2.Items[0].Value.Rank.ShouldBe(4); + page2.Items[2].Value.Rank.ShouldBe(6); + page2.HasMoreData.ShouldBeTrue(); + + page3.Items.Count.ShouldBe(3); + page3.Items[0].Value.Rank.ShouldBe(7); + page3.Items[2].Value.Rank.ShouldBe(9); + page3.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryFieldsCursorContinuationWithNextTokenShouldReturnCorrectSubsequentPagesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 7; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i * 10); + } + + var filter = Query.All(); + var sort = new SortParameter(new StringField("name"), SortDirection.Descending); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 3); + var fields = new List { new StringField("name"), new NumberField("rank") }; + + // Act - Page 1 + var page1 = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, cursor, Ct.None); + + // Act - Page 2 + var cursor2 = DataRange.FromContinuationToken(page1.NextToken!.Value, 3); + var page2 = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, cursor2, Ct.None); + + // Assert + page1.Items.Count.ShouldBe(3); + page1.Items[0].Fields["NAME"].ShouldBe("ITEM07"); + page1.Items[2].Fields["NAME"].ShouldBe("ITEM05"); + + page2.Items.Count.ShouldBe(3); + page2.Items[0].Fields["NAME"].ShouldBe("ITEM04"); + page2.Items[2].Fields["NAME"].ShouldBe("ITEM02"); + } + + [Fact] + public async Task QueryCursorLastPageShouldHaveHasMoreFalseAndNullNextTokenAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 7 items (3+3+1) + for (var i = 1; i <= 7; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i * 5); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + + // Act - Navigate to last page + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromContinuationToken(ContinuationToken.Beginning, 3), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromContinuationToken(page1.NextToken!.Value, 3), Ct.None); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromContinuationToken(page2.NextToken!.Value, 3), Ct.None); + + // Assert + page3.Items.Count.ShouldBe(1); + page3.Items[0].Value.Rank.ShouldBe(35); + _ = page3.NextToken.ShouldNotBeNull(); // Token always set when items exist (enables resumption) + page3.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QueryFieldsCursorLastPageShouldHaveHasMoreFalseAndNullNextTokenAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create exactly 5 items with page size 5 + for (var i = 1; i <= 5; i++) + { + _ = await CreateEntityAsync(store, $"Item{i}", i); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 5); + var fields = new List { new StringField("name") }; + + // Act + var result = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, cursor, Ct.None); + + // Assert - Exactly one full page, no more pages + result.Items.Count.ShouldBe(5); + _ = result.NextToken.ShouldNotBeNull(); // Token always set when items exist + result.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QueryCursorEmptyResultShouldReturnEmptyWithNoNextTokenAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Item1", 10); + _ = await CreateEntityAsync(store, "Item2", 20); + + var filter = new NumberField("rank").GreaterThan(100); + var sort = new SortParameter(new NumberField("rank")); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, cursor, Ct.None); + + // Assert + result.Items.Count.ShouldBe(0); + result.NextToken.ShouldBeNull(); + result.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QueryCursorWithoutSortShouldThrowArgumentNullExceptionAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Item1", 10); + + var filter = Query.All(); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 10); + + // Act & Assert + _ = await Should.ThrowAsync(async () => + await store.QueryAsync(_testEntityType, filter, null!, cursor, Ct.None)); + } + + [Fact] + public async Task QueryFieldsCursorWithoutSortShouldThrowArgumentNullExceptionAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Item1", 10); + + var filter = Query.All(); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 10); + var fields = new List { new StringField("name") }; + + // Act & Assert + _ = await Should.ThrowAsync(async () => + await store.QueryFieldsAsync(_testEntityType, fields, filter, null!, cursor, Ct.None)); + } + + [Fact] + public async Task QueryCursorWithDuplicateSortValuesShouldMaintainStableOrderingAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create items with duplicate rank values + var id1 = await CreateEntityAsync(store, "Alice", 100); + var id2 = await CreateEntityAsync(store, "Bob", 100); + var id3 = await CreateEntityAsync(store, "Charlie", 100); + var id4 = await CreateEntityAsync(store, "David", 50); + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank"), SortDirection.Descending); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 2); + + // Act - Page 1 + var page1 = await store.QueryAsync(_testEntityType, filter, sort, cursor, Ct.None); + + // Act - Page 2 + var cursor2 = DataRange.FromContinuationToken(page1.NextToken!.Value, 2); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, cursor2, Ct.None); + + // Assert - All items with rank 100 should come before rank 50 + page1.Items.Count.ShouldBe(2); + page1.Items[0].Value.Rank.ShouldBe(100); + page1.Items[1].Value.Rank.ShouldBe(100); + + page2.Items.Count.ShouldBe(2); + page2.Items[0].Value.Rank.ShouldBe(100); + page2.Items[1].Value.Rank.ShouldBe(50); + page2.Items[1].Value.Name.ShouldBe("David"); + } + + [Fact] + public async Task QueryCursorDescendingSortShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 9; i++) + { + _ = await CreateEntityAsync(store, $"Item{i}", i * 10); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank"), SortDirection.Descending); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 4); + + // Act + var page1 = await store.QueryAsync(_testEntityType, filter, sort, cursor, Ct.None); + + // Assert + page1.Items.Count.ShouldBe(4); + page1.Items[0].Value.Rank.ShouldBe(90); // Highest first + page1.Items[3].Value.Rank.ShouldBe(60); + page1.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryCursorWithFilterShouldPageFilteredResultsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 20 items, half odd, half even + for (var i = 1; i <= 20; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i); + } + + var filter = new NumberField("rank").GreaterThan(10); + var sort = new SortParameter(new NumberField("rank")); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 5); + + // Act + var page1 = await store.QueryAsync(_testEntityType, filter, sort, cursor, Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromContinuationToken(page1.NextToken!.Value, 5), Ct.None); + + // Assert + page1.Items.Count.ShouldBe(5); + page1.Items[0].Value.Rank.ShouldBe(11); + page1.Items[4].Value.Rank.ShouldBe(15); + + page2.Items.Count.ShouldBe(5); + page2.Items[0].Value.Rank.ShouldBe(16); + page2.Items[4].Value.Rank.ShouldBe(20); + } + + [Fact] + public async Task QueryCursorSingleItemShouldReturnOneItemWithNoNextToken() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "OnlyOne", 42); + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, cursor, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("OnlyOne"); + _ = result.NextToken.ShouldNotBeNull(); // Token always set when items exist + result.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QueryCursorSortByDateTimeAscendingShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var baseDate = new DateTimeOffset(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + for (var i = 1; i <= 10; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i * 10, baseDate.AddDays(i), true); + } + + var filter = Query.All(); + var sort = new SortParameter(new DateTimeField("recordedAt")); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 3); + + // Act + var page1 = await store.QueryAsync(_testEntityType, filter, sort, cursor, Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromContinuationToken(page1.NextToken!.Value, 3), Ct.None); + + // Assert + page1.Items.Count.ShouldBe(3); + page1.Items[0].Value.CreatedAt.ShouldBe(baseDate.AddDays(1)); + page1.Items[1].Value.CreatedAt.ShouldBe(baseDate.AddDays(2)); + page1.Items[2].Value.CreatedAt.ShouldBe(baseDate.AddDays(3)); + page1.HasMoreData.ShouldBeTrue(); + + page2.Items.Count.ShouldBe(3); + page2.Items[0].Value.CreatedAt.ShouldBe(baseDate.AddDays(4)); + page2.Items[1].Value.CreatedAt.ShouldBe(baseDate.AddDays(5)); + page2.Items[2].Value.CreatedAt.ShouldBe(baseDate.AddDays(6)); + } + + [Fact] + public async Task QueryCursorSortByDateTimeDescendingShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var baseDate = new DateTimeOffset(2024, 6, 15, 12, 0, 0, TimeSpan.Zero); + for (var i = 1; i <= 8; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i, baseDate.AddHours(i), true); + } + + var filter = Query.All(); + var sort = new SortParameter(new DateTimeField("recordedAt"), SortDirection.Descending); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 4); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, cursor, Ct.None); + + // Assert - Should return most recent dates first + result.Items.Count.ShouldBe(4); + result.Items[0].Value.CreatedAt.ShouldBe(baseDate.AddHours(8)); + result.Items[1].Value.CreatedAt.ShouldBe(baseDate.AddHours(7)); + result.Items[2].Value.CreatedAt.ShouldBe(baseDate.AddHours(6)); + result.Items[3].Value.CreatedAt.ShouldBe(baseDate.AddHours(5)); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryFieldsCursorSortByDateTimeAscendingShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var baseDate = new DateTimeOffset(2024, 3, 10, 0, 0, 0, TimeSpan.Zero); + for (var i = 1; i <= 6; i++) + { + _ = await CreateEntityAsync(store, $"Item{i}", i * 5, baseDate.AddMonths(i), true); + } + + var filter = Query.All(); + var sort = new SortParameter(new DateTimeField("recordedAt")); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 3); + var fields = new List { new StringField("name"), new DateTimeField("recordedAt") }; + + // Act + var result = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, cursor, Ct.None); + + // Assert + result.Items.Count.ShouldBe(3); + ((DateTimeOffset)result.Items[0].Fields["RECORDEDAT"]!).ShouldBe(baseDate.AddMonths(1).UtcDateTime); + ((DateTimeOffset)result.Items[1].Fields["RECORDEDAT"]!).ShouldBe(baseDate.AddMonths(2).UtcDateTime); + ((DateTimeOffset)result.Items[2].Fields["RECORDEDAT"]!).ShouldBe(baseDate.AddMonths(3).UtcDateTime); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryCursorSortByBooleanAscendingShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var baseDate = DateTimeOffset.UtcNow; + // Create items with mixed boolean values + _ = await CreateEntityAsync(store, "Item1", 10, baseDate.AddDays(1), false); + _ = await CreateEntityAsync(store, "Item2", 20, baseDate.AddDays(2), false); + _ = await CreateEntityAsync(store, "Item3", 30, baseDate.AddDays(3), false); + _ = await CreateEntityAsync(store, "Item4", 40, baseDate.AddDays(4), true); + _ = await CreateEntityAsync(store, "Item5", 50, baseDate.AddDays(5), true); + _ = await CreateEntityAsync(store, "Item6", 60, baseDate.AddDays(6), true); + + var filter = Query.All(); + var sort = new SortParameter(new BooleanField("isActive")); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 4); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, cursor, Ct.None); + + // Assert - false values should come before true (ascending) + result.Items.Count.ShouldBe(4); + result.Items[0].Value.IsActive.ShouldBeFalse(); + result.Items[1].Value.IsActive.ShouldBeFalse(); + result.Items[2].Value.IsActive.ShouldBeFalse(); + result.Items[3].Value.IsActive.ShouldBeTrue(); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryCursorSortByBooleanDescendingShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var baseDate = DateTimeOffset.UtcNow; + // Create items with mixed boolean values + for (var i = 1; i <= 8; i++) + { + var isActive = i % 3 == 0; // Every third item is active + _ = await CreateEntityAsync(store, $"Item{i}", i * 10, baseDate.AddDays(i), isActive); + } + + var filter = Query.All(); + var sort = new SortParameter(new BooleanField("isActive"), SortDirection.Descending); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 3); + + // Act + var page1 = await store.QueryAsync(_testEntityType, filter, sort, cursor, Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromContinuationToken(page1.NextToken!.Value, 3), Ct.None); + + // Assert - true values should come before false (descending) + page1.Items.Count.ShouldBe(3); + page1.Items[0].Value.IsActive.ShouldBeTrue(); + page1.Items[1].Value.IsActive.ShouldBeTrue(); + // Third item depends on the count of true values + + page2.Items.Count.ShouldBe(3); + // Should contain mix or all false values + } + + [Fact] + public async Task QueryFieldsCursorSortByBooleanAscendingShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var baseDate = DateTimeOffset.UtcNow; + _ = await CreateEntityAsync(store, "Inactive1", 10, baseDate, false); + _ = await CreateEntityAsync(store, "Inactive2", 20, baseDate.AddDays(1), false); + _ = await CreateEntityAsync(store, "Active1", 30, baseDate.AddDays(2), true); + _ = await CreateEntityAsync(store, "Active2", 40, baseDate.AddDays(3), true); + _ = await CreateEntityAsync(store, "Active3", 50, baseDate.AddDays(4), true); + + var filter = Query.All(); + var sort = new SortParameter(new BooleanField("isActive")); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 3); + var fields = new List { new StringField("name"), new BooleanField("isActive") }; + + // Act + var result = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, cursor, Ct.None); + + // Assert + result.Items.Count.ShouldBe(3); + ((bool)result.Items[0].Fields["ISACTIVE"]!).ShouldBeFalse(); + ((bool)result.Items[1].Fields["ISACTIVE"]!).ShouldBeFalse(); + ((bool)result.Items[2].Fields["ISACTIVE"]!).ShouldBeTrue(); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryCursorResumeFromEndShouldReturnNewlyAddedItemsAsync() + { + // Arrange — create 5 items, page size 3 → two pages (3 + 2) + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 5; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i * 10); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + + // Act — page through to the end + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromContinuationToken(ContinuationToken.Beginning, 3), _ct); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromContinuationToken(page1.NextToken!.Value, 3), _ct); + + // Assert — we've reached the end + page2.Items.Count.ShouldBe(2); + page2.Items[0].Value.Rank.ShouldBe(40); + page2.Items[1].Value.Rank.ShouldBe(50); + page2.HasMoreData.ShouldBeFalse(); + var resumeToken = page2.NextToken.ShouldNotBeNull(); // Token is still set for resumption + + // Act — new data arrives after we hit the end + _ = await CreateEntityAsync(store, "Item06", 60); + _ = await CreateEntityAsync(store, "Item07", 70); + + // Resume from the saved token — should pick up the new items + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromContinuationToken(resumeToken.Value, 3), _ct); + + // Assert — the new items are returned + page3.Items.Count.ShouldBe(2); + page3.Items[0].Value.Rank.ShouldBe(60); + page3.Items[1].Value.Rank.ShouldBe(70); + page3.HasMoreData.ShouldBeFalse(); + _ = page3.NextToken.ShouldNotBeNull(); + } + + [Fact] + public async Task QueryCursorResumeFromEndWithNoNewDataShouldReturnEmptyAsync() + { + // Arrange — create 3 items, page size 3 → exactly one page + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 3; i++) + { + _ = await CreateEntityAsync(store, $"Item{i}", i * 10); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + + // Act — fetch the only page + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromContinuationToken(ContinuationToken.Beginning, 3), _ct); + + page1.Items.Count.ShouldBe(3); + page1.HasMoreData.ShouldBeFalse(); + var resumeToken = page1.NextToken.ShouldNotBeNull(); + + // Act — resume with no new data + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromContinuationToken(resumeToken.Value, 3), _ct); + + // Assert — empty result, no token (no items returned) + page2.Items.Count.ShouldBe(0); + page2.NextToken.ShouldBeNull(); + page2.HasMoreData.ShouldBeFalse(); + } +} diff --git a/storage/test/SharedIntegrationTests/QueryStoreGuidFieldTests.cs b/storage/test/SharedIntegrationTests/QueryStoreGuidFieldTests.cs new file mode 100644 index 000000000..07dc10e11 --- /dev/null +++ b/storage/test/SharedIntegrationTests/QueryStoreGuidFieldTests.cs @@ -0,0 +1,724 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Internal.Querying.SearchFields; +using Duende.Storage.Pagination; +using SortParameter = Duende.Storage.Internal.Querying.Sorting.SortParameter; + +namespace Duende.Storage.IntegrationTests; + +/// +/// Tests for GuidField, ExactMatchField, and the guid_value column across all store implementations. +/// Covers: +/// - GuidField: Equals, In, Present +/// - ExactMatchField: Equals, In, Present, case-insensitivity +/// - StringField.Equals/In routing through guid_value (deterministic hash) +/// - Logical composition (And, Or, Not) with guid-based fields +/// - Array fields with guid_value +/// +public partial class QueryStoreGuidFieldTests +{ + + private readonly EntityType _guidEntityType = new(6, "GuidTestEntity"); + private readonly EntityType _userEntityType = new(2, "UserEntity"); + + private readonly Ct _ct = TestContext.Current.CancellationToken; + + private async Task CreateProviderAsync() => + await FixtureFactory.CreateAsync(_ct, services => + { + services.AddDsoRegistration(); + services.AddDsoRegistration(); + }); + + private static async Task CreateGuidEntityAsync( + IStore store, + string name, + Guid? resourceId = null, + string? apiKey = null, + string? tag = null, + Ct ct = default) + { + var id = UuidV7.New(); + var dso = new TestGuidEntityDso + { + Name = name, + ResourceId = resourceId, + ApiKey = apiKey, + Tag = tag + }; + + var searchFieldsBuilder = new SearchFieldsBuilder(); + _ = searchFieldsBuilder.Add("name", name); + + if (resourceId.HasValue) + { + _ = searchFieldsBuilder.Add("resourceId", resourceId.Value); + } + + if (apiKey != null) + { + _ = searchFieldsBuilder.AddExactMatch("apiKey", apiKey); + } + + if (tag != null) + { + _ = searchFieldsBuilder.Add("tag", tag); + } + + var searchFields = searchFieldsBuilder.Build(); + var storeInterface = store; + var result = await storeInterface.CreateAsync(id, dso, Array.Empty(), searchFields, Expiration.NoExpiration, [], ct); + result.ShouldBe(CreateResult.Success); + return id; + } + + private static async Task CreateUserWithEmailsAndGuidsAsync( + IStore store, + string name, + (string type, string value, Guid? correlationId)[] emails, + Ct ct) + { + var id = UuidV7.New(); + var dso = new TestUserDso + { + Name = name, + Emails = emails.Select(e => new EmailAddress { Type = e.type, Value = e.value }).ToArray() + }; + + var searchFieldsBuilder = new SearchFieldsBuilder(); + _ = searchFieldsBuilder.Add("name", name); + + for (var i = 0; i < emails.Length; i++) + { + _ = searchFieldsBuilder.Add("emails.type", i, emails[i].type); + _ = searchFieldsBuilder.Add("emails.value", i, emails[i].value); + if (emails[i].correlationId is { } correlationId) + { + _ = searchFieldsBuilder.Add("emails.correlationId", i, correlationId); + } + } + + var searchFields = searchFieldsBuilder.Build(); + var storeInterface = store; + var result = await storeInterface.CreateAsync(id, dso, Array.Empty(), searchFields, Expiration.NoExpiration, [], ct); + result.ShouldBe(CreateResult.Success); + return id; + } + + [Fact] + public async Task GuidFieldEqualsShouldReturnExactMatchAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var guid1 = Guid.NewGuid(); + var guid2 = Guid.NewGuid(); + var guid3 = Guid.NewGuid(); + + _ = await CreateGuidEntityAsync(store, "Entity1", resourceId: guid1); + _ = await CreateGuidEntityAsync(store, "Entity2", resourceId: guid2); + _ = await CreateGuidEntityAsync(store, "Entity3", resourceId: guid3); + + var filter = new GuidField("resourceId").Equals(guid2); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Entity2"); + } + + [Fact] + public async Task GuidFieldEqualsWithNoMatchShouldReturnEmptyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateGuidEntityAsync(store, "Entity1", resourceId: Guid.NewGuid()); + + var filter = new GuidField("resourceId").Equals(Guid.NewGuid()); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(0); + } + + [Fact] + public async Task GuidFieldInShouldReturnMatchingEntitiesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var guid1 = Guid.NewGuid(); + var guid2 = Guid.NewGuid(); + var guid3 = Guid.NewGuid(); + + _ = await CreateGuidEntityAsync(store, "Entity1", resourceId: guid1); + _ = await CreateGuidEntityAsync(store, "Entity2", resourceId: guid2); + _ = await CreateGuidEntityAsync(store, "Entity3", resourceId: guid3); + + Guid[] searchGuids = [guid1, guid3]; + var filter = new GuidField("resourceId").In(searchGuids); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Entity1"); + result.Items.ShouldContain(x => x.Value.Name == "Entity3"); + } + + [Fact] + public async Task GuidFieldPresentShouldReturnEntitiesWithFieldSetAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateGuidEntityAsync(store, "WithGuid", resourceId: Guid.NewGuid()); + _ = await CreateGuidEntityAsync(store, "WithoutGuid"); + _ = await CreateGuidEntityAsync(store, "AlsoWithGuid", resourceId: Guid.NewGuid()); + + var filter = new GuidField("resourceId").Present(); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "WithGuid"); + result.Items.ShouldContain(x => x.Value.Name == "AlsoWithGuid"); + } + + [Fact] + public async Task ExactMatchFieldEqualsShouldReturnExactMatchAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateGuidEntityAsync(store, "Entity1", apiKey: "secret-key-123"); + _ = await CreateGuidEntityAsync(store, "Entity2", apiKey: "different-key-456"); + _ = await CreateGuidEntityAsync(store, "Entity3", apiKey: "another-key-789"); + + var filter = new ExactMatchField("apiKey").Equals("secret-key-123"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Entity1"); + } + + [Fact] + public async Task ExactMatchFieldEqualsShouldBeCaseInsensitiveAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateGuidEntityAsync(store, "Entity1", apiKey: "My-Secret-Key"); + _ = await CreateGuidEntityAsync(store, "Entity2", apiKey: "other-key"); + + // Act — query with different casing than what was stored + var filter = new ExactMatchField("apiKey").Equals("MY-SECRET-KEY"); + var page = DataRange.FromPage(1, 10); + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert — should match because both are uppercased before hashing + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Entity1"); + } + + [Fact] + public async Task ExactMatchFieldEqualsWithLowerCaseQueryShouldMatchAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateGuidEntityAsync(store, "Entity1", apiKey: "ABC-DEF"); + + // Query with lowercase + var filter = new ExactMatchField("apiKey").Equals("abc-def"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Entity1"); + } + + [Fact] + public async Task ExactMatchFieldInShouldReturnMatchingEntitiesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateGuidEntityAsync(store, "Entity1", apiKey: "key-alpha"); + _ = await CreateGuidEntityAsync(store, "Entity2", apiKey: "key-beta"); + _ = await CreateGuidEntityAsync(store, "Entity3", apiKey: "key-gamma"); + _ = await CreateGuidEntityAsync(store, "Entity4", apiKey: "key-delta"); + + string[] searchKeys = ["key-alpha", "key-gamma", "key-delta"]; + var filter = new ExactMatchField("apiKey").In(searchKeys); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(3); + result.Items.ShouldContain(x => x.Value.Name == "Entity1"); + result.Items.ShouldContain(x => x.Value.Name == "Entity3"); + result.Items.ShouldContain(x => x.Value.Name == "Entity4"); + } + + [Fact] + public async Task ExactMatchFieldInShouldBeCaseInsensitiveAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateGuidEntityAsync(store, "Entity1", apiKey: "Key-Alpha"); + _ = await CreateGuidEntityAsync(store, "Entity2", apiKey: "KEY-BETA"); + + // Query with mixed casing + string[] searchKeys = ["key-alpha", "key-beta"]; + var filter = new ExactMatchField("apiKey").In(searchKeys); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Entity1"); + result.Items.ShouldContain(x => x.Value.Name == "Entity2"); + } + + [Fact] + public async Task ExactMatchFieldPresentShouldReturnEntitiesWithFieldSetAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateGuidEntityAsync(store, "WithApiKey", apiKey: "some-key"); + _ = await CreateGuidEntityAsync(store, "WithoutApiKey"); + _ = await CreateGuidEntityAsync(store, "AlsoWithApiKey", apiKey: "another-key"); + + var filter = new ExactMatchField("apiKey").Present(); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "WithApiKey"); + result.Items.ShouldContain(x => x.Value.Name == "AlsoWithApiKey"); + } + + [Fact] + public async Task ExactMatchFieldEqualsWithNoMatchShouldReturnEmptyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateGuidEntityAsync(store, "Entity1", apiKey: "existing-key"); + + var filter = new ExactMatchField("apiKey").Equals("nonexistent-key"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(0); + } + + [Fact] + public async Task ExactMatchFieldWithOrShouldMatchEitherAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateGuidEntityAsync(store, "Entity1", apiKey: "key-a"); + _ = await CreateGuidEntityAsync(store, "Entity2", apiKey: "key-b"); + _ = await CreateGuidEntityAsync(store, "Entity3", apiKey: "key-c"); + + var filter = new ExactMatchField("apiKey").Equals("key-a") + .Or(new ExactMatchField("apiKey").Equals("key-c")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Entity1"); + result.Items.ShouldContain(x => x.Value.Name == "Entity3"); + } + + [Fact] + public async Task NotGuidFieldEqualsShouldExcludeMatchAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var excludeGuid = Guid.NewGuid(); + _ = await CreateGuidEntityAsync(store, "Excluded", resourceId: excludeGuid); + _ = await CreateGuidEntityAsync(store, "Included1", resourceId: Guid.NewGuid()); + _ = await CreateGuidEntityAsync(store, "Included2", resourceId: Guid.NewGuid()); + + var filter = Query.Not(new GuidField("resourceId").Equals(excludeGuid)); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Included1"); + result.Items.ShouldContain(x => x.Value.Name == "Included2"); + } + + [Fact] + public async Task NotExactMatchFieldEqualsShouldExcludeMatchAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateGuidEntityAsync(store, "Excluded", apiKey: "banned-key"); + _ = await CreateGuidEntityAsync(store, "Included1", apiKey: "good-key"); + _ = await CreateGuidEntityAsync(store, "Included2", apiKey: "another-key"); + + var filter = Query.Not(new ExactMatchField("apiKey").Equals("banned-key")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Included1"); + result.Items.ShouldContain(x => x.Value.Name == "Included2"); + } + + [Fact] + public async Task GuidFieldAndStringFieldShouldCombineAsync() + { + // Arrange — tests combining GuidField with regular StringField queries + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var targetGuid = Guid.NewGuid(); + _ = await CreateGuidEntityAsync(store, "Match", resourceId: targetGuid, tag: "premium"); + _ = await CreateGuidEntityAsync(store, "WrongTag", resourceId: targetGuid, tag: "standard"); + _ = await CreateGuidEntityAsync(store, "WrongGuid", resourceId: Guid.NewGuid(), tag: "premium"); + + var filter = new GuidField("resourceId").Equals(targetGuid) + .And(new StringField("tag").Equals("premium")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Match"); + } + + [Fact] + public async Task ArrayGuidFieldEqualsShouldMatchWithinArrayItemsAsync() + { + // Arrange — GuidField within array items (e.g., emails with correlation IDs) + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var targetCorrelationId = Guid.NewGuid(); + var otherCorrelationId = Guid.NewGuid(); + + _ = await CreateUserWithEmailsAndGuidsAsync(store, "Alice", + [ + ("work", "alice@work.com", targetCorrelationId), + ("personal", "alice@home.com", otherCorrelationId) + ], _ct); + + _ = await CreateUserWithEmailsAndGuidsAsync(store, "Bob", + [ + ("work", "bob@work.com", Guid.NewGuid()), + ("personal", "bob@home.com", Guid.NewGuid()) + ], _ct); + + var filter = Query.ArrayFilter("emails", + new GuidField("correlationId").Equals(targetCorrelationId)); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_userEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Alice"); + } + + [Fact] + public async Task ArrayGuidFieldInShouldMatchWithinArrayItemsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var correlationId1 = Guid.NewGuid(); + var correlationId2 = Guid.NewGuid(); + var correlationId3 = Guid.NewGuid(); + + _ = await CreateUserWithEmailsAndGuidsAsync(store, "Alice", + [ + ("work", "alice@work.com", correlationId1) + ], _ct); + + _ = await CreateUserWithEmailsAndGuidsAsync(store, "Bob", + [ + ("work", "bob@work.com", correlationId2) + ], _ct); + + _ = await CreateUserWithEmailsAndGuidsAsync(store, "Charlie", + [ + ("work", "charlie@work.com", correlationId3) + ], _ct); + + Guid[] searchGuids = [correlationId1, correlationId3]; + var filter = Query.ArrayFilter("emails", + new GuidField("correlationId").In(searchGuids)); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_userEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Alice"); + result.Items.ShouldContain(x => x.Value.Name == "Charlie"); + } + + [Fact] + public async Task ArrayGuidFieldWithStringFieldAndShouldMatchSameArrayItemAsync() + { + // Arrange — combines GuidField and StringField within the same array item + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var targetCorrelationId = Guid.NewGuid(); + + // Alice: work email has target correlation ID + _ = await CreateUserWithEmailsAndGuidsAsync(store, "Alice", + [ + ("work", "alice@work.com", targetCorrelationId), + ("personal", "alice@home.com", Guid.NewGuid()) + ], _ct); + + // Bob: has target correlation ID but on a personal email, not work + _ = await CreateUserWithEmailsAndGuidsAsync(store, "Bob", + [ + ("work", "bob@work.com", Guid.NewGuid()), + ("personal", "bob@home.com", targetCorrelationId) + ], _ct); + + // Query: array item must have BOTH type="work" AND correlationId=target + var filter = Query.ArrayFilter("emails", + new StringField("type").Equals("work") + .And(new GuidField("correlationId").Equals(targetCorrelationId))); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_userEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert — Only Alice has a work email with the target correlation ID + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Alice"); + } + + [Fact] + public async Task ExactMatchFieldWithDuplicateValuesShouldReturnAllAsync() + { + // Arrange — multiple entities with the same exact-match value + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateGuidEntityAsync(store, "Entity1", apiKey: "shared-key"); + _ = await CreateGuidEntityAsync(store, "Entity2", apiKey: "shared-key"); + _ = await CreateGuidEntityAsync(store, "Entity3", apiKey: "different-key"); + + var filter = new ExactMatchField("apiKey").Equals("shared-key"); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Entity1"); + result.Items.ShouldContain(x => x.Value.Name == "Entity2"); + } + + [Fact] + public async Task GuidFieldWithSameGuidAcrossEntitiesShouldReturnAllAsync() + { + // Arrange — multiple entities sharing the same GUID value + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var sharedGuid = Guid.NewGuid(); + _ = await CreateGuidEntityAsync(store, "Entity1", resourceId: sharedGuid); + _ = await CreateGuidEntityAsync(store, "Entity2", resourceId: sharedGuid); + _ = await CreateGuidEntityAsync(store, "Entity3", resourceId: Guid.NewGuid()); + + var filter = new GuidField("resourceId").Equals(sharedGuid); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_guidEntityType, filter, SortParameter.Empty, page, _ct); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Entity1"); + result.Items.ShouldContain(x => x.Value.Name == "Entity2"); + } + + [Fact] + public async Task GuidFieldCursorPaginationFirstPageShouldReturnNextTokenAsync() + { + // Arrange — create 5 entities with distinct GUIDs, paginate with page size 2 + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 0; i < 5; i++) + { + _ = await CreateGuidEntityAsync(store, $"Item{i}", resourceId: Guid.NewGuid(), ct: _ct); + } + + var filter = new GuidField("resourceId").Present(); + var sort = new SortParameter(new GuidField("resourceId")); + var cursor = DataRange.FromContinuationToken(ContinuationToken.Beginning, 2); + + // Act + var page1 = await store.QueryAsync(_guidEntityType, filter, sort, cursor, _ct); + + // Assert + page1.Items.Count.ShouldBe(2); + _ = page1.NextToken.ShouldNotBeNull(); + page1.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task GuidFieldCursorPaginationContinuationShouldReturnAllItemsAsync() + { + // Arrange — create 5 entities, paginate through all with page size 2 + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 0; i < 5; i++) + { + _ = await CreateGuidEntityAsync(store, $"Item{i}", resourceId: Guid.NewGuid(), ct: _ct); + } + + var filter = new GuidField("resourceId").Present(); + var sort = new SortParameter(new GuidField("resourceId")); + + // Act — page through all results + var allItems = new List>(); + ContinuationToken? token = null; + var pageCount = 0; + + do + { + var cursor = DataRange.FromContinuationToken(token?.Value ?? ContinuationToken.Beginning, 2); + var page = await store.QueryAsync(_guidEntityType, filter, sort, cursor, _ct); + allItems.AddRange(page.Items); + token = page.NextToken; + pageCount++; + + if (!page.HasMoreData || pageCount > 10) + { + break; + } + } + while (true); + + // Assert — all 5 items returned across pages, no duplicates + allItems.Count.ShouldBe(5); + allItems.Select(x => x.Value.Name).Distinct().Count().ShouldBe(5); + pageCount.ShouldBe(3); // 2 + 2 + 1 + } + + [Fact] + public async Task ExactMatchFieldCursorPaginationShouldWorkAsync() + { + // Arrange — create 5 entities with distinct api keys, paginate with ExactMatchField sort + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 0; i < 5; i++) + { + _ = await CreateGuidEntityAsync(store, $"Item{i}", apiKey: $"key-{i:D3}", ct: _ct); + } + + var filter = new ExactMatchField("apiKey").Present(); + var sort = new SortParameter(new ExactMatchField("apiKey")); + + // Act — page through all results + var allItems = new List>(); + ContinuationToken? token = null; + var pageCount = 0; + + do + { + var cursor = DataRange.FromContinuationToken(token?.Value ?? ContinuationToken.Beginning, 2); + var page = await store.QueryAsync(_guidEntityType, filter, sort, cursor, _ct); + allItems.AddRange(page.Items); + token = page.NextToken; + pageCount++; + + if (!page.HasMoreData || pageCount > 10) + { + break; + } + } + while (true); + + // Assert — all 5 items returned across pages, no duplicates + allItems.Count.ShouldBe(5); + allItems.Select(x => x.Value.Name).Distinct().Count().ShouldBe(5); + pageCount.ShouldBe(3); // 2 + 2 + 1 + } +} diff --git a/storage/test/SharedIntegrationTests/QueryStorePagingTests.cs b/storage/test/SharedIntegrationTests/QueryStorePagingTests.cs new file mode 100644 index 000000000..2b34494eb --- /dev/null +++ b/storage/test/SharedIntegrationTests/QueryStorePagingTests.cs @@ -0,0 +1,1182 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Internal.Querying.SearchFields; +using Duende.Storage.Pagination; +using Duende.Storage.Querying; +using SortDirection = Duende.Storage.Querying.SortDirection; +using SortParameter = Duende.Storage.Internal.Querying.Sorting.SortParameter; + +namespace Duende.Storage.IntegrationTests; + +/// +/// Tests for offset-based pagination functionality across all store implementations. +/// Covers first page, continuation, last page, total counts, and edge cases. +/// +public partial class QueryStorePagingTests +{ + + private readonly EntityType _testEntityType = TestPageDso.DsoVersion.EntityType; + + private readonly Ct _ct = TestContext.Current.CancellationToken; + private async Task CreateProviderAsync() => + await FixtureFactory.CreateAsync(_ct, services => + { + services.AddDsoRegistration(); + }); + + private static async Task CreateEntityAsync( + IStore store, + string name, + int rank, + DateTimeOffset? createdAt = null, + bool? isActive = null, + Ct ct = default) + { + var id = UuidV7.New(); + var dso = new TestPageDso + { + Name = name, + Rank = rank, + CreatedAt = createdAt ?? DateTimeOffset.UtcNow, + IsActive = isActive ?? true + }; + + var searchFieldsBuilder = new SearchFieldsBuilder(); + _ = searchFieldsBuilder.Add("name", name); + _ = searchFieldsBuilder.Add("rank", rank); + _ = searchFieldsBuilder.Add("recordedAt", dso.CreatedAt); + _ = searchFieldsBuilder.Add("isActive", dso.IsActive); + + var searchFields = searchFieldsBuilder.Build(); + + var storeInterface = store; + var result = await storeInterface.CreateAsync(id, dso, Array.Empty(), searchFields, Expiration.NoExpiration, [], ct); + result.ShouldBe(CreateResult.Success); + return id; + } + + [Fact] + public async Task QueryFirstPageShouldReturnCorrectItemsAndMetadataAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 10 items + for (var i = 1; i <= 10; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i * 10); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + var page = DataRange.FromPage(1, 3); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(3); + result.Items[0].Value.Name.ShouldBe("Item01"); + result.Items[1].Value.Name.ShouldBe("Item02"); + result.Items[2].Value.Name.ShouldBe("Item03"); + result.TotalCount.ShouldBe(10); + result.TotalPages.ShouldBe(4); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryFieldsFirstPageShouldReturnCorrectItemsAndMetadataAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 8; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i * 5); + } + + var filter = Query.All(); + var sort = new SortParameter(new StringField("name")); + var page = DataRange.FromPage(1, 4); + var fields = new List { new StringField("name"), new NumberField("rank") }; + + // Act + var result = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(4); + result.Items[0].Fields["NAME"].ShouldBe("ITEM01"); + result.Items[3].Fields["NAME"].ShouldBe("ITEM04"); + result.TotalCount.ShouldBe(8); + result.TotalPages.ShouldBe(2); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryMultiplePagesShouldReturnCorrectSubsequentPagesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 10 items + for (var i = 1; i <= 10; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + + // Act - Page 1 + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 3), Ct.None); + + // Act - Page 2 + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 3), Ct.None); + + // Act - Page 3 + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 3), Ct.None); + + // Assert + page1.Items.Count.ShouldBe(3); + page1.Items[0].Value.Rank.ShouldBe(1); + page1.Items[2].Value.Rank.ShouldBe(3); + page1.HasMoreData.ShouldBeTrue(); + page1.TotalPages.ShouldBe(4); + + page2.Items.Count.ShouldBe(3); + page2.Items[0].Value.Rank.ShouldBe(4); + page2.Items[2].Value.Rank.ShouldBe(6); + page2.HasMoreData.ShouldBeTrue(); + page2.TotalPages.ShouldBe(4); + + page3.Items.Count.ShouldBe(3); + page3.Items[0].Value.Rank.ShouldBe(7); + page3.Items[2].Value.Rank.ShouldBe(9); + page3.HasMoreData.ShouldBeTrue(); + page3.TotalPages.ShouldBe(4); + } + + [Fact] + public async Task QueryFieldsMultiplePagesShouldReturnCorrectSubsequentPagesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 7; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i * 10); + } + + var filter = Query.All(); + var sort = new SortParameter(new StringField("name"), SortDirection.Descending); + var fields = new List { new StringField("name"), new NumberField("rank") }; + + // Act - Page 1 + var page1 = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, DataRange.FromPage(1, 3), Ct.None); + + // Act - Page 2 + var page2 = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, DataRange.FromPage(2, 3), Ct.None); + + // Assert + page1.Items.Count.ShouldBe(3); + page1.Items[0].Fields["NAME"].ShouldBe("ITEM07"); + page1.Items[2].Fields["NAME"].ShouldBe("ITEM05"); + page1.HasMoreData.ShouldBeTrue(); + page1.TotalPages.ShouldBe(3); + + page2.Items.Count.ShouldBe(3); + page2.Items[0].Fields["NAME"].ShouldBe("ITEM04"); + page2.Items[2].Fields["NAME"].ShouldBe("ITEM02"); + page2.HasMoreData.ShouldBeTrue(); + page2.TotalPages.ShouldBe(3); + } + + [Fact] + public async Task QueryLastPageShouldHaveCorrectMetadataAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 7 items (3+3+1) + for (var i = 1; i <= 7; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i * 5); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + + // Act - Navigate to last page + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 3), Ct.None); + + // Assert + page3.Items.Count.ShouldBe(1); + page3.Items[0].Value.Rank.ShouldBe(35); + page3.TotalCount.ShouldBe(7); + page3.TotalPages.ShouldBe(3); + page3.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QueryFieldsLastPageShouldHaveCorrectMetadataAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create exactly 5 items with page size 5 + for (var i = 1; i <= 5; i++) + { + _ = await CreateEntityAsync(store, $"Item{i}", i); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + var page = DataRange.FromPage(1, 5); + var fields = new List { new StringField("name") }; + + // Act + var result = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, page, Ct.None); + + // Assert - Exactly one full page, no more pages + result.Items.Count.ShouldBe(5); + result.TotalCount.ShouldBe(5); + result.TotalPages.ShouldBe(1); + result.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QueryEmptyResultShouldReturnEmptyPageWithCorrectMetadataAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Item1", 10); + _ = await CreateEntityAsync(store, "Item2", 20); + + var filter = new NumberField("rank").GreaterThan(100); + var sort = new SortParameter(new NumberField("rank")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(0); + result.TotalCount.ShouldBe(0); + result.TotalPages.ShouldBe(0); + result.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QueryAllPagesShouldHaveConsistentTotalCountAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 15 items + for (var i = 1; i <= 15; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + + // Act - Request multiple pages + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 5), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 5), Ct.None); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 5), Ct.None); + + // Assert - Total count and total pages should be consistent across all pages + page1.TotalCount.ShouldBe(15); + page2.TotalCount.ShouldBe(15); + page3.TotalCount.ShouldBe(15); + + page1.TotalPages.ShouldBe(3); + page2.TotalPages.ShouldBe(3); + page3.TotalPages.ShouldBe(3); + } + + [Fact] + public async Task QueryWithFilterShouldReturnFilteredTotalCountAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 20 items + for (var i = 1; i <= 20; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i); + } + + var filter = new NumberField("rank").GreaterThan(10); + var sort = new SortParameter(new NumberField("rank")); + var page = DataRange.FromPage(1, 5); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert - Should only count items matching the filter (11-20 = 10 items) + result.Items.Count.ShouldBe(5); + result.TotalCount.ShouldBe(10); + result.TotalPages.ShouldBe(2); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryWithoutSortShouldPageByIdAscendingAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create items in random order + for (var i = 5; i >= 1; i--) + { + var id = await CreateEntityAsync(store, $"Item{i}", i * 10); + } + + var filter = Query.All(); + var page = DataRange.FromPage(1, 3); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert - Should be ordered by ID when no sort is specified + result.Items.Count.ShouldBe(3); + result.TotalCount.ShouldBe(5); + result.TotalPages.ShouldBe(2); + } + + [Fact] + public async Task QueryDescendingSortShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 9; i++) + { + _ = await CreateEntityAsync(store, $"Item{i}", i * 10); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank"), SortDirection.Descending); + var page = DataRange.FromPage(1, 4); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(4); + result.Items[0].Value.Rank.ShouldBe(90); // Highest first + result.Items[3].Value.Rank.ShouldBe(60); + result.TotalCount.ShouldBe(9); + result.TotalPages.ShouldBe(3); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryPageBeyondRangeShouldReturnEmptyPageAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 5; i++) + { + _ = await CreateEntityAsync(store, $"Item{i}", i); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + var page = DataRange.FromPage(10, 5); // Page 10 when only 1 page exists + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(0); + result.TotalCount.ShouldBe(5); + result.TotalPages.ShouldBe(1); + result.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QueryWithDuplicateSortValuesShouldMaintainStableOrderingAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create items with duplicate rank values + var id1 = await CreateEntityAsync(store, "Alice", 100); + var id2 = await CreateEntityAsync(store, "Bob", 100); + var id3 = await CreateEntityAsync(store, "Charlie", 100); + var id4 = await CreateEntityAsync(store, "David", 50); + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank"), SortDirection.Descending); + + // Act - Page 1 + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 2), Ct.None); + + // Act - Page 2 + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 2), Ct.None); + + // Assert - All items with rank 100 should come before rank 50 + page1.Items.Count.ShouldBe(2); + page1.Items[0].Value.Rank.ShouldBe(100); + page1.Items[1].Value.Rank.ShouldBe(100); + page1.TotalCount.ShouldBe(4); + page1.TotalPages.ShouldBe(2); + page2.Items.Count.ShouldBe(2); + page2.Items[0].Value.Rank.ShouldBe(100); + page2.Items[1].Value.Rank.ShouldBe(50); + page2.Items[1].Value.Name.ShouldBe("David"); + } + + [Fact] + public async Task QueryWithFilterShouldPageFilteredResultsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 20 items + for (var i = 1; i <= 20; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i); + } + + var filter = new NumberField("rank").GreaterThan(10); + var sort = new SortParameter(new NumberField("rank")); + + // Act + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 5), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 5), Ct.None); + + // Assert + page1.Items.Count.ShouldBe(5); + page1.Items[0].Value.Rank.ShouldBe(11); + page1.Items[4].Value.Rank.ShouldBe(15); + page1.TotalCount.ShouldBe(10); + page1.TotalPages.ShouldBe(2); + + page2.Items.Count.ShouldBe(5); + page2.Items[0].Value.Rank.ShouldBe(16); + page2.Items[4].Value.Rank.ShouldBe(20); + page2.TotalCount.ShouldBe(10); + page2.TotalPages.ShouldBe(2); + } + + [Fact] + public async Task QuerySingleItemShouldReturnOneItemPage() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "OnlyOne", 42); + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("OnlyOne"); + result.TotalCount.ShouldBe(1); + result.TotalPages.ShouldBe(1); + result.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QueryLargePageSizeShouldReturnAllItemsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 5; i++) + { + _ = await CreateEntityAsync(store, $"Item{i}", i); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + var page = DataRange.FromPage(1, 100); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(5); + result.TotalCount.ShouldBe(5); + result.TotalPages.ShouldBe(1); + result.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QueryFieldsWithComplexFilterShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 12; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i * 5); + } + + var filter = new NumberField("rank").Between(20, 50); + var sort = new SortParameter(new NumberField("rank")); + var fields = new List { new StringField("name"), new NumberField("rank") }; + + // Act + var page1 = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, DataRange.FromPage(1, 3), Ct.None); + var page2 = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, DataRange.FromPage(2, 3), Ct.None); + + // Assert - ranks 20, 25, 30, 35, 40, 45, 50 = 7 items + page1.Items.Count.ShouldBe(3); + page1.Items[0].Fields["RANK"].ShouldBe(20m); + page1.TotalCount.ShouldBe(7); + page1.TotalPages.ShouldBe(3); + page2.Items.Count.ShouldBe(3); + page2.Items[0].Fields["RANK"].ShouldBe(35m); + } + + [Fact] + public async Task QuerySortByDateTimeAscendingShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var baseDate = new DateTimeOffset(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + for (var i = 1; i <= 10; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i * 10, baseDate.AddDays(i), true); + } + + var filter = Query.All(); + var sort = new SortParameter(new DateTimeField("recordedAt")); + var page = DataRange.FromPage(1, 3); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(3); + result.Items[0].Value.CreatedAt.ShouldBe(baseDate.AddDays(1)); + result.Items[1].Value.CreatedAt.ShouldBe(baseDate.AddDays(2)); + result.Items[2].Value.CreatedAt.ShouldBe(baseDate.AddDays(3)); + result.TotalCount.ShouldBe(10); + result.TotalPages.ShouldBe(4); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QuerySortByDateTimeDescendingShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var baseDate = new DateTimeOffset(2024, 6, 15, 12, 0, 0, TimeSpan.Zero); + for (var i = 1; i <= 12; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i, baseDate.AddHours(i), true); + } + + var filter = Query.All(); + var sort = new SortParameter(new DateTimeField("recordedAt"), SortDirection.Descending); + var page = DataRange.FromPage(2, 4); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert - Should return most recent dates first, page 2 + result.Items.Count.ShouldBe(4); + result.Items[0].Value.CreatedAt.ShouldBe(baseDate.AddHours(8)); + result.Items[1].Value.CreatedAt.ShouldBe(baseDate.AddHours(7)); + result.Items[2].Value.CreatedAt.ShouldBe(baseDate.AddHours(6)); + result.Items[3].Value.CreatedAt.ShouldBe(baseDate.AddHours(5)); + result.TotalCount.ShouldBe(12); + result.TotalPages.ShouldBe(3); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryFieldsSortByDateTimeAscendingShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var baseDate = new DateTimeOffset(2024, 3, 10, 0, 0, 0, TimeSpan.Zero); + for (var i = 1; i <= 6; i++) + { + _ = await CreateEntityAsync(store, $"Item{i}", i * 5, baseDate.AddMonths(i), true); + } + + var filter = Query.All(); + var sort = new SortParameter(new DateTimeField("recordedAt")); + var page = DataRange.FromPage(1, 3); + var fields = new List { new StringField("name"), new DateTimeField("recordedAt") }; + + // Act + var result = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(3); + ((DateTimeOffset)result.Items[0].Fields["RECORDEDAT"]!).ShouldBe(baseDate.AddMonths(1)); + ((DateTimeOffset)result.Items[1].Fields["RECORDEDAT"]!).ShouldBe(baseDate.AddMonths(2)); + ((DateTimeOffset)result.Items[2].Fields["RECORDEDAT"]!).ShouldBe(baseDate.AddMonths(3)); + result.TotalCount.ShouldBe(6); + result.TotalPages.ShouldBe(2); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QuerySortByDateTimeWithMultiplePagesAndFilterShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var baseDate = new DateTimeOffset(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + for (var i = 1; i <= 15; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i, baseDate.AddDays(i), true); + } + + var cutoffDate = baseDate.AddDays(5); + var filter = new DateTimeField("recordedAt").GreaterThan(cutoffDate.UtcDateTime); + var sort = new SortParameter(new DateTimeField("recordedAt")); + + // Act + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 5), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 5), Ct.None); + + // Assert - Should only include items after cutoff (days 6-15 = 10 items) + page1.Items.Count.ShouldBe(5); + page1.Items[0].Value.CreatedAt.ShouldBe(baseDate.AddDays(6)); + page1.TotalCount.ShouldBe(10); + page1.TotalPages.ShouldBe(2); + page1.HasMoreData.ShouldBeTrue(); + + page2.Items.Count.ShouldBe(5); + page2.Items[0].Value.CreatedAt.ShouldBe(baseDate.AddDays(11)); + page2.TotalCount.ShouldBe(10); + page2.TotalPages.ShouldBe(2); + page2.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QuerySortByBooleanAscendingShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var baseDate = DateTimeOffset.UtcNow; + // Create items with mixed boolean values + _ = await CreateEntityAsync(store, "Item1", 10, baseDate.AddDays(1), false); + _ = await CreateEntityAsync(store, "Item2", 20, baseDate.AddDays(2), false); + _ = await CreateEntityAsync(store, "Item3", 30, baseDate.AddDays(3), false); + _ = await CreateEntityAsync(store, "Item4", 40, baseDate.AddDays(4), true); + _ = await CreateEntityAsync(store, "Item5", 50, baseDate.AddDays(5), true); + _ = await CreateEntityAsync(store, "Item6", 60, baseDate.AddDays(6), true); + _ = await CreateEntityAsync(store, "Item7", 70, baseDate.AddDays(7), true); + + var filter = Query.All(); + var sort = new SortParameter(new BooleanField("isActive")); + var page = DataRange.FromPage(1, 4); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert - false values should come before true (ascending) + result.Items.Count.ShouldBe(4); + result.Items[0].Value.IsActive.ShouldBeFalse(); + result.Items[1].Value.IsActive.ShouldBeFalse(); + result.Items[2].Value.IsActive.ShouldBeFalse(); + result.Items[3].Value.IsActive.ShouldBeTrue(); + result.TotalCount.ShouldBe(7); + result.TotalPages.ShouldBe(2); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QuerySortByBooleanDescendingShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var baseDate = DateTimeOffset.UtcNow; + // Create 10 items with mixed boolean values + for (var i = 1; i <= 10; i++) + { + var isActive = i % 3 == 0; // Every third item is active + _ = await CreateEntityAsync(store, $"Item{i:D2}", i * 10, baseDate.AddDays(i), isActive); + } + + var filter = Query.All(); + var sort = new SortParameter(new BooleanField("isActive"), SortDirection.Descending); + var page = DataRange.FromPage(1, 4); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert - true values should come before false (descending) + result.Items.Count.ShouldBe(4); + result.Items[0].Value.IsActive.ShouldBeTrue(); + result.Items[1].Value.IsActive.ShouldBeTrue(); + result.Items[2].Value.IsActive.ShouldBeTrue(); + result.TotalCount.ShouldBe(10); + result.TotalPages.ShouldBe(3); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryFieldsSortByBooleanAscendingShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var baseDate = DateTimeOffset.UtcNow; + _ = await CreateEntityAsync(store, "Inactive1", 10, baseDate, false); + _ = await CreateEntityAsync(store, "Inactive2", 20, baseDate.AddDays(1), false); + _ = await CreateEntityAsync(store, "Active1", 30, baseDate.AddDays(2), true); + _ = await CreateEntityAsync(store, "Active2", 40, baseDate.AddDays(3), true); + _ = await CreateEntityAsync(store, "Active3", 50, baseDate.AddDays(4), true); + + var filter = Query.All(); + var sort = new SortParameter(new BooleanField("isActive")); + var page = DataRange.FromPage(1, 3); + var fields = new List { new StringField("name"), new BooleanField("isActive") }; + + // Act + var result = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(3); + ((bool)result.Items[0].Fields["ISACTIVE"]!).ShouldBeFalse(); + ((bool)result.Items[1].Fields["ISACTIVE"]!).ShouldBeFalse(); + ((bool)result.Items[2].Fields["ISACTIVE"]!).ShouldBeTrue(); + result.TotalCount.ShouldBe(5); + result.TotalPages.ShouldBe(2); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QuerySortByBooleanWithFilterShouldPageCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var baseDate = DateTimeOffset.UtcNow; + // Create items with various rank values and boolean states + for (var i = 1; i <= 12; i++) + { + var isActive = i <= 6; // First half is active + _ = await CreateEntityAsync(store, $"Item{i:D2}", i * 10, baseDate.AddDays(i), isActive); + } + + var filter = new NumberField("rank").GreaterThan(30); // Items 4-12 + var sort = new SortParameter(new BooleanField("isActive"), SortDirection.Descending); + var page = DataRange.FromPage(1, 5); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert - Should include filtered items (rank > 30) sorted by isActive desc + result.Items.Count.ShouldBe(5); + // First results should be true (items 4-6 are active and rank > 30) + result.Items[0].Value.IsActive.ShouldBeTrue(); + result.Items[1].Value.IsActive.ShouldBeTrue(); + result.Items[2].Value.IsActive.ShouldBeTrue(); + result.TotalCount.ShouldBe(9); // Items 4-12 = 9 items + result.TotalPages.ShouldBe(2); + result.HasMoreData.ShouldBeTrue(); + } + + [Fact] + public async Task QueryWithoutSortShouldPageThroughAllRecordsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 15 items with unique rank values + var expectedRanks = new HashSet(); + for (var i = 1; i <= 15; i++) + { + var rank = i * 10; + _ = await CreateEntityAsync(store, $"Item{i:D2}", rank); + _ = expectedRanks.Add(rank); + } + + var filter = Query.All(); + var pageSize = 4; + + // Act - Page through all results + var retrievedRanks = new HashSet(); + var pageNumber = 1; + QueryResult> result; + + do + { + result = await store.QueryAsync( + _testEntityType, + filter, + SortParameter.Empty, + DataRange.FromPage(pageNumber, pageSize), + Ct.None); + + foreach (var item in result.Items) + { + // Track by unique rank values + var wasAdded = retrievedRanks.Add(item.Value.Rank); + wasAdded.ShouldBeTrue($"Duplicate rank {item.Value.Rank} found - item returned multiple times"); + } + + pageNumber++; + } + while (result.HasMoreData); + + // Assert - Should have retrieved all items exactly once + result.TotalCount.ShouldBe(15); + retrievedRanks.Count.ShouldBe(15); + retrievedRanks.ShouldBe(expectedRanks, ignoreOrder: true); + } + + [Fact] + public async Task QueryWithoutSortMultiplePagesShouldHaveConsistentTotalCountAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 25 items + for (var i = 1; i <= 25; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", i); + } + + var filter = Query.All(); + var pageSize = 7; + + // Act - Fetch multiple pages without sort + var page1 = await store.QueryAsync( + _testEntityType, + filter, + SortParameter.Empty, + DataRange.FromPage(1, pageSize), + Ct.None); + + var page2 = await store.QueryAsync( + _testEntityType, + filter, + SortParameter.Empty, + DataRange.FromPage(2, pageSize), + Ct.None); + + var page3 = await store.QueryAsync( + _testEntityType, + filter, + SortParameter.Empty, + DataRange.FromPage(3, pageSize), + Ct.None); + + var page4 = await store.QueryAsync( + _testEntityType, + filter, + SortParameter.Empty, + DataRange.FromPage(4, pageSize), + Ct.None); + + // Assert - All pages should report consistent total count and total pages + page1.TotalCount.ShouldBe(25); + page2.TotalCount.ShouldBe(25); + page3.TotalCount.ShouldBe(25); + page4.TotalCount.ShouldBe(25); + + page1.TotalPages.ShouldBe(4); + page2.TotalPages.ShouldBe(4); + page3.TotalPages.ShouldBe(4); + page4.TotalPages.ShouldBe(4); + + // Check page sizes + page1.Items.Count.ShouldBe(7); + page2.Items.Count.ShouldBe(7); + page3.Items.Count.ShouldBe(7); + page4.Items.Count.ShouldBe(4); // Last page has remainder + + // Verify pagination flags + page1.HasMoreData.ShouldBeTrue(); + + page2.HasMoreData.ShouldBeTrue(); + + page3.HasMoreData.ShouldBeTrue(); + + page4.HasMoreData.ShouldBeFalse(); + } + + [Fact] + public async Task QueryWithoutSortShouldRetrieveAllUniqueRecordsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create items with unique rank values that we can track + var expectedRanks = new HashSet(); + for (var i = 1; i <= 20; i++) + { + var rank = i * 7; // Use a multiplier to ensure uniqueness + _ = await CreateEntityAsync(store, $"Item{i:D2}", rank); + _ = expectedRanks.Add(rank); + } + + var filter = Query.All(); + var pageSize = 6; + + // Act - Page through all results and collect ranks + var retrievedRanks = new HashSet(); + var pageNumber = 1; + QueryResult> result; + + do + { + result = await store.QueryAsync( + _testEntityType, + filter, + SortParameter.Empty, + DataRange.FromPage(pageNumber, pageSize), + Ct.None); + + foreach (var item in result.Items) + { + // Track by rank to ensure no duplicates + var wasAdded = retrievedRanks.Add(item.Value.Rank); + wasAdded.ShouldBeTrue($"Duplicate rank {item.Value.Rank} found - item returned multiple times"); + } + + pageNumber++; + } + while (result.HasMoreData); + + // Assert - Should have all unique ranks (order doesn't matter without sort) + retrievedRanks.Count.ShouldBe(20); + retrievedRanks.ShouldBe(expectedRanks, ignoreOrder: true); + } + + [Fact] + public async Task QueryFieldsWithoutSortShouldPageThroughAllRecordsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 18 items + var expectedRanks = new HashSet(); + for (var i = 1; i <= 18; i++) + { + var rank = i * 5; + _ = await CreateEntityAsync(store, $"Item{i:D2}", rank); + _ = expectedRanks.Add(rank); + } + + var filter = Query.All(); + var pageSize = 5; + var fields = new List { new StringField("name"), new NumberField("rank") }; + + // Act - Page through all results + var retrievedRanks = new HashSet(); + var pageNumber = 1; + QueryResult result; + + do + { + result = await store.QueryFieldsAsync( + _testEntityType, + fields, + filter, + SortParameter.Empty, + DataRange.FromPage(pageNumber, pageSize), + Ct.None); + + foreach (var item in result.Items) + { + var rank = (decimal)item.Fields["RANK"]!; + var wasAdded = retrievedRanks.Add(rank); + wasAdded.ShouldBeTrue($"Duplicate rank {rank} found - item returned multiple times"); + } + + pageNumber++; + } + while (result.HasMoreData); + + // Assert + result.TotalCount.ShouldBe(18); + retrievedRanks.Count.ShouldBe(18); + retrievedRanks.ShouldBe(expectedRanks.Select(r => (decimal)r).ToHashSet(), ignoreOrder: true); + } + + [Fact] + public async Task QueryWithoutSortWithFilterShouldPageThroughFilteredRecordsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 30 items + var expectedRanksInRange = new HashSet(); + for (var i = 1; i <= 30; i++) + { + var rank = i * 10; + _ = await CreateEntityAsync(store, $"Item{i:D2}", rank); + if (rank >= 100 && rank <= 200) + { + _ = expectedRanksInRange.Add(rank); + } + } + + var filter = new NumberField("rank").Between(100, 200); + var pageSize = 4; + + // Act - Page through filtered results + var retrievedRanks = new HashSet(); + var pageNumber = 1; + QueryResult> result; + + do + { + result = await store.QueryAsync( + _testEntityType, + filter, + SortParameter.Empty, + DataRange.FromPage(pageNumber, pageSize), + Ct.None); + + foreach (var item in result.Items) + { + var wasAdded = retrievedRanks.Add(item.Value.Rank); + wasAdded.ShouldBeTrue($"Duplicate rank {item.Value.Rank} found"); + } + + pageNumber++; + } + while (result.HasMoreData); + + // Assert - Should have all filtered items + result.TotalCount.ShouldBe(11); // Ranks 100, 110, 120, ..., 200 = 11 items + retrievedRanks.Count.ShouldBe(11); + retrievedRanks.ShouldBe(expectedRanksInRange, ignoreOrder: true); + } + + [Fact] + public async Task QueryFields_field_path_text_should_be_human_readable_not_a_GUID_async() + { + // Arrange — create entities with multiple distinct field paths + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Alice", rank: 42); + _ = await CreateEntityAsync(store, "Bob", rank: 99); + + // Request both "name" and "rank" fields via QueryFieldsAsync + var fields = new List { new StringField("name"), new NumberField("rank") }; + var filter = Query.All(); + var sort = SortParameter.Empty; + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, page, _ct); + + // Assert — the Fields dictionary must be keyed by the human-readable (uppercased) path, + // NOT by a GUID string. If field_path_text stored a GUID, the keys would look like + // "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" and the lookups below would fail. + result.Items.Count.ShouldBe(2); + + foreach (var item in result.Items) + { + // Keys must be the uppercased field path strings, not GUIDs + item.Fields.Keys.ShouldContain("NAME", + "Expected field key 'NAME' but got GUID-like key — field_path_text is storing a GUID instead of the human-readable path."); + item.Fields.Keys.ShouldContain("RANK", + "Expected field key 'RANK' but got GUID-like key — field_path_text is storing a GUID instead of the human-readable path."); + + // Verify none of the keys look like a GUID (36-char format with dashes) + foreach (var key in item.Fields.Keys) + { + Guid.TryParse(key, out _).ShouldBeFalse( + $"Field key '{key}' looks like a GUID — field_path_text is storing a GUID instead of the human-readable path."); + } + } + } + + [Fact] + public async Task QueryFields_multiple_field_paths_should_all_return_human_readable_keys_async() + { + // Arrange — create entities with several different field paths to ensure + // each field_path_text entry is stored correctly + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "TestUser", rank: 75); + + var fields = new List + { + new StringField("name"), + new NumberField("rank") + }; + var filter = Query.All(); + var sort = SortParameter.Empty; + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryFieldsAsync(_testEntityType, fields, filter, sort, page, _ct); + + // Assert — all field paths must appear as human-readable keys + result.Items.Count.ShouldBe(1); + var item = result.Items[0]; + + item.Fields.Keys.ShouldContain("NAME"); + item.Fields.Keys.ShouldContain("RANK"); + + // Verify values are populated (not null) — proves the key lookup succeeded + _ = item.Fields["NAME"].ShouldNotBeNull(); + _ = item.Fields["RANK"].ShouldNotBeNull(); + + // Verify no key is a GUID + foreach (var key in item.Fields.Keys) + { + Guid.TryParse(key, out _).ShouldBeFalse( + $"Field key '{key}' is a GUID — field_path_text must store the human-readable path."); + } + } +} diff --git a/storage/test/SharedIntegrationTests/QueryStoreSortTests.cs b/storage/test/SharedIntegrationTests/QueryStoreSortTests.cs new file mode 100644 index 000000000..d676f17fc --- /dev/null +++ b/storage/test/SharedIntegrationTests/QueryStoreSortTests.cs @@ -0,0 +1,884 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Internal.Querying.SearchFields; +using Duende.Storage.Pagination; +using SortDirection = Duende.Storage.Querying.SortDirection; +using SortParameter = Duende.Storage.Internal.Querying.Sorting.SortParameter; + +namespace Duende.Storage.IntegrationTests; + +/// +/// Tests for sorting functionality across all store implementations. +/// Covers sorting by different field types (string, number, datetime) in both directions, +/// and verifies correct behavior with filtering and pagination. +/// +public partial class QueryStoreSortTests +{ + + private readonly EntityType _testEntityType = new(4, "SortTestEntity"); + + private readonly Ct _ct = TestContext.Current.CancellationToken; + + private async Task CreateProviderAsync() => + await FixtureFactory.CreateAsync(_ct, services => + { + services.AddDsoRegistration(); + services.AddDsoRegistration(); + services.AddDsoRegistration(); + }); + + private static async Task CreateEntityAsync( + IStore store, + string name, + int? rank = null, + decimal? rating = null, + DateTimeOffset? timestamp = null, + string? category = null, + Ct ct = default) + { + var id = UuidV7.New(); + var dso = new TestSortDso + { + Name = name, + Rank = rank, + Rating = rating, + Timestamp = timestamp, + Category = category + }; + + var searchFieldsBuilder = new SearchFieldsBuilder(); + _ = searchFieldsBuilder.Add("name", name); + if (rank.HasValue) + { + _ = searchFieldsBuilder.Add("rank", rank.Value); + } + + if (rating.HasValue) + { + _ = searchFieldsBuilder.Add("rating", rating.Value); + } + + if (timestamp.HasValue) + { + _ = searchFieldsBuilder.Add("timestamp", timestamp.Value); + } + + if (category != null) + { + _ = searchFieldsBuilder.Add("category", category); + } + + var searchFields = searchFieldsBuilder.Build(); + + var storeInterface = store; + var result = await storeInterface.CreateAsync(id, dso, Array.Empty(), searchFields, Expiration.NoExpiration, [], ct); + result.ShouldBe(CreateResult.Success); + return id; + } + + [Fact] + public async Task QuerySortByStringAscendingShouldReturnAlphabeticalOrderAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Zebra"); + _ = await CreateEntityAsync(store, "Apple"); + _ = await CreateEntityAsync(store, "Mango"); + _ = await CreateEntityAsync(store, "Banana"); + + var filter = Query.All(); + var sort = new SortParameter(new StringField("name")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(4); + result.Items[0].Value.Name.ShouldBe("Apple"); + result.Items[1].Value.Name.ShouldBe("Banana"); + result.Items[2].Value.Name.ShouldBe("Mango"); + result.Items[3].Value.Name.ShouldBe("Zebra"); + } + + [Fact] + public async Task QuerySortByStringDescendingShouldReturnReverseAlphabeticalOrderAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Alpha"); + _ = await CreateEntityAsync(store, "Bravo"); + _ = await CreateEntityAsync(store, "Charlie"); + _ = await CreateEntityAsync(store, "Delta"); + + var filter = Query.All(); + var sort = new SortParameter(new StringField("name"), SortDirection.Descending); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(4); + result.Items[0].Value.Name.ShouldBe("Delta"); + result.Items[1].Value.Name.ShouldBe("Charlie"); + result.Items[2].Value.Name.ShouldBe("Bravo"); + result.Items[3].Value.Name.ShouldBe("Alpha"); + } + + [Fact] + public async Task QuerySortByStringWithNumbersShouldSortLexicographicallyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Item10"); + _ = await CreateEntityAsync(store, "Item2"); + _ = await CreateEntityAsync(store, "Item1"); + _ = await CreateEntityAsync(store, "Item20"); + + var filter = Query.All(); + var sort = new SortParameter(new StringField("name")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert - Lexicographic order, not numeric + result.Items.Count.ShouldBe(4); + result.Items[0].Value.Name.ShouldBe("Item1"); + result.Items[1].Value.Name.ShouldBe("Item10"); + result.Items[2].Value.Name.ShouldBe("Item2"); + result.Items[3].Value.Name.ShouldBe("Item20"); + } + + [Fact] + public async Task QuerySortByNumberAscendingShouldReturnSmallestToLargestAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Item1", rank: 42); + _ = await CreateEntityAsync(store, "Item2", rank: 7); + _ = await CreateEntityAsync(store, "Item3", rank: 99); + _ = await CreateEntityAsync(store, "Item4", rank: 23); + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(4); + result.Items[0].Value.Rank.ShouldBe(7); + result.Items[1].Value.Rank.ShouldBe(23); + result.Items[2].Value.Rank.ShouldBe(42); + result.Items[3].Value.Rank.ShouldBe(99); + } + + [Fact] + public async Task QuerySortByNumberDescendingShouldReturnLargestToSmallestAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Item1", rank: 10); + _ = await CreateEntityAsync(store, "Item2", rank: 50); + _ = await CreateEntityAsync(store, "Item3", rank: 30); + _ = await CreateEntityAsync(store, "Item4", rank: 40); + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank"), SortDirection.Descending); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(4); + result.Items[0].Value.Rank.ShouldBe(50); + result.Items[1].Value.Rank.ShouldBe(40); + result.Items[2].Value.Rank.ShouldBe(30); + result.Items[3].Value.Rank.ShouldBe(10); + } + + [Fact] + public async Task QuerySortByDecimalAscendingShouldHandlePrecisionAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Product1", rating: 4.5m); + _ = await CreateEntityAsync(store, "Product2", rating: 4.25m); + _ = await CreateEntityAsync(store, "Product3", rating: 4.75m); + _ = await CreateEntityAsync(store, "Product4", rating: 4.1m); + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rating")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(4); + result.Items[0].Value.Rating.ShouldBe(4.1m); + result.Items[1].Value.Rating.ShouldBe(4.25m); + result.Items[2].Value.Rating.ShouldBe(4.5m); + result.Items[3].Value.Rating.ShouldBe(4.75m); + } + + [Fact] + public async Task QuerySortByNumberWithNegativeValuesShouldSortCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Item1", rank: -10); + _ = await CreateEntityAsync(store, "Item2", rank: 5); + _ = await CreateEntityAsync(store, "Item3", rank: -25); + _ = await CreateEntityAsync(store, "Item4", rank: 0); + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(4); + result.Items[0].Value.Rank.ShouldBe(-25); + result.Items[1].Value.Rank.ShouldBe(-10); + result.Items[2].Value.Rank.ShouldBe(0); + result.Items[3].Value.Rank.ShouldBe(5); + } + + [Fact] + public async Task QuerySortByDateTimeAscendingShouldReturnOldestToNewestAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var date1 = new DateTimeOffset(2024, 6, 1, 0, 0, 0, TimeSpan.Zero); + var date2 = new DateTimeOffset(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + var date3 = new DateTimeOffset(2024, 12, 1, 0, 0, 0, TimeSpan.Zero); + var date4 = new DateTimeOffset(2024, 3, 1, 0, 0, 0, TimeSpan.Zero); + + _ = await CreateEntityAsync(store, "Event1", timestamp: date1); + _ = await CreateEntityAsync(store, "Event2", timestamp: date2); + _ = await CreateEntityAsync(store, "Event3", timestamp: date3); + _ = await CreateEntityAsync(store, "Event4", timestamp: date4); + + var filter = Query.All(); + var sort = new SortParameter(new DateTimeField("timestamp")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(4); + result.Items[0].Value.Name.ShouldBe("Event2"); // Jan 1 + result.Items[1].Value.Name.ShouldBe("Event4"); // Mar 1 + result.Items[2].Value.Name.ShouldBe("Event1"); // Jun 1 + result.Items[3].Value.Name.ShouldBe("Event3"); // Dec 1 + } + + [Fact] + public async Task QuerySortByDateTimeDescendingShouldReturnNewestToOldestAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var date1 = new DateTimeOffset(2023, 1, 1, 0, 0, 0, TimeSpan.Zero); + var date2 = new DateTimeOffset(2024, 1, 1, 0, 0, 0, TimeSpan.Zero); + var date3 = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero); + var date4 = new DateTimeOffset(2022, 1, 1, 0, 0, 0, TimeSpan.Zero); + + _ = await CreateEntityAsync(store, "Event2023", timestamp: date1); + _ = await CreateEntityAsync(store, "Event2024", timestamp: date2); + _ = await CreateEntityAsync(store, "Event2025", timestamp: date3); + _ = await CreateEntityAsync(store, "Event2022", timestamp: date4); + + var filter = Query.All(); + var sort = new SortParameter(new DateTimeField("timestamp"), SortDirection.Descending); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(4); + result.Items[0].Value.Name.ShouldBe("Event2025"); + result.Items[1].Value.Name.ShouldBe("Event2024"); + result.Items[2].Value.Name.ShouldBe("Event2023"); + result.Items[3].Value.Name.ShouldBe("Event2022"); + } + + [Fact] + public async Task QuerySortByDateTimeWithSameDateShouldSortByTimeAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var time1 = new DateTimeOffset(2024, 6, 1, 14, 0, 0, TimeSpan.Zero); + var time2 = new DateTimeOffset(2024, 6, 1, 9, 0, 0, TimeSpan.Zero); + var time3 = new DateTimeOffset(2024, 6, 1, 18, 0, 0, TimeSpan.Zero); + var time4 = new DateTimeOffset(2024, 6, 1, 6, 0, 0, TimeSpan.Zero); + + _ = await CreateEntityAsync(store, "Event14:00", timestamp: time1); + _ = await CreateEntityAsync(store, "Event09:00", timestamp: time2); + _ = await CreateEntityAsync(store, "Event18:00", timestamp: time3); + _ = await CreateEntityAsync(store, "Event06:00", timestamp: time4); + + var filter = Query.All(); + var sort = new SortParameter(new DateTimeField("timestamp")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(4); + result.Items[0].Value.Name.ShouldBe("Event06:00"); + result.Items[1].Value.Name.ShouldBe("Event09:00"); + result.Items[2].Value.Name.ShouldBe("Event14:00"); + result.Items[3].Value.Name.ShouldBe("Event18:00"); + } + + [Fact] + public async Task QuerySortWithFilterShouldApplyBothCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Alice", rank: 80, category: "premium"); + _ = await CreateEntityAsync(store, "Bob", rank: 60, category: "basic"); + _ = await CreateEntityAsync(store, "Charlie", rank: 95, category: "premium"); + _ = await CreateEntityAsync(store, "David", rank: 70, category: "basic"); + _ = await CreateEntityAsync(store, "Eve", rank: 85, category: "premium"); + + var filter = new StringField("category").Equals("premium"); + var sort = new SortParameter(new NumberField("rank"), SortDirection.Descending); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(3); + result.Items[0].Value.Name.ShouldBe("Charlie"); // 95 + result.Items[1].Value.Name.ShouldBe("Eve"); // 85 + result.Items[2].Value.Name.ShouldBe("Alice"); // 80 + } + + [Fact] + public async Task QuerySortByDateTimeWithRangeFilterShouldWorkAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var jan = new DateTimeOffset(2024, 1, 15, 0, 0, 0, TimeSpan.Zero); + var mar = new DateTimeOffset(2024, 3, 15, 0, 0, 0, TimeSpan.Zero); + var jun = new DateTimeOffset(2024, 6, 15, 0, 0, 0, TimeSpan.Zero); + var sep = new DateTimeOffset(2024, 9, 15, 0, 0, 0, TimeSpan.Zero); + var dec = new DateTimeOffset(2024, 12, 15, 0, 0, 0, TimeSpan.Zero); + + _ = await CreateEntityAsync(store, "Event1", timestamp: jan); + _ = await CreateEntityAsync(store, "Event2", timestamp: mar); + _ = await CreateEntityAsync(store, "Event3", timestamp: jun); + _ = await CreateEntityAsync(store, "Event4", timestamp: sep); + _ = await CreateEntityAsync(store, "Event5", timestamp: dec); + + // Filter: events in second half of year (after June) + var midYear = new DateTime(2024, 6, 30, 23, 59, 59, DateTimeKind.Utc); + var filter = new DateTimeField("timestamp").GreaterThan(midYear); + var sort = new SortParameter(new DateTimeField("timestamp")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items[0].Value.Name.ShouldBe("Event4"); // Sep (earlier) + result.Items[1].Value.Name.ShouldBe("Event5"); // Dec (later) + } + + [Fact] + public async Task QuerySortWithPaginationShouldMaintainOrderAcrossPagesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 10; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", rank: i * 10); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank"), SortDirection.Descending); + + // Act - Get first page + var page1 = DataRange.FromPage(1, 3); + var result1 = await store.QueryAsync(_testEntityType, filter, sort, page1, Ct.None); + + // Act - Get second page + var page2 = DataRange.FromPage(2, 3); + var result2 = await store.QueryAsync(_testEntityType, filter, sort, page2, Ct.None); + + // Act - Get third page + var page3 = DataRange.FromPage(3, 3); + var result3 = await store.QueryAsync(_testEntityType, filter, sort, page3, Ct.None); + + // Assert + result1.Items.Count.ShouldBe(3); + result1.Items[0].Value.Rank.ShouldBe(100); + result1.Items[1].Value.Rank.ShouldBe(90); + result1.Items[2].Value.Rank.ShouldBe(80); + + result2.Items.Count.ShouldBe(3); + result2.Items[0].Value.Rank.ShouldBe(70); + result2.Items[1].Value.Rank.ShouldBe(60); + result2.Items[2].Value.Rank.ShouldBe(50); + + result3.Items.Count.ShouldBe(3); + result3.Items[0].Value.Rank.ShouldBe(40); + result3.Items[1].Value.Rank.ShouldBe(30); + result3.Items[2].Value.Rank.ShouldBe(20); + } + + [Fact] + public async Task QueryNoSortWithPaginationShouldReturnConsistentResultsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 5; i++) + { + _ = await CreateEntityAsync(store, $"Item{i}"); + } + + var filter = Query.All(); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(5); + result.TotalCount.ShouldBe(5); + } + + [Fact] + public async Task QuerySortWithDuplicateValuesShouldReturnAllItemsAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Alice", rank: 100); + _ = await CreateEntityAsync(store, "Bob", rank: 100); + _ = await CreateEntityAsync(store, "Charlie", rank: 100); + _ = await CreateEntityAsync(store, "David", rank: 50); + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank"), SortDirection.Descending); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(4); + // First three should all have rank 100 + result.Items[0].Value.Rank.ShouldBe(100); + result.Items[1].Value.Rank.ShouldBe(100); + result.Items[2].Value.Rank.ShouldBe(100); + // Last should have rank 50 + result.Items[3].Value.Rank.ShouldBe(50); + result.Items[3].Value.Name.ShouldBe("David"); + } + + [Fact] + public async Task QuerySortEmptyResultShouldReturnEmptyListAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "Alice", rank: 100); + _ = await CreateEntityAsync(store, "Bob", rank: 200); + + var filter = new NumberField("rank").GreaterThan(300); + var sort = new SortParameter(new NumberField("rank")); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(0); + result.TotalCount.ShouldBe(0); + } + + [Fact] + public async Task QuerySortSingleItemShouldReturnThatItemAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "OnlyOne", rank: 42); + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank"), SortDirection.Descending); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("OnlyOne"); + result.Items[0].Value.Rank.ShouldBe(42); + } + + [Fact] + public async Task QuerySortLargeDatasetShouldHandleEfficientlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 100 items with random-ish ranks + for (var i = 1; i <= 100; i++) + { + var rank = (i * 7) % 100; // Creates a pseudo-random distribution + _ = await CreateEntityAsync(store, $"Item{i:D3}", rank: rank); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + var page = DataRange.FromPage(1, 20); + + // Act + var result = await store.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(20); + result.TotalCount.ShouldBe(100); + + // Verify ordering of first page + for (var i = 0; i < result.Items.Count - 1; i++) + { + _ = result.Items[i].Value.Rank.ShouldNotBeNull(); + _ = result.Items[i + 1].Value.Rank.ShouldNotBeNull(); + result.Items[i].Value.Rank!.Value.ShouldBeLessThanOrEqualTo(result.Items[i + 1].Value.Rank!.Value); + } + } + + [Fact] + public async Task QuerySortedPagingShouldPreserveOrderAcrossAllPagesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 17 items (to test partial last page) + for (var i = 1; i <= 17; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", rank: i * 5); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank"), SortDirection.Descending); + + // Act - Get all 4 pages (5+5+5+2) + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 5), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 5), Ct.None); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 5), Ct.None); + var page4 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(4, 5), Ct.None); + + // Assert metadata + page1.TotalCount.ShouldBe(17); + page1.HasMoreData.ShouldBeTrue(); + + page2.HasMoreData.ShouldBeTrue(); + + page3.HasMoreData.ShouldBeTrue(); + + page4.Items.Count.ShouldBe(2); // Partial last page + page4.HasMoreData.ShouldBeFalse(); + + // Assert descending order across pages + page1.Items[0].Value.Rank.ShouldBe(85); // Highest + page1.Items[4].Value.Rank.ShouldBe(65); + + page2.Items[0].Value.Rank.ShouldBe(60); + page2.Items[4].Value.Rank.ShouldBe(40); + + page3.Items[0].Value.Rank.ShouldBe(35); + page3.Items[4].Value.Rank.ShouldBe(15); + + page4.Items[0].Value.Rank.ShouldBe(10); + page4.Items[1].Value.Rank.ShouldBe(5); // Lowest + } + + [Fact] + public async Task QuerySortByDateTimeWithPagingShouldHandlePageBreaksCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 11 events (to test exact break: 4+4+3) + for (var i = 1; i <= 11; i++) + { + var date = new DateTimeOffset(2024, 1, i, 10, 0, 0, TimeSpan.Zero); + _ = await CreateEntityAsync(store, $"Event{i:D2}", timestamp: date); + } + + var filter = Query.All(); + var sort = new SortParameter(new DateTimeField("timestamp")); + + // Act + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 4), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 4), Ct.None); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 4), Ct.None); + + // Assert correct dates across pages + page1.Items.Count.ShouldBe(4); + page1.Items[0].Value.Name.ShouldBe("Event01"); + page1.Items[3].Value.Name.ShouldBe("Event04"); + + page2.Items.Count.ShouldBe(4); + page2.Items[0].Value.Name.ShouldBe("Event05"); + page2.Items[3].Value.Name.ShouldBe("Event08"); + + page3.Items.Count.ShouldBe(3); + page3.Items[0].Value.Name.ShouldBe("Event09"); + page3.Items[2].Value.Name.ShouldBe("Event11"); + } + + [Fact] + public async Task QuerySortByStringWithPageBreakAtDuplicatesShouldHandleCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create items with duplicate names around page break + _ = await CreateEntityAsync(store, "Alpha", rank: 1); + _ = await CreateEntityAsync(store, "Alpha", rank: 2); + _ = await CreateEntityAsync(store, "Alpha", rank: 3); // Page 1 ends here (page size 3) + _ = await CreateEntityAsync(store, "Beta", rank: 4); // Page 2 starts here + _ = await CreateEntityAsync(store, "Beta", rank: 5); + _ = await CreateEntityAsync(store, "Charlie", rank: 6); + + var filter = Query.All(); + var sort = new SortParameter(new StringField("name")); + + // Act + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 3), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 3), Ct.None); + + // Assert - All Alphas should be on page 1, Betas and Charlie on page 2 + page1.Items.Count.ShouldBe(3); + page1.Items.ShouldAllBe(x => x.Value.Name == "Alpha"); + + page2.Items.Count.ShouldBe(3); + page2.Items[0].Value.Name.ShouldBe("Beta"); + page2.Items[1].Value.Name.ShouldBe("Beta"); + page2.Items[2].Value.Name.ShouldBe("Charlie"); + } + + [Fact] + public async Task QuerySortDescendingWithSmallPageSizeShouldReverseOrderAcrossPagesAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 10; i <= 19; i++) + { + _ = await CreateEntityAsync(store, $"Item{i}", rank: i); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank"), SortDirection.Descending); + + // Act - Page size of 3 creates 4 pages (3+3+3+1) + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 3), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 3), Ct.None); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 3), Ct.None); + var page4 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(4, 3), Ct.None); + + // Assert - Should be in descending order + page1.Items[0].Value.Rank.ShouldBe(19); + page1.Items[2].Value.Rank.ShouldBe(17); + + page2.Items[0].Value.Rank.ShouldBe(16); + page2.Items[2].Value.Rank.ShouldBe(14); + + page3.Items[0].Value.Rank.ShouldBe(13); + page3.Items[2].Value.Rank.ShouldBe(11); + + page4.Items.Count.ShouldBe(1); + page4.Items[0].Value.Rank.ShouldBe(10); + } + + [Fact] + public async Task QuerySortWithFilterPagingAcrossPageBreaksShouldMaintainOrderAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create 30 items + for (var i = 1; i <= 30; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", + rank: i, + category: i % 2 == 0 ? "even" : "odd"); + } + + // Filter for even numbers (15 results) + var filter = new StringField("category").Equals("even"); + var sort = new SortParameter(new NumberField("rank")); + + // Act - Page size 4 creates 4 pages (4+4+4+3) + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 4), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 4), Ct.None); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 4), Ct.None); + var page4 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(4, 4), Ct.None); + + // Assert + page1.TotalCount.ShouldBe(15); + page1.Items[0].Value.Rank.ShouldBe(2); + page1.Items[3].Value.Rank.ShouldBe(8); + + page2.Items[0].Value.Rank.ShouldBe(10); + page2.Items[3].Value.Rank.ShouldBe(16); + + page3.Items[0].Value.Rank.ShouldBe(18); + page3.Items[3].Value.Rank.ShouldBe(24); + + page4.Items.Count.ShouldBe(3); + page4.Items[0].Value.Rank.ShouldBe(26); + page4.Items[2].Value.Rank.ShouldBe(30); + } + + [Fact] + public async Task QueryExactMultipleOfPageSizeShouldNotHaveExtraEmptyPageAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Create exactly 15 items (exactly 3 pages of 5) + for (var i = 1; i <= 15; i++) + { + _ = await CreateEntityAsync(store, $"Item{i:D2}", rank: i); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + + // Act + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 5), Ct.None); + var page4 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(4, 5), Ct.None); + + // Assert + page3.Items.Count.ShouldBe(5); // Full last page + page3.HasMoreData.ShouldBeFalse(); + page3.Items[0].Value.Rank.ShouldBe(11); + page3.Items[4].Value.Rank.ShouldBe(15); + + // Page 4 should be empty + page4.Items.Count.ShouldBe(0); + } + + [Fact] + public async Task QuerySingleItemPerPageShouldIterateCorrectlyAsync() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + _ = await CreateEntityAsync(store, "First", rank: 1); + _ = await CreateEntityAsync(store, "Second", rank: 2); + _ = await CreateEntityAsync(store, "Third", rank: 3); + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + + // Act - Page size of 1 + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 1), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 1), Ct.None); + var page3 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(3, 1), Ct.None); + var page4 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(4, 1), Ct.None); + + // Assert + page1.Items.Count.ShouldBe(1); + page1.Items[0].Value.Name.ShouldBe("First"); + page1.HasMoreData.ShouldBeTrue(); + + page2.Items.Count.ShouldBe(1); + page2.Items[0].Value.Name.ShouldBe("Second"); + + page3.Items.Count.ShouldBe(1); + page3.Items[0].Value.Name.ShouldBe("Third"); + page3.HasMoreData.ShouldBeFalse(); + + page4.Items.Count.ShouldBe(0); + } + + [Fact] + public async Task QueryLargePageSizeShouldFitAllOnOnePage() + { + // Arrange + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + for (var i = 1; i <= 10; i++) + { + _ = await CreateEntityAsync(store, $"Item{i}", rank: i); + } + + var filter = Query.All(); + var sort = new SortParameter(new NumberField("rank")); + + // Act - Page size larger than total items + var page1 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(1, 100), Ct.None); + var page2 = await store.QueryAsync(_testEntityType, filter, sort, DataRange.FromPage(2, 100), Ct.None); + + // Assert + page1.Items.Count.ShouldBe(10); + page1.HasMoreData.ShouldBeFalse(); + + page2.Items.Count.ShouldBe(0); + } + +} diff --git a/storage/test/SharedIntegrationTests/StoreBatchOperations.cs b/storage/test/SharedIntegrationTests/StoreBatchOperations.cs new file mode 100644 index 000000000..ba4a341a1 --- /dev/null +++ b/storage/test/SharedIntegrationTests/StoreBatchOperations.cs @@ -0,0 +1,446 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Querying.SearchFields; + +namespace Duende.Storage.IntegrationTests; +/// +/// Tests for batch operations across all store implementations. +/// +public partial class StoreBatchOperations +{ + + private static readonly EntityType EntityType = TestDso.DsoVersion.EntityType; + private static readonly EntityType EntityType2 = TestDso2.DsoVersion.EntityType; + private readonly Ct _ct = TestContext.Current.CancellationToken; + + [Fact] + public async Task CanExecuteEmptyBatchAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var result = await store.ExecuteBatchAsync([], [], _ct); + result.Success.ShouldBeTrue(); + result.Results.ShouldBeEmpty(); + } + + [Fact] + public async Task Can_create_single_entity() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var id = UuidV7.New(); + var testValue = new TestDso($"value-{Guid.NewGuid()}"); + var operations = new IStoreOperation[] + { + CreateOperation.For(id, testValue, [], SearchFieldCollection.Empty, Expiration.NoExpiration) + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + result.Success.ShouldBeTrue(); + result.Results.Count.ShouldBe(1); + result.Results[0].Index.ShouldBe(0); + result.Results[0].Outcome.ShouldBe(OperationOutcome.Success); + var readResult = await store.TryReadAsync(EntityType, id, _ct); + readResult.Found.ShouldBeTrue(); + ((TestDso)readResult.Dso!).Value.ShouldBe(testValue.Value); + } + + [Fact] + public async Task Can_create_multiple_entities_of_same_type() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var id1 = UuidV7.New(); + var id2 = UuidV7.New(); + var id3 = UuidV7.New(); + var value1 = new TestDso($"value1-{Guid.NewGuid()}"); + var value2 = new TestDso($"value2-{Guid.NewGuid()}"); + var value3 = new TestDso($"value3-{Guid.NewGuid()}"); + var operations = new IStoreOperation[] + { + CreateOperation.For(id1, value1, [], SearchFieldCollection.Empty, Expiration.NoExpiration), + CreateOperation.For(id2, value2, [], SearchFieldCollection.Empty, Expiration.NoExpiration), + CreateOperation.For(id3, value3, [], SearchFieldCollection.Empty, Expiration.NoExpiration) + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + result.Success.ShouldBeTrue(); + result.Results.Count.ShouldBe(3); + result.Results.ShouldAllBe(r => r.Outcome == OperationOutcome.Success); + (await store.TryReadAsync(EntityType, id1, _ct)).Found.ShouldBeTrue(); + (await store.TryReadAsync(EntityType, id2, _ct)).Found.ShouldBeTrue(); + (await store.TryReadAsync(EntityType, id3, _ct)).Found.ShouldBeTrue(); + } + + [Fact] + public async Task Can_create_different_entity_types() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var id1 = UuidV7.New(); + var id2 = UuidV7.New(); + var value1 = new TestDso($"testdso-{Guid.NewGuid()}"); + var value2 = new TestDso2($"testdso2-{Guid.NewGuid()}"); + var operations = new IStoreOperation[] + { + CreateOperation.For(id1, value1, [], SearchFieldCollection.Empty, Expiration.NoExpiration), + CreateOperation.For(id2, value2, [], SearchFieldCollection.Empty, Expiration.NoExpiration) + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + result.Success.ShouldBeTrue(); + result.Results.Count.ShouldBe(2); + result.Results.ShouldAllBe(r => r.Outcome == OperationOutcome.Success); + (await store.TryReadAsync(EntityType, id1, _ct)).Found.ShouldBeTrue(); + (await store.TryReadAsync(EntityType2, id2, _ct)).Found.ShouldBeTrue(); + } + + [Fact] + public async Task CanMixCreateUpdateDeleteAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + // Pre-create entities to update and delete + var updateId = UuidV7.New(); + var deleteId = UuidV7.New(); + var originalValue = new TestDso($"original-{Guid.NewGuid()}"); + var deleteValue = new TestDso($"delete-{Guid.NewGuid()}"); + (await store.CreateAsync(updateId, originalValue, [], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(CreateResult.Success); + (await store.CreateAsync(deleteId, deleteValue, [], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(CreateResult.Success); + var updateVersion = (await store.TryReadAsync(EntityType, updateId, _ct)).Version!.Value; + // Now execute batch with create + update + delete + var createId = UuidV7.New(); + var createValue = new TestDso($"created-{Guid.NewGuid()}"); + var updatedValue = new TestDso($"updated-{Guid.NewGuid()}"); + var operations = new IStoreOperation[] + { + CreateOperation.For(createId, createValue, [], SearchFieldCollection.Empty, Expiration.NoExpiration), + UpdateOperation.For(updateId, updatedValue, updateVersion, [], SearchFieldCollection.Empty, null), + DeleteOperation.ById(EntityType, deleteId) + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + result.Success.ShouldBeTrue(); + result.Results.Count.ShouldBe(3); + result.Results.ShouldAllBe(r => r.Outcome == OperationOutcome.Success); + // Verify create + var createRead = await store.TryReadAsync(EntityType, createId, _ct); + createRead.Found.ShouldBeTrue(); + ((TestDso)createRead.Dso).Value.ShouldBe(createValue.Value); + // Verify update + var updateRead = await store.TryReadAsync(EntityType, updateId, _ct); + updateRead.Found.ShouldBeTrue(); + ((TestDso)updateRead.Dso).Value.ShouldBe(updatedValue.Value); + // Verify delete + (await store.TryReadAsync(EntityType, deleteId, _ct)).Found.ShouldBeFalse(); + } + + [Fact] + public async Task RollsBackOnCreateAlreadyExists() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + // Pre-create an entity that will cause conflict + var existingId = UuidV7.New(); + var existingValue = new TestDso($"existing-{Guid.NewGuid()}"); + (await store.CreateAsync(existingId, existingValue, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + // Batch: create new entity + create with existing ID (should fail) + var newId = UuidV7.New(); + var newValue = new TestDso($"new-{Guid.NewGuid()}"); + var conflictValue = new TestDso($"conflict-{Guid.NewGuid()}"); + var operations = new IStoreOperation[] + { + CreateOperation.For(newId, newValue, [], SearchFieldCollection.Empty, Expiration.NoExpiration), + CreateOperation.For(existingId, conflictValue, [], SearchFieldCollection.Empty, Expiration.NoExpiration) // This will fail + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + result.Success.ShouldBeFalse(); + result.Results.Count.ShouldBe(2); + result.Results[0].Outcome.ShouldBe(OperationOutcome.Success); + result.Results[1].Outcome.ShouldBe(OperationOutcome.AlreadyExists); + // Verify rollback - newId should NOT exist + (await store.TryReadAsync(EntityType, newId, _ct)).Found.ShouldBeFalse(); + // Original entity should be unchanged + var readResult = await store.TryReadAsync(EntityType, existingId, _ct); + readResult.Found.ShouldBeTrue(); + ((TestDso)readResult.Dso).Value.ShouldBe(existingValue.Value); + } + + [Fact] + public async Task RollsBackOnKeyConflict() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + // Pre-create an entity with a key + var existingId = UuidV7.New(); + var existingValue = new TestDso($"existing-{Guid.NewGuid()}"); + var conflictKey = new TestJsonKeyDsk($"conflict-key-{Guid.NewGuid()}"); + (await store.CreateAsync(existingId, existingValue, [DataStorageKey.Create(conflictKey)], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult + .Success); + // Batch: create new entity + create with same key (should fail) + var newId1 = UuidV7.New(); + var newId2 = UuidV7.New(); + var newValue1 = new TestDso($"new1-{Guid.NewGuid()}"); + var newValue2 = new TestDso($"new2-{Guid.NewGuid()}"); + var operations = new IStoreOperation[] + { + CreateOperation.For(newId1, newValue1, [], SearchFieldCollection.Empty, Expiration.NoExpiration), + CreateOperation.For(newId2, newValue2, [DataStorageKey.Create(conflictKey)], SearchFieldCollection.Empty, Expiration.NoExpiration) // Key conflict + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + result.Success.ShouldBeFalse(); + result.Results.Count.ShouldBe(2); + result.Results[0].Outcome.ShouldBe(OperationOutcome.Success); + result.Results[1].Outcome.ShouldBe(OperationOutcome.KeyConflict); + // Verify rollback - neither new entity should exist + (await store.TryReadAsync(EntityType, newId1, _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, newId2, _ct)).Found.ShouldBeFalse(); + } + + [Fact] + public async Task RollsBackOnUpdateVersionMismatch() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + // Pre-create an entity + var existingId = UuidV7.New(); + var existingValue = new TestDso($"existing-{Guid.NewGuid()}"); + (await store.CreateAsync(existingId, existingValue, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var correctVersion = (await store.TryReadAsync(EntityType, existingId, _ct)).Version!.Value; + // Batch: create new entity + update with wrong version (should fail) + var newId = UuidV7.New(); + var newValue = new TestDso($"new-{Guid.NewGuid()}"); + var updatedValue = new TestDso($"updated-{Guid.NewGuid()}"); + var operations = new IStoreOperation[] + { + CreateOperation.For(newId, newValue, [], SearchFieldCollection.Empty, Expiration.NoExpiration), + UpdateOperation.For(existingId, updatedValue, correctVersion + 999, [], SearchFieldCollection.Empty, null) // Wrong version + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + result.Success.ShouldBeFalse(); + result.Results.Count.ShouldBe(2); + result.Results[0].Outcome.ShouldBe(OperationOutcome.Success); + result.Results[1].Outcome.ShouldBe(OperationOutcome.UnexpectedVersion); + // Verify rollback - new entity should NOT exist + (await store.TryReadAsync(EntityType, newId, _ct)).Found.ShouldBeFalse(); + // Original entity should be unchanged + var readResult = await store.TryReadAsync(EntityType, existingId, _ct); + ((TestDso)readResult.Dso!).Value.ShouldBe(existingValue.Value); + } + [Fact] + public async Task RollsBackOnUpdateDoesNotExist() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var nonExistentId = UuidV7.New(); + var newId = UuidV7.New(); + var newValue = new TestDso($"new-{Guid.NewGuid()}"); + var updateValue = new TestDso($"update-{Guid.NewGuid()}"); + var operations = new IStoreOperation[] + { + CreateOperation.For(newId, newValue, [], SearchFieldCollection.Empty, Expiration.NoExpiration), + UpdateOperation.For(nonExistentId, updateValue, 1, [], SearchFieldCollection.Empty, null) // Does not exist + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + result.Success.ShouldBeFalse(); + result.Results.Count.ShouldBe(2); + result.Results[0].Outcome.ShouldBe(OperationOutcome.Success); + result.Results[1].Outcome.ShouldBe(OperationOutcome.DoesNotExist); + // Verify rollback - new entity should NOT exist + (await store.TryReadAsync(EntityType, newId, _ct)).Found.ShouldBeFalse(); + } + [Fact] + public async Task StopsOnFirstFailure() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + // Pre-create an entity for ID conflict + var existingId = UuidV7.New(); + var existingValue = new TestDso($"existing-{Guid.NewGuid()}"); + (await store.CreateAsync(existingId, existingValue, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var nonExistentId = UuidV7.New(); + var conflictValue = new TestDso($"conflict-{Guid.NewGuid()}"); + var updateValue = new TestDso($"update-{Guid.NewGuid()}"); + var operations = new IStoreOperation[] + { + CreateOperation.For(existingId, conflictValue, [], SearchFieldCollection.Empty, Expiration.NoExpiration), // AlreadyExists + UpdateOperation.For(nonExistentId, updateValue, 1, [], SearchFieldCollection.Empty, null) // DoesNotExist + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + result.Success.ShouldBeFalse(); + result.Results.Count.ShouldBe(1); + result.Results[0].Index.ShouldBe(0); + result.Results[0].Outcome.ShouldBe(OperationOutcome.AlreadyExists); + } + [Fact] + public async Task Can_delete_by_id() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + // Pre-create entities to delete + var id1 = UuidV7.New(); + var id2 = UuidV7.New(); + var value1 = new TestDso($"delete1-{Guid.NewGuid()}"); + var value2 = new TestDso($"delete2-{Guid.NewGuid()}"); + (await store.CreateAsync(id1, value1, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + (await store.CreateAsync(id2, value2, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var operations = new IStoreOperation[] + { + DeleteOperation.ById(EntityType, id1), + DeleteOperation.ById(EntityType, id2) + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + result.Success.ShouldBeTrue(); + result.Results.Count.ShouldBe(2); + result.Results.ShouldAllBe(r => r.Outcome == OperationOutcome.Success); + (await store.TryReadAsync(EntityType, id1, _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, id2, _ct)).Found.ShouldBeFalse(); + } + [Fact] + public async Task Can_delete_by_key() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + // Pre-create entity with a key + var id = UuidV7.New(); + var value = new TestDso($"delete-{Guid.NewGuid()}"); + var key = new TestJsonKeyDsk($"delete-key-{Guid.NewGuid()}"); + (await store.CreateAsync(id, value, [DataStorageKey.Create(key)], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var operations = new IStoreOperation[] + { + DeleteOperation.ByKey(EntityType, DataStorageKey.Create(key)) + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + result.Success.ShouldBeTrue(); + result.Results.Count.ShouldBe(1); + result.Results[0].Outcome.ShouldBe(OperationOutcome.Success); + (await store.TryReadAsync(EntityType, id, _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(key), _ct)).Found.ShouldBeFalse(); + } + [Fact] + public async Task DeleteByIdSucceedsWhenNotFoundAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var newId = UuidV7.New(); + var newValue = new TestDso($"new-{Guid.NewGuid()}"); + var nonExistentId = UuidV7.New(); + var operations = new IStoreOperation[] + { + CreateOperation.For(newId, newValue, [], SearchFieldCollection.Empty, Expiration.NoExpiration), + DeleteOperation.ById(EntityType, nonExistentId) // Does not exist - should still succeed + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + // Delete of non-existent entity should NOT be treated as a failure + result.Success.ShouldBeTrue(); + result.Results.Count.ShouldBe(2); + result.Results[0].Outcome.ShouldBe(OperationOutcome.Success); + result.Results[1].Outcome.ShouldBe(OperationOutcome.Success); + // The create should have been committed (no rollback) + (await store.TryReadAsync(EntityType, newId, _ct)).Found.ShouldBeTrue(); + } + + [Fact] + public async Task DeleteByKeySucceedsWhenNotFoundAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var newId = UuidV7.New(); + var newValue = new TestDso($"new-{Guid.NewGuid()}"); + var nonExistentKey = new TestJsonKeyDsk($"nonexistent-key-{Guid.NewGuid()}"); + var operations = new IStoreOperation[] + { + CreateOperation.For(newId, newValue, [], SearchFieldCollection.Empty, Expiration.NoExpiration), + DeleteOperation.ByKey(EntityType, DataStorageKey.Create(nonExistentKey)) // Does not exist - should still succeed + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + // Delete of non-existent entity should NOT be treated as a failure + result.Success.ShouldBeTrue(); + result.Results.Count.ShouldBe(2); + result.Results[0].Outcome.ShouldBe(OperationOutcome.Success); + result.Results[1].Outcome.ShouldBe(OperationOutcome.Success); + // The create should have been committed (no rollback) + (await store.TryReadAsync(EntityType, newId, _ct)).Found.ShouldBeTrue(); + } + [Fact] + public async Task ConcurrentBatchesAreIsolatedAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var concurrencyLevel = Math.Min(Environment.ProcessorCount, 5); + var tasks = new Task[concurrencyLevel]; + for (var i = 0; i < concurrencyLevel; i++) + { + var batchIndex = i; + tasks[i] = Task.Run(async () => + { + var id1 = UuidV7.New(); + var id2 = UuidV7.New(); + var value1 = new TestDso($"batch{batchIndex}-entity1-{Guid.NewGuid()}"); + var value2 = new TestDso($"batch{batchIndex}-entity2-{Guid.NewGuid()}"); + var operations = new IStoreOperation[] + { + CreateOperation.For(id1, value1, [], SearchFieldCollection.Empty, Expiration.NoExpiration), + CreateOperation.For(id2, value2, [], SearchFieldCollection.Empty, Expiration.NoExpiration) + }; + return await store.ExecuteBatchAsync(operations, [], _ct); + }, _ct); + } + var results = await Task.WhenAll(tasks); + // All batches should succeed + results.ShouldAllBe(r => r.Success); + results.ShouldAllBe(r => r.Results.Count == 2); + results.ShouldAllBe(r => r.Results.All(op => op.Outcome == OperationOutcome.Success)); + } + [Fact] + public async Task BatchWithKeysCreatesAndDeletesKeysCorrectlyAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var id = UuidV7.New(); + var value = new TestDso($"value-{Guid.NewGuid()}"); + var key1 = new TestJsonKeyDsk($"key1-{Guid.NewGuid()}"); + var key2 = new TestUuidV7KeyDsk(Guid.CreateVersion7()); + var operations = new IStoreOperation[] + { + CreateOperation.For(id, value, [DataStorageKey.Create(key1), DataStorageKey.Create(key2)], SearchFieldCollection.Empty, Expiration.NoExpiration) + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + result.Success.ShouldBeTrue(); + // Verify entity can be read by all keys + (await store.TryReadAsync(EntityType, id, _ct)).Found.ShouldBeTrue(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(key1), _ct)).Found.ShouldBeTrue(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(key2), _ct)).Found.ShouldBeTrue(); + } + [Fact] + public async Task UpdateInBatchUpdatesVersionAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + // Pre-create entity + var id = UuidV7.New(); + var originalValue = new TestDso($"original-{Guid.NewGuid()}"); + (await store.CreateAsync(id, originalValue, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var version1 = (await store.TryReadAsync(EntityType, id, _ct)).Version!.Value; + // Update in batch + var updatedValue = new TestDso($"updated-{Guid.NewGuid()}"); + var operations = new IStoreOperation[] + { + UpdateOperation.For(id, updatedValue, version1, [], SearchFieldCollection.Empty, null) + }; + var result = await store.ExecuteBatchAsync(operations, [], _ct); + result.Success.ShouldBeTrue(); + // Verify version incremented + var readResult = await store.TryReadAsync(EntityType, id, _ct); + readResult.Version.ShouldBe(version1 + 1); + ((TestDso)readResult.Dso!).Value.ShouldBe(updatedValue.Value); + } + private async Task CreateProviderAsync() => + await FixtureFactory.CreateAsync(_ct, services => + { + services.AddDsoRegistration(); + services.AddDsoRegistration(); + }); +} diff --git a/storage/test/SharedIntegrationTests/StoreLinkOperations.cs b/storage/test/SharedIntegrationTests/StoreLinkOperations.cs new file mode 100644 index 000000000..fc7bbc709 --- /dev/null +++ b/storage/test/SharedIntegrationTests/StoreLinkOperations.cs @@ -0,0 +1,458 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.SearchFields; +using Duende.Storage.Pagination; + +namespace Duende.Storage.IntegrationTests; + +/// +/// Integration tests for Link/Unlink operations across all store types. +/// Covers basic link/unlink, batch operations, and cascade delete behavior. +/// +public partial class StoreLinkOperations +{ + + private readonly Ct _ct = TestContext.Current.CancellationToken; + + private static readonly EntityType LeftEntityType = TestDso.DsoVersion.EntityType; + private static readonly EntityType RightEntityType = TestDso2.DsoVersion.EntityType; + private static readonly LinkDefinition TestLink = TestLinkData.TestLink; + private static readonly LinkDefinition TestLink2 = TestLinkData.TestLink2; + + // ========================================================================= + // Link / Unlink basic operations + // ========================================================================= + + [Fact] + public async Task CanLinkAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + // Create entities on both sides + _ = await store.CreateAsync(leftId, new TestDso("left"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(rightId, new TestDso2("right"), [], [], Expiration.NoExpiration, [], _ct); + + var result = await store.LinkAsync(TestLink, leftId, rightId, [], _ct); + + result.ShouldBe(LinkResult.Success); + + // Verify via QueryLinks: find TestDso2 entities linked from leftId + var query = LinkQuery.From(RightEntityType) + .Join(TestLink) + .Where(LeftEntityType, leftId) + .Build(); + var page = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + page.Items.Count.ShouldBe(1); + page.Items[0].Value.Value.ShouldBe("right"); + } + + [Fact] + public async Task LinkDuplicateReturnsAlreadyLinkedAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + _ = await store.LinkAsync(TestLink, leftId, rightId, [], _ct); + var second = await store.LinkAsync(TestLink, leftId, rightId, [], _ct); + + second.ShouldBe(LinkResult.AlreadyLinked); + } + + [Fact] + public async Task UnlinkRemovesLinkAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + _ = await store.CreateAsync(leftId, new TestDso("left"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(rightId, new TestDso2("right"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.LinkAsync(TestLink, leftId, rightId, [], _ct); + + var unlinkResult = await store.UnlinkAsync(TestLink, leftId, rightId, [], _ct); + unlinkResult.ShouldBe(UnlinkResult.Success); + + // Verify the link is gone + var query = LinkQuery.From(RightEntityType) + .Join(TestLink) + .Where(LeftEntityType, leftId) + .Build(); + var page = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + page.Items.ShouldBeEmpty(); + } + + [Fact] + public async Task UnlinkNonExistentReturnsSuccessAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + // No link was ever created + var result = await store.UnlinkAsync(TestLink, leftId, rightId, [], _ct); + + result.ShouldBe(UnlinkResult.Success); + } + + [Fact] + public async Task LinkWithoutEntityExistingSucceedsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + // No entities created — no referential integrity check + var result = await store.LinkAsync(TestLink, leftId, rightId, [], _ct); + + result.ShouldBe(LinkResult.Success); + } + + // ========================================================================= + // Batch operations with links + // ========================================================================= + + [Fact] + public async Task CanLinkInBatchAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + _ = await store.CreateAsync(leftId, new TestDso("left"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(rightId, new TestDso2("right"), [], [], Expiration.NoExpiration, [], _ct); + + var batchResult = await store.ExecuteBatchAsync( + [LinkOperation.For(TestLink, leftId, rightId)], + [], + _ct); + + batchResult.Success.ShouldBeTrue(); + batchResult.Results.Count.ShouldBe(1); + batchResult.Results[0].Outcome.ShouldBe(OperationOutcome.Success); + + // Verify link exists + var query = LinkQuery.From(RightEntityType) + .Join(TestLink) + .Where(LeftEntityType, leftId) + .Build(); + var page = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + page.Items.Count.ShouldBe(1); + } + + [Fact] + public async Task CanMixCreateAndLinkInBatchAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + var batchResult = await store.ExecuteBatchAsync( + [ + CreateOperation.For(leftId, new TestDso("created-left"), [], SearchFieldCollection.Empty, Expiration.NoExpiration), + CreateOperation.For(rightId, new TestDso2("created-right"), [], SearchFieldCollection.Empty, Expiration.NoExpiration), + LinkOperation.For(TestLink, leftId, rightId) + ], + [], + _ct); + + batchResult.Success.ShouldBeTrue(); + batchResult.Results.Count.ShouldBe(3); + batchResult.Results.ShouldAllBe(r => r.Outcome == OperationOutcome.Success); + + // Verify entity exists + (await store.TryReadAsync(LeftEntityType, leftId, _ct)).Found.ShouldBeTrue(); + + // Verify link exists + var query = LinkQuery.From(RightEntityType) + .Join(TestLink) + .Where(LeftEntityType, leftId) + .Build(); + var page = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + page.Items.Count.ShouldBe(1); + } + + [Fact] + public async Task BatchLinkDuplicateIsIdempotentAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + // First link + _ = await store.LinkAsync(TestLink, leftId, rightId, [], _ct); + + // Batch with duplicate link — should succeed (idempotent) + var batchResult = await store.ExecuteBatchAsync( + [LinkOperation.For(TestLink, leftId, rightId)], + [], + _ct); + + batchResult.Success.ShouldBeTrue(); + batchResult.Results[0].Outcome.ShouldBe(OperationOutcome.AlreadyLinked); + } + + [Fact] + public async Task CanUnlinkInBatchAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + _ = await store.CreateAsync(leftId, new TestDso("left"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(rightId, new TestDso2("right"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.LinkAsync(TestLink, leftId, rightId, [], _ct); + + var batchResult = await store.ExecuteBatchAsync( + [UnlinkOperation.For(TestLink, leftId, rightId)], + [], + _ct); + + batchResult.Success.ShouldBeTrue(); + batchResult.Results[0].Outcome.ShouldBe(OperationOutcome.Success); + + // Verify link gone + var query = LinkQuery.From(RightEntityType) + .Join(TestLink) + .Where(LeftEntityType, leftId) + .Build(); + var page = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + page.Items.ShouldBeEmpty(); + } + + // ========================================================================= + // Cascade delete behavior + // ========================================================================= + + [Fact] + public async Task DeleteEntityRemovesLinksWhereEntityIsLeftAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + _ = await store.CreateAsync(leftId, new TestDso("left"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(rightId, new TestDso2("right"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.LinkAsync(TestLink, leftId, rightId, [], _ct); + + // Delete the left entity + _ = await store.DeleteAsync(LeftEntityType, leftId, [], _ct); + + // Query from right side — link should be gone + var query = LinkQuery.From(LeftEntityType) + .Join(TestLink) + .Where(RightEntityType, rightId) + .Build(); + var page = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + page.Items.ShouldBeEmpty(); + } + + [Fact] + public async Task DeleteEntityRemovesLinksWhereEntityIsRightAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + _ = await store.CreateAsync(leftId, new TestDso("left"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(rightId, new TestDso2("right"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.LinkAsync(TestLink, leftId, rightId, [], _ct); + + // Delete the right entity + _ = await store.DeleteAsync(RightEntityType, rightId, [], _ct); + + // Query from left side — link should be gone + var query = LinkQuery.From(RightEntityType) + .Join(TestLink) + .Where(LeftEntityType, leftId) + .Build(); + var page = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + page.Items.ShouldBeEmpty(); + } + + [Fact] + public async Task DeleteEntityRemovesMultipleLinksAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var leftId = UuidV7.New(); + var rightId1 = UuidV7.New(); + var rightId2 = UuidV7.New(); + + _ = await store.CreateAsync(leftId, new TestDso("left"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(rightId1, new TestDso2("right1"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(rightId2, new TestDso2("right2"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.LinkAsync(TestLink, leftId, rightId1, [], _ct); + _ = await store.LinkAsync(TestLink, leftId, rightId2, [], _ct); + + // Delete the left entity — both links should go + _ = await store.DeleteAsync(LeftEntityType, leftId, [], _ct); + + var query = LinkQuery.From(LeftEntityType) + .Join(TestLink) + .Where(RightEntityType, rightId1) + .Build(); + var page = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + page.Items.ShouldBeEmpty(); + } + + [Fact] + public async Task BatchDeleteEntityRemovesLinksAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + _ = await store.CreateAsync(leftId, new TestDso("left"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(rightId, new TestDso2("right"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.LinkAsync(TestLink, leftId, rightId, [], _ct); + + // Delete via batch + var batchResult = await store.ExecuteBatchAsync( + [DeleteOperation.ById(LeftEntityType, leftId)], + [], + _ct); + + batchResult.Success.ShouldBeTrue(); + + // Link should be gone + var query = LinkQuery.From(LeftEntityType) + .Join(TestLink) + .Where(RightEntityType, rightId) + .Build(); + var page = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + page.Items.ShouldBeEmpty(); + } + + [Fact] + public async Task DeleteEntityByKeyRemovesLinksAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + var leftKey = new TestJsonKeyDsk($"left-key-{Guid.NewGuid()}"); + + _ = await store.CreateAsync(leftId, new TestDso("left"), [DataStorageKey.Create(leftKey)], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(rightId, new TestDso2("right"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.LinkAsync(TestLink, leftId, rightId, [], _ct); + + // Delete the left entity by key (not by ID) + var result = await store.DeleteAsync(LeftEntityType, DataStorageKey.Create(leftKey), [], _ct); + result.ShouldBe(DeleteResult.Success); + + // Link should be gone + var query = LinkQuery.From(LeftEntityType) + .Join(TestLink) + .Where(RightEntityType, rightId) + .Build(); + var page = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + page.Items.ShouldBeEmpty(); + } + + [Fact] + public async Task BatchDeleteEntityByKeyRemovesLinksAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + var leftKey = new TestJsonKeyDsk($"left-key-{Guid.NewGuid()}"); + + _ = await store.CreateAsync(leftId, new TestDso("left"), [DataStorageKey.Create(leftKey)], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(rightId, new TestDso2("right"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.LinkAsync(TestLink, leftId, rightId, [], _ct); + + // Delete via batch by key + var batchResult = await store.ExecuteBatchAsync( + [DeleteOperation.ByKey(LeftEntityType, DataStorageKey.Create(leftKey))], + [], + _ct); + batchResult.Success.ShouldBeTrue(); + + // Link should be gone + var query = LinkQuery.From(LeftEntityType) + .Join(TestLink) + .Where(RightEntityType, rightId) + .Build(); + var page = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + page.Items.ShouldBeEmpty(); + } + + [Fact] + public async Task ConcurrentLinkCallsNeverThrowAsync() + { + // When multiple concurrent Link calls target the same entity pair, + // exactly one should succeed and the rest should return AlreadyLinked. + // Before the race-condition fix, the MsSql INSERT WHERE NOT EXISTS + // could allow two transactions to both observe "not exists" and then + // one would hit an unhandled PK violation exception. + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + const int Concurrency = 10; + var tasks = Enumerable.Range(0, Concurrency) + .Select(_ => store.LinkAsync(TestLink, leftId, rightId, [], _ct)) + .ToArray(); + + var results = await Task.WhenAll(tasks); + + results.Count(r => r == LinkResult.Success).ShouldBe(1); + results.Count(r => r == LinkResult.AlreadyLinked).ShouldBe(Concurrency - 1); + } + + private async Task CreateProviderAsync() => + await FixtureFactory.CreateAsync(_ct, services => + { + services.AddDsoRegistration(); + services.AddDsoRegistration(); + }); +} diff --git a/storage/test/SharedIntegrationTests/StoreLinkQueryTests.cs b/storage/test/SharedIntegrationTests/StoreLinkQueryTests.cs new file mode 100644 index 000000000..3a051b386 --- /dev/null +++ b/storage/test/SharedIntegrationTests/StoreLinkQueryTests.cs @@ -0,0 +1,347 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Pagination; + +namespace Duende.Storage.IntegrationTests; + +// --------------------------------------------------------------------------- +// Domain-flavored test DSOs: User, Role, Group +// Each gets a unique EntityType id in the test-reserved range. +// --------------------------------------------------------------------------- + +/// +/// Integration tests for QueryLinks across all store types. +/// Uses a Users / Roles / Groups domain to make link traversals readable: +/// UserRole : User → Role (a user has a role) +/// UserGroup : User → Group (a user belongs to a group) +/// GroupRole : Group → Role (a group has a role) +/// +public partial class StoreLinkQueryTests +{ + + private readonly Ct _ct = TestContext.Current.CancellationToken; + + // Entity types + private static readonly EntityType User = UserDso.DsoVersion.EntityType; + private static readonly EntityType Role = RoleDso.DsoVersion.EntityType; + private static readonly EntityType Group = GroupDso.DsoVersion.EntityType; + + // Link definitions + private static readonly LinkDefinition UserRole = new() + { + Left = User, + Right = Role, + Link = LinkTypeRegistry.MembershipRole + }; + + private static readonly LinkDefinition UserGroup = new() + { + Left = User, + Right = Group, + Link = LinkTypeRegistry.MembershipGroup + }; + + private static readonly LinkDefinition GroupRole = new() + { + Left = Group, + Right = Role, + Link = LinkTypeRegistry.GroupRole + }; + + // ========================================================================= + // Single-hop queries + // ========================================================================= + + [Fact] + public async Task FindRolesAssignedToUserAsync() + { + // Given a user linked to a role, querying roles for that user returns the role. + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var userId = UuidV7.New(); + var roleId = UuidV7.New(); + + _ = await store.CreateAsync(userId, new UserDso("alice"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(roleId, new RoleDso("admin"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.LinkAsync(UserRole, userId, roleId, [], _ct); + + var query = LinkQuery.From(Role) + .Join(UserRole) + .Where(User, userId) + .Build(); + + var result = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("admin"); + } + + [Fact] + public async Task FindUsersWithRoleAsync() + { + // Given a user linked to a role, querying users for that role returns the user (reverse traversal). + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var userId = UuidV7.New(); + var roleId = UuidV7.New(); + + _ = await store.CreateAsync(userId, new UserDso("alice"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(roleId, new RoleDso("admin"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.LinkAsync(UserRole, userId, roleId, [], _ct); + + var query = LinkQuery.From(User) + .Join(UserRole) + .Where(Role, roleId) + .Build(); + + var result = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("alice"); + } + + [Fact] + public async Task EmptyResultWhenNoRolesAssignedAsync() + { + // Querying roles for a user that has none returns an empty page. + await using var fixture = await CreateProviderAsync(); + var queryStore = fixture.Store; + + var query = LinkQuery.From(Role) + .Join(UserRole) + .Where(User, UuidV7.New()) + .Build(); + + var result = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + + result.Items.ShouldBeEmpty(); + result.TotalCount.ShouldBe(0); + } + + // ========================================================================= + // Pagination + // ========================================================================= + + [Fact] + public async Task PaginateThroughGroupsForUserAsync() + { + // A user belongs to 5 groups; paging with size 3 yields two pages. + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var userId = UuidV7.New(); + _ = await store.CreateAsync(userId, new UserDso("alice"), [], [], Expiration.NoExpiration, [], _ct); + + for (var i = 0; i < 5; i++) + { + var groupId = UuidV7.New(); + _ = await store.CreateAsync(groupId, new GroupDso($"group-{i}"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.LinkAsync(UserGroup, userId, groupId, [], _ct); + } + + var query = LinkQuery.From(Group) + .Join(UserGroup) + .Where(User, userId) + .Build(); + + var page1 = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 3), _ct); + var page2 = await queryStore.QueryLinksAsync(query, DataRange.FromPage(2, 3), _ct); + + page1.Items.Count.ShouldBe(3); + page1.TotalCount.ShouldBe(5); + + page2.Items.Count.ShouldBe(2); + page2.TotalCount.ShouldBe(5); + } + + // ========================================================================= + // Distinct results + // ========================================================================= + + [Fact] + public async Task RoleSharedByTwoUsersAppearsOnce() + { + // Two users share the same role — querying all roles should return it only once. + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var aliceId = UuidV7.New(); + var bobId = UuidV7.New(); + var roleId = UuidV7.New(); + + _ = await store.CreateAsync(aliceId, new UserDso("alice"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(bobId, new UserDso("bob"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(roleId, new RoleDso("admin"), [], [], Expiration.NoExpiration, [], _ct); + + _ = await store.LinkAsync(UserRole, aliceId, roleId, [], _ct); + _ = await store.LinkAsync(UserRole, bobId, roleId, [], _ct); + + var query = LinkQuery.From(Role) + .Join(UserRole) + .Build(); + + var result = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + + result.Items.Count(r => r.Value.Name == "admin").ShouldBe(1); + } + + // ========================================================================= + // Multi-hop: User → Group → Role + // ========================================================================= + + [Fact] + public async Task FindUsersWhoHaveRoleThroughGroupAsync() + { + // alice → engineers (UserGroup) → admin (GroupRole) + // Multi-hop query: starting from User, traverse UserGroup then GroupRole, + // filtered to a specific role — should return alice. + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var queryStore = fixture.Store; + + var aliceId = UuidV7.New(); + var engineersId = UuidV7.New(); + var adminRoleId = UuidV7.New(); + + _ = await store.CreateAsync(aliceId, new UserDso("alice"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(engineersId, new GroupDso("engineers"), [], [], Expiration.NoExpiration, [], _ct); + _ = await store.CreateAsync(adminRoleId, new RoleDso("admin"), [], [], Expiration.NoExpiration, [], _ct); + + _ = await store.LinkAsync(UserGroup, aliceId, engineersId, [], _ct); + _ = await store.LinkAsync(GroupRole, engineersId, adminRoleId, [], _ct); + + // From User, hop through UserGroup (User→Group), then GroupRole (Group→Role), + // filter where Role = adminRoleId + var query = LinkQuery.From(User) + .Join(UserGroup) + .Join(GroupRole) + .Where(Role, adminRoleId) + .Build(); + + var result = await queryStore.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("alice"); + } + + // ========================================================================= + // Multi-space isolation + // ========================================================================= + + [Fact] + public async Task UserRoleLinkInSpaceANotVisibleInSpaceBAsync() + { + // Space A: alice → admin + await using var fixtureA = await CreateProviderAsync(); + var storeA = fixtureA.Store; + + // Space B: separate provider = separate SpaceId + await using var fixtureB = await CreateProviderAsync(); + var queryStoreB = fixtureB.Store; + + var userId = UuidV7.New(); + var roleId = UuidV7.New(); + + _ = await storeA.CreateAsync(userId, new UserDso("alice"), [], [], Expiration.NoExpiration, [], _ct); + _ = await storeA.CreateAsync(roleId, new RoleDso("admin"), [], [], Expiration.NoExpiration, [], _ct); + _ = await storeA.LinkAsync(UserRole, userId, roleId, [], _ct); + + // Querying in space B should return nothing + var query = LinkQuery.From(Role) + .Join(UserRole) + .Where(User, userId) + .Build(); + + var result = await queryStoreB.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + result.Items.ShouldBeEmpty(); + } + + [Fact] + public async Task SameUserRoleLinkInDifferentSpacesAreIndependentAsync() + { + await using var fixtureA = await CreateProviderAsync(); + var storeA = fixtureA.Store; + var queryStoreA = fixtureA.Store; + + await using var fixtureB = await CreateProviderAsync(); + var storeB = fixtureB.Store; + var queryStoreB = fixtureB.Store; + + var userId = UuidV7.New(); + var roleId = UuidV7.New(); + + // Create the entities in both spaces + _ = await storeA.CreateAsync(userId, new UserDso("user-a"), [], [], Expiration.NoExpiration, [], _ct); + _ = await storeA.CreateAsync(roleId, new RoleDso("role-a"), [], [], Expiration.NoExpiration, [], _ct); + _ = await storeB.CreateAsync(userId, new UserDso("user-b"), [], [], Expiration.NoExpiration, [], _ct); + _ = await storeB.CreateAsync(roleId, new RoleDso("role-b"), [], [], Expiration.NoExpiration, [], _ct); + + // Same link in both spaces + _ = await storeA.LinkAsync(UserRole, userId, roleId, [], _ct); + _ = await storeB.LinkAsync(UserRole, userId, roleId, [], _ct); + + var query = LinkQuery.From(Role) + .Join(UserRole) + .Where(User, userId) + .Build(); + + var resultA = await queryStoreA.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + var resultB = await queryStoreB.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + + // Each space sees only its own link + resultA.TotalCount.ShouldBe(1); + resultA.Items.Count.ShouldBe(1); + resultB.TotalCount.ShouldBe(1); + resultB.Items.Count.ShouldBe(1); + } + + [Fact] + public async Task DeleteInSpaceADoesNotAffectSpaceBAsync() + { + await using var fixtureA = await CreateProviderAsync(); + var storeA = fixtureA.Store; + + await using var fixtureB = await CreateProviderAsync(); + var storeB = fixtureB.Store; + var queryStoreB = fixtureB.Store; + + var userId = UuidV7.New(); + var roleId = UuidV7.New(); + + // Space B: create user and role, link them + _ = await storeB.CreateAsync(userId, new UserDso("bob"), [], [], Expiration.NoExpiration, [], _ct); + _ = await storeB.CreateAsync(roleId, new RoleDso("editor"), [], [], Expiration.NoExpiration, [], _ct); + _ = await storeB.LinkAsync(UserRole, userId, roleId, [], _ct); + + // Space A: create and delete the same user id — should not affect space B + _ = await storeA.CreateAsync(userId, new UserDso("alice"), [], [], Expiration.NoExpiration, [], _ct); + _ = await storeA.DeleteAsync(User, userId, [], _ct); + + // Space B link still intact + var query = LinkQuery.From(Role) + .Join(UserRole) + .Where(User, userId) + .Build(); + + var result = await queryStoreB.QueryLinksAsync(query, DataRange.FromPage(1, 100), _ct); + result.Items.Count.ShouldBe(1); + } + + private async Task CreateProviderAsync() => + await FixtureFactory.CreateAsync(_ct, services => + { + services.AddDsoRegistration(); + services.AddDsoRegistration(); + services.AddDsoRegistration(); + }); +} diff --git a/storage/test/SharedIntegrationTests/StoreOutboxOperations.cs b/storage/test/SharedIntegrationTests/StoreOutboxOperations.cs new file mode 100644 index 000000000..90f8c92a7 --- /dev/null +++ b/storage/test/SharedIntegrationTests/StoreOutboxOperations.cs @@ -0,0 +1,456 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Outbox; +using Duende.Storage.Internal.Querying.SearchFields; +using Microsoft.Extensions.DependencyInjection; +using OutboxEventId = Duende.Storage.Internal.Outbox.OutboxEventId; +using OutboxEventName = Duende.Storage.Internal.Outbox.OutboxEventName; +using SubscriberName = Duende.Storage.Internal.Outbox.SubscriberName; + +namespace Duende.Storage.IntegrationTests; + +/// +/// Integration tests for outbox event write and read operations across all store types. +/// +public partial class StoreOutboxOperations +{ + + private readonly Ct _ct = TestContext.Current.CancellationToken; + + private static readonly EntityType EntityType = TestDso.DsoVersion.EntityType; + private static readonly LinkDefinition TestLink = TestLinkData.TestLink; + private static readonly SubscriberName WildcardSubscriberName = + SubscriberName.Create("test-subscriber"); + + [Fact] + public async Task OutboxEventsAreWrittenOnCreate() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var id = UuidV7.New(); + var evt = MakeEvent(); + + var result = await store.CreateAsync(id, new TestDso("v"), [], SearchFieldCollection.Empty, Expiration.NoExpiration, [evt], _ct); + result.ShouldBe(CreateResult.Success); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 10, _ct); + page.Events.ShouldContain(e => e.EventId == evt.Id); + } + + [Fact] + public async Task OutboxEventsAreWrittenOnUpdate() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var id = UuidV7.New(); + (await store.CreateAsync(id, new TestDso("v"), [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var version = (await store.TryReadAsync(EntityType, id, _ct)).Version!.Value; + + var evt = MakeEvent(); + var result = await store.UpdateAsync(id, new TestDso("v2"), version, [], SearchFieldCollection.Empty, null, [evt], _ct); + result.ShouldBe(UpdateResult.Success); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 10, _ct); + page.Events.ShouldContain(e => e.EventId == evt.Id); + } + + [Fact] + public async Task OutboxEventsAreWrittenOnDelete() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var id = UuidV7.New(); + (await store.CreateAsync(id, new TestDso("v"), [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + + var evt = MakeEvent(); + var result = await store.DeleteAsync(EntityType, id, [evt], _ct); + result.ShouldBe(DeleteResult.Success); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 10, _ct); + page.Events.ShouldContain(e => e.EventId == evt.Id); + } + + [Fact] + public async Task OutboxEventsAreWrittenOnLink() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + + var evt = MakeEvent(); + var result = await store.LinkAsync(TestLink, leftId, rightId, [evt], _ct); + result.ShouldBe(LinkResult.Success); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 10, _ct); + page.Events.ShouldContain(e => e.EventId == evt.Id); + } + + [Fact] + public async Task OutboxEventsAreWrittenOnUnlink() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var leftId = UuidV7.New(); + var rightId = UuidV7.New(); + _ = await store.LinkAsync(TestLink, leftId, rightId, [], _ct); + + var evt = MakeEvent(); + var result = await store.UnlinkAsync(TestLink, leftId, rightId, [evt], _ct); + result.ShouldBe(UnlinkResult.Success); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 10, _ct); + page.Events.ShouldContain(e => e.EventId == evt.Id); + } + + [Fact] + public async Task OutboxEventsAreWrittenOnBatch() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var id = UuidV7.New(); + var operations = new IStoreOperation[] + { + CreateOperation.For(id, new TestDso("v"), [], SearchFieldCollection.Empty, Expiration.NoExpiration) + }; + + var evt = MakeEvent(); + var result = await store.ExecuteBatchAsync(operations, [evt], _ct); + result.Success.ShouldBeTrue(); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 10, _ct); + page.Events.ShouldContain(e => e.EventId == evt.Id); + } + + [Fact] + public async Task MultipleOutboxEventsPerTransactionAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var id = UuidV7.New(); + var evt1 = MakeEvent(); + var evt2 = MakeEvent(); + var evt3 = MakeEvent(); + + var result = await store.CreateAsync(id, new TestDso("v"), [], SearchFieldCollection.Empty, Expiration.NoExpiration, [evt1, evt2, evt3], _ct); + result.ShouldBe(CreateResult.Success); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 10, _ct); + page.Events.Select(e => e.EventId).ShouldContain(evt1.Id); + page.Events.Select(e => e.EventId).ShouldContain(evt2.Id); + page.Events.Select(e => e.EventId).ShouldContain(evt3.Id); + } + + [Fact] + public async Task DeleteOutboxEventsRemovesByIdAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var evt1 = MakeEvent(); + var evt2 = MakeEvent(); + var evt3 = MakeEvent(); + + var id = UuidV7.New(); + _ = await store.CreateAsync(id, new TestDso("v"), [], SearchFieldCollection.Empty, Expiration.NoExpiration, [evt1, evt2, evt3], _ct); + + // Get persisted events to retrieve their MessageIds + var allEvents = (await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 10, _ct)).Events; + var msgId1 = allEvents.Single(e => e.EventId == evt1.Id).MessageId; + var msgId2 = allEvents.Single(e => e.EventId == evt2.Id).MessageId; + + // Delete first two by MessageId + await store.DeleteOutboxEventsAsync([msgId1, msgId2], _ct); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 10, _ct); + page.Events.ShouldNotContain(e => e.EventId == evt1.Id); + page.Events.ShouldNotContain(e => e.EventId == evt2.Id); + page.Events.ShouldContain(e => e.EventId == evt3.Id); + } + + [Fact] + public async Task OutboxEventsNotWrittenWhenOperationFailsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var id = UuidV7.New(); + (await store.CreateAsync(id, new TestDso("existing"), [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + + var evt = MakeEvent(); + var result = await store.CreateAsync(id, new TestDso("duplicate"), [], SearchFieldCollection.Empty, Expiration.NoExpiration, [evt], _ct); + result.ShouldBe(CreateResult.AlreadyExists); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 10, _ct); + page.Events.ShouldNotContain(e => e.EventId == evt.Id); + } + + [Fact] + public async Task BatchOutboxEventsNotWrittenWhenBatchFailsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + // Pre-create an entity to cause conflict + var existingId = UuidV7.New(); + (await store.CreateAsync(existingId, new TestDso("existing"), [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + + var newId = UuidV7.New(); + var operations = new IStoreOperation[] + { + CreateOperation.For(newId, new TestDso("new"), [], SearchFieldCollection.Empty, Expiration.NoExpiration), + CreateOperation.For(existingId, new TestDso("conflict"), [], SearchFieldCollection.Empty, Expiration.NoExpiration), // will fail + }; + + var evt = MakeEvent(); + var result = await store.ExecuteBatchAsync(operations, [evt], _ct); + result.Success.ShouldBeFalse(); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 10, _ct); + page.Events.ShouldNotContain(e => e.EventId == evt.Id); + } + + [Fact] + public async Task OutboxEventsNotWrittenWhenDeletingNonExistentEntityAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var nonExistentId = UuidV7.New(); + var evt = MakeEvent(); + + var result = await store.DeleteAsync(EntityType, nonExistentId, [evt], _ct); + result.ShouldBe(DeleteResult.Success); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 10, _ct); + page.Events.ShouldNotContain(e => e.EventId == evt.Id); + } + + [Fact] + public async Task NoMessagesWrittenWhenNoSubscribersAsync() + { + await using var fixture = await CreateProviderAsync([]); + var store = fixture.Store; + + var id = UuidV7.New(); + var evt = MakeEvent(); + + var result = await store.CreateAsync(id, new TestDso("v"), [], SearchFieldCollection.Empty, Expiration.NoExpiration, [evt], _ct); + result.ShouldBe(CreateResult.Success); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 10, _ct); + page.Events.ShouldBeEmpty(); + } + + [Fact] + public async Task OneMessagePerSubscriber() + { + var subscriber = new TestSubscriber("sub-a"); + await using var fixture = await CreateProviderAsync([subscriber]); + var store = fixture.Store; + + var id = UuidV7.New(); + var evt = MakeEvent(); + + var result = await store.CreateAsync(id, new TestDso("v"), [], SearchFieldCollection.Empty, Expiration.NoExpiration, [evt], _ct); + result.ShouldBe(CreateResult.Success); + + var page = await store.GetOutboxEventsForSubscriberAsync(subscriber.SubscriberName, 10, _ct); + page.Events.Count.ShouldBe(1); + page.Events[0].EventId.ShouldBe(evt.Id); + page.Events[0].SubscriberName.ShouldBe(subscriber.SubscriberName); + } + + [Fact] + public async Task MultipleSubscribersProduceMultipleMessagesAsync() + { + var subA = new TestSubscriber("sub-a"); + var subB = new TestSubscriber("sub-b"); + var subC = new TestSubscriber("sub-c"); + await using var fixture = await CreateProviderAsync([subA, subB, subC]); + var store = fixture.Store; + + var id = UuidV7.New(); + var evt = MakeEvent(); + + var result = await store.CreateAsync(id, new TestDso("v"), [], SearchFieldCollection.Empty, Expiration.NoExpiration, [evt], _ct); + result.ShouldBe(CreateResult.Success); + + var pageA = await store.GetOutboxEventsForSubscriberAsync(subA.SubscriberName, 10, _ct); + var pageB = await store.GetOutboxEventsForSubscriberAsync(subB.SubscriberName, 10, _ct); + var pageC = await store.GetOutboxEventsForSubscriberAsync(subC.SubscriberName, 10, _ct); + var allEvents = pageA.Events.Concat(pageB.Events).Concat(pageC.Events).ToList(); + + // All 3 rows share the same EventId but have distinct MessageIds and SubscriberNames + allEvents.Count.ShouldBe(3); + allEvents.ShouldAllBe(e => e.EventId == evt.Id); + allEvents.Select(e => e.MessageId).Distinct().Count().ShouldBe(3); + allEvents.Select(e => e.SubscriberName).ShouldBe( + [subA.SubscriberName, subB.SubscriberName, subC.SubscriberName], ignoreOrder: true); + } + + [Fact] + public async Task OutboxEventsNotWrittenWhenDisabledAsync() + { + await using var fixture = await FixtureFactory.CreateAsync( + _ct, + services => + { + // No IOutboxSubscriber registrations → outbox is effectively disabled + services.AddDsoRegistration(); + services.AddDsoRegistration(); + }); + + var store = fixture.Store; + + var id = UuidV7.New(); + var evt = MakeEvent(); + + var result = await store.CreateAsync(id, new TestDso("v"), [], SearchFieldCollection.Empty, Expiration.NoExpiration, [evt], _ct); + result.ShouldBe(CreateResult.Success); + + var page = await store.GetOutboxEventsForSubscriberAsync(WildcardSubscriberName, 10, _ct); + page.Events.ShouldNotContain(e => e.EventId == evt.Id); + } + + [Fact] + public async Task GetOutboxEventsForSubscriberFiltersBySubscriberAsync() + { + var subA = new TestSubscriber("sub-a"); + var subB = new TestSubscriber("sub-b"); + await using var fixture = await CreateProviderAsync([subA, subB]); + var store = fixture.Store; + + var id = UuidV7.New(); + var evt = MakeEvent(); + _ = await store.CreateAsync(id, new TestDso("v"), [], SearchFieldCollection.Empty, Expiration.NoExpiration, [evt], _ct); + + var pageA = await store.GetOutboxEventsForSubscriberAsync(subA.SubscriberName, 10, _ct); + pageA.Events.ShouldAllBe(e => e.SubscriberName == subA.SubscriberName); + pageA.Events.Count.ShouldBe(1); + + var pageB = await store.GetOutboxEventsForSubscriberAsync(subB.SubscriberName, 10, _ct); + pageB.Events.ShouldAllBe(e => e.SubscriberName == subB.SubscriberName); + pageB.Events.Count.ShouldBe(1); + } + + [Fact] + public async Task GetOutboxEventsForSubscriberReturnsPagedAsync() + { + var sub = new TestSubscriber("sub-paged"); + await using var fixture = await CreateProviderAsync([sub]); + var store = fixture.Store; + + for (var i = 1; i <= 5; i++) + { + var id = UuidV7.New(); + _ = await store.CreateAsync(id, new TestDso($"v{i}"), [], SearchFieldCollection.Empty, Expiration.NoExpiration, [MakeEvent()], _ct); + } + + var page1 = await store.GetOutboxEventsForSubscriberAsync(sub.SubscriberName, 3, _ct); + page1.Events.Count.ShouldBe(3); + page1.HasMore.ShouldBeTrue(); + + await store.DeleteOutboxEventsAsync(page1.Events.Select(e => e.MessageId).ToList(), _ct); + + var page2 = await store.GetOutboxEventsForSubscriberAsync(sub.SubscriberName, 3, _ct); + page2.Events.Count.ShouldBe(2); + page2.HasMore.ShouldBeFalse(); + } + + [Fact] + public async Task GetOutboxEventsForSubscriberReturnsEmptyWhenNoMatchAsync() + { + var subA = new TestSubscriber("sub-a"); + var subB = new TestSubscriber("sub-b"); + await using var fixture = await CreateProviderAsync([subA]); + var store = fixture.Store; + + var id = UuidV7.New(); + _ = await store.CreateAsync(id, new TestDso("v"), [], SearchFieldCollection.Empty, Expiration.NoExpiration, [MakeEvent()], _ct); + + var page = await store.GetOutboxEventsForSubscriberAsync(subB.SubscriberName, 10, _ct); + page.Events.ShouldBeEmpty(); + page.HasMore.ShouldBeFalse(); + } + + [Fact] + public async Task GetOutboxEventsForSubscriberReturnsInSequenceOrderAsync() + { + var sub = new TestSubscriber("sub-ordered"); + await using var fixture = await CreateProviderAsync([sub]); + var store = fixture.Store; + + for (var i = 1; i <= 3; i++) + { + var id = UuidV7.New(); + _ = await store.CreateAsync(id, new TestDso($"v{i}"), [], SearchFieldCollection.Empty, Expiration.NoExpiration, [MakeEvent()], _ct); + } + + var page = await store.GetOutboxEventsForSubscriberAsync(sub.SubscriberName, 10, _ct); + page.Events.Count.ShouldBe(3); + page.Events.Select(e => e.SequenceNumber) + .ShouldBe(page.Events.Select(e => e.SequenceNumber).OrderBy(n => n)); + } + + private static OutboxEvent MakeEvent() => new() + { + Id = OutboxEventId.New(), + Timestamp = DateTimeOffset.UtcNow, + EventName = OutboxEventName.Create("TestEvent"), + SubjectId = UuidV7.New(), + EntityTypeName = nameof(TestDso), + EntityTypeId = (int)TestDso.DsoVersion.EntityType.Id, + Payload = "{}", + }; + + private async Task CreateProviderAsync() + => await CreateProviderAsync([new WildcardTestSubscriber()]); + + private async Task CreateProviderAsync(IOutboxSubscriber[] subscribers) => + await FixtureFactory.CreateAsync( + _ct, + services => + { + foreach (var subscriber in subscribers) + { + _ = services.AddSingleton(subscriber); + } + services.AddDsoRegistration(); + services.AddDsoRegistration(); + }); + + /// + /// Wildcard subscriber that matches all entity types and event names, used to ensure + /// outbox events are written to the store in tests. + /// + private sealed class WildcardTestSubscriber : IOutboxSubscriber + { + public SubscriberName SubscriberName => SubscriberName.Create("test-subscriber"); + public bool IsEnabled => true; + public IReadOnlySet EventNames => new HashSet(); + public IReadOnlySet EntityTypeIds => new HashSet(); + } + + /// + /// Named wildcard subscriber for multi-subscriber fanout tests. + /// + private sealed class TestSubscriber(string name) : IOutboxSubscriber + { + public SubscriberName SubscriberName => SubscriberName.Create(name); + public bool IsEnabled => true; + public IReadOnlySet EventNames => new HashSet(); + public IReadOnlySet EntityTypeIds => new HashSet(); + } +} diff --git a/storage/test/SharedIntegrationTests/StoreTryReadManyTests.cs b/storage/test/SharedIntegrationTests/StoreTryReadManyTests.cs new file mode 100644 index 000000000..81c595c6f --- /dev/null +++ b/storage/test/SharedIntegrationTests/StoreTryReadManyTests.cs @@ -0,0 +1,184 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; + +namespace Duende.Storage.IntegrationTests; + +public partial class StoreTryReadManyTests +{ + private static readonly EntityType EntityType = TestDso.DsoVersion.EntityType; + private readonly Ct _ct = TestContext.Current.CancellationToken; + + + private async Task CreateProviderAsync() => + await FixtureFactory.CreateAsync(_ct, services => + { + services.AddDsoRegistration(); + }); + + [Fact] + public async Task TryReadManyReturnsAllFoundIdsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var id1 = UuidV7.New(); + var id2 = UuidV7.New(); + var id3 = UuidV7.New(); + var value1 = new TestDso($"v1-{Guid.NewGuid()}"); + var value2 = new TestDso($"v2-{Guid.NewGuid()}"); + var value3 = new TestDso($"v3-{Guid.NewGuid()}"); + + (await store.CreateAsync(id1, value1, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + (await store.CreateAsync(id2, value2, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + (await store.CreateAsync(id3, value3, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + + var results = await store.TryReadManyAsync(EntityType, new HashSet { id1, id2, id3 }, 100, _ct); + + results.Count.ShouldBe(3); + results.ShouldAllBe(r => r.Found); + results.Select(r => r.Id).ShouldBe([id1.Value, id2.Value, id3.Value], ignoreOrder: true); + } + + [Fact] + public async Task TryReadManySkipsMissingIdsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var id1 = UuidV7.New(); + var id2 = UuidV7.New(); + var missingId = UuidV7.New(); + var value1 = new TestDso($"v1-{Guid.NewGuid()}"); + var value2 = new TestDso($"v2-{Guid.NewGuid()}"); + + (await store.CreateAsync(id1, value1, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + (await store.CreateAsync(id2, value2, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + + var results = await store.TryReadManyAsync(EntityType, new HashSet { id1, missingId, id2 }, 100, _ct); + + results.Count.ShouldBe(2); + results.ShouldAllBe(r => r.Found); + results.Select(r => r.Id).ShouldBe([id1.Value, id2.Value], ignoreOrder: true); + } + + [Fact] + public async Task TryReadManyReturnsEmptyListWhenAllIdsMissingAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var missingId1 = UuidV7.New(); + var missingId2 = UuidV7.New(); + + var results = await store.TryReadManyAsync(EntityType, new HashSet { missingId1, missingId2 }, 100, _ct); + + results.Count.ShouldBe(0); + } + + [Fact] + public async Task TryReadManyReturnsEmptyListWhenInputIsEmptyAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var results = await store.TryReadManyAsync(EntityType, new HashSet(), 100, _ct); + + results.Count.ShouldBe(0); + } + + [Fact] + public async Task TryReadManyThrowsWhenIdsExceedsMaximumAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var ids = Enumerable.Range(0, 5).Select(_ => UuidV7.New()).ToHashSet(); + + var ex = await Should.ThrowAsync( + () => store.TryReadManyAsync(EntityType, ids, 3, _ct)); + + ex.Message.ShouldContain("5"); + ex.Message.ShouldContain("3"); + } + + [Fact] + public async Task TryReadManySucceedsWhenIdsEqualsMaximumAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var id1 = UuidV7.New(); + var id2 = UuidV7.New(); + (await store.CreateAsync(id1, new TestDso($"v1-{Guid.NewGuid()}"), [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + (await store.CreateAsync(id2, new TestDso($"v2-{Guid.NewGuid()}"), [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + + // Exactly at the maximum — should not throw + var results = await store.TryReadManyAsync(EntityType, new HashSet { id1, id2 }, 2, _ct); + + results.Count.ShouldBe(2); + } + + [Fact] + public async Task TryReadManyReturnsCorrectDsoValuesAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var id = UuidV7.New(); + var value = new TestDso($"expected-{Guid.NewGuid()}"); + (await store.CreateAsync(id, value, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + + var results = await store.TryReadManyAsync(EntityType, new HashSet { id }, 100, _ct); + + results.Count.ShouldBe(1); + var result = results[0]; + result.Found.ShouldBeTrue(); + result.Id.ShouldBe(id.Value); + result.Version.ShouldBe(1); + var dso = result.Dso.ShouldBeOfType(); + dso.Value.ShouldBe(value.Value); + } + + [Fact] + public async Task TryReadManyReflectsUpdatedVersionAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var id = UuidV7.New(); + var original = new TestDso($"original-{Guid.NewGuid()}"); + var updated = new TestDso($"updated-{Guid.NewGuid()}"); + (await store.CreateAsync(id, original, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var version = (await store.TryReadAsync(EntityType, id, _ct)).Version!.Value; + (await store.UpdateAsync(id, updated, version, [], [], expiration: null, [], _ct)).ShouldBe(UpdateResult.Success); + + var results = await store.TryReadManyAsync(EntityType, new HashSet { id }, 100, _ct); + + results.Count.ShouldBe(1); + results[0].Version.ShouldBe(version + 1); + ((TestDso)results[0].Dso!).Value.ShouldBe(updated.Value); + } + + [Fact] + public async Task TryReadManyOmitsDeletedEntitiesAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var id1 = UuidV7.New(); + var id2 = UuidV7.New(); + (await store.CreateAsync(id1, new TestDso("keep"), [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + (await store.CreateAsync(id2, new TestDso("delete"), [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + + (await store.DeleteAsync(EntityType, id2, [], _ct)).ShouldBe(DeleteResult.Success); + + var results = await store.TryReadManyAsync(EntityType, new HashSet { id1, id2 }, 100, _ct); + + results.Count.ShouldBe(1); + results[0].Id.ShouldBe(id1.Value); + } +} diff --git a/storage/test/SharedIntegrationTests/StoreTtlTests.cs b/storage/test/SharedIntegrationTests/StoreTtlTests.cs new file mode 100644 index 000000000..8b8108daf --- /dev/null +++ b/storage/test/SharedIntegrationTests/StoreTtlTests.cs @@ -0,0 +1,285 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.SearchFields; +using Duende.Storage.Internal.Querying.Sorting; +using Duende.Storage.Pagination; +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.IntegrationTests; + +public partial class StoreTtlTests +{ + + + private static readonly EntityType EntityType = TestDso.DsoVersion.EntityType; + + private readonly Ct _ct = TestContext.Current.CancellationToken; + + [Fact] + public async Task CreateWithTtlReadBeforeExpiryShouldSucceedAsync() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var id = UuidV7.New(); + var value = new TestDso($"ttl-{Guid.NewGuid()}"); + + var result = await store.CreateAsync(id, value, [], [], Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct); + + result.ShouldBe(CreateResult.Success); + (await store.TryReadAsync(EntityType, id, _ct)).Found.ShouldBeTrue(); + } + + [Fact] + public async Task CreateWithTtlReadAfterExpiryShouldStillBeFoundAsync() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var id = UuidV7.New(); + var value = new TestDso($"ttl-{Guid.NewGuid()}"); + + (await store.CreateAsync(id, value, [], [], Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)) + .ShouldBe(CreateResult.Success); + + // Advance past expiration — store still returns the record (TTL is best-effort) + tp.Advance(TimeSpan.FromHours(2)); + + (await store.TryReadAsync(EntityType, id, _ct)).Found.ShouldBeTrue(); + } + + [Fact] + public async Task CreateWithPastTtlShouldBeNoopAndReturnSuccessAsync() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 12, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var id = UuidV7.New(); + var value = new TestDso($"past-{Guid.NewGuid()}"); + var pastExpiration = Expiration.AtAbsolute(tp.GetUtcNow().AddHours(-1)); + + var result = await store.CreateAsync(id, value, [], [], pastExpiration, [], _ct); + + result.ShouldBe(CreateResult.Success); + // Entity should NOT have been stored + (await store.TryReadAsync(EntityType, id, _ct)).Found.ShouldBeFalse(); + } + + [Fact] + public async Task UpdateWithTtlReadAfterExpiryShouldStillBeFoundAsync() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var id = UuidV7.New(); + var value = new TestDso($"val-{Guid.NewGuid()}"); + + (await store.CreateAsync(id, value, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var version = (await store.TryReadAsync(EntityType, id, _ct)).Version.ShouldNotBeNull(); + + var updated = new TestDso($"updated-{Guid.NewGuid()}"); + (await store.UpdateAsync(id, updated, version, [], [], + Expiration.InRelative(TimeSpan.FromMinutes(30)), [], _ct)).ShouldBe(UpdateResult.Success); + + // Before expiry — visible + (await store.TryReadAsync(EntityType, id, _ct)).Found.ShouldBeTrue(); + + // Advance past expiration — still visible (TTL is best-effort) + tp.Advance(TimeSpan.FromHours(1)); + + (await store.TryReadAsync(EntityType, id, _ct)).Found.ShouldBeTrue(); + } + + [Fact] + public async Task UpdateWithPastTtlEntityShouldStillBeFoundAsync() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 12, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var id = UuidV7.New(); + var value = new TestDso($"val-{Guid.NewGuid()}"); + + (await store.CreateAsync(id, value, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var version = (await store.TryReadAsync(EntityType, id, _ct)).Version.ShouldNotBeNull(); + + var pastExpiration = Expiration.AtAbsolute(tp.GetUtcNow().AddHours(-1)); + var result = await store.UpdateAsync(id, value, version, [], [], pastExpiration, [], _ct); + + result.ShouldBe(UpdateResult.Success); + // Entity is still returned by reads (TTL is best-effort, domain decides visibility) + (await store.TryReadAsync(EntityType, id, _ct)).Found.ShouldBeTrue(); + } + + [Fact] + public async Task UpdateWithNullExpirationShouldNotChangeExistingAsync() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var id = UuidV7.New(); + var value = new TestDso($"val-{Guid.NewGuid()}"); + + // Create with 1-hour TTL + (await store.CreateAsync(id, value, [], [], Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)) + .ShouldBe(CreateResult.Success); + var version = (await store.TryReadAsync(EntityType, id, _ct)).Version.ShouldNotBeNull(); + + // Update with null expiration (don't change) + var updated = new TestDso($"updated-{Guid.NewGuid()}"); + (await store.UpdateAsync(id, updated, version, [], [], expiration: null, [], _ct)) + .ShouldBe(UpdateResult.Success); + + // Advance past original expiration — entity still returned (TTL is best-effort) + tp.Advance(TimeSpan.FromHours(2)); + (await store.TryReadAsync(EntityType, id, _ct)).Found.ShouldBeTrue(); + } + + [Fact] + public async Task UpdateWithNoExpirationShouldClearExistingExpirationAsync() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var id = UuidV7.New(); + var value = new TestDso($"val-{Guid.NewGuid()}"); + + // Create with 1-hour TTL + (await store.CreateAsync(id, value, [], [], Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)) + .ShouldBe(CreateResult.Success); + var version = (await store.TryReadAsync(EntityType, id, _ct)).Version.ShouldNotBeNull(); + + // Update with NoExpiration (explicitly clear) + var updated = new TestDso($"updated-{Guid.NewGuid()}"); + (await store.UpdateAsync(id, updated, version, [], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(UpdateResult.Success); + + // Advance past original expiration — entity should still be visible (expiration was cleared) + tp.Advance(TimeSpan.FromHours(2)); + (await store.TryReadAsync(EntityType, id, _ct)).Found.ShouldBeTrue(); + } + + [Fact] + public async Task TryReadByKeyShouldReturnExpiredRecordsAsync() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var id = UuidV7.New(); + var value = new TestDso($"val-{Guid.NewGuid()}"); + var key = new TestJsonKeyDsk($"key-{Guid.NewGuid()}"); + + (await store.CreateAsync(id, value, [DataStorageKey.Create(key)], [], Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)) + .ShouldBe(CreateResult.Success); + + // Before expiry + (await store.TryReadAsync(EntityType, DataStorageKey.Create(key), _ct)).Found.ShouldBeTrue(); + + tp.Advance(TimeSpan.FromHours(2)); + + // After expiry — still returned (TTL is best-effort) + (await store.TryReadAsync(EntityType, DataStorageKey.Create(key), _ct)).Found.ShouldBeTrue(); + } + + [Fact] + public async Task TryReadManyShouldReturnExpiredRecordsAsync() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var id1 = UuidV7.New(); + var id2 = UuidV7.New(); + var value1 = new TestDso($"val1-{Guid.NewGuid()}"); + var value2 = new TestDso($"val2-{Guid.NewGuid()}"); + + // id1 expires in 1 hour, id2 never expires + (await store.CreateAsync(id1, value1, [], [], Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)) + .ShouldBe(CreateResult.Success); + (await store.CreateAsync(id2, value2, [], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(CreateResult.Success); + + // Before expiry — both visible + var before = await store.TryReadManyAsync(EntityType, new HashSet { id1, id2 }, 100, _ct); + before.Count.ShouldBe(2); + + tp.Advance(TimeSpan.FromHours(2)); + + // After expiry — both still returned (TTL is best-effort) + var after = await store.TryReadManyAsync(EntityType, new HashSet { id1, id2 }, 100, _ct); + after.Count.ShouldBe(2); + } + + [Fact] + public async Task QueryShouldReturnExpiredEntitiesAsync() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var queryStore = fixture.Store; + var id1 = UuidV7.New(); + var id2 = UuidV7.New(); + var value1 = new TestDso($"expires-{Guid.NewGuid()}"); + var value2 = new TestDso($"persists-{Guid.NewGuid()}"); + + // id1 expires, id2 does not + (await store.CreateAsync(id1, value1, [], [], Expiration.InRelative(TimeSpan.FromHours(1)), [], _ct)) + .ShouldBe(CreateResult.Success); + (await store.CreateAsync(id2, value2, [], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(CreateResult.Success); + + // Before expiry + var beforeResult = await queryStore.QueryAsync( + EntityType, Query.All(), SortParameter.Empty, DataRange.FromPage(1, 100), _ct); + beforeResult.Items.Count.ShouldBeGreaterThanOrEqualTo(2); + + tp.Advance(TimeSpan.FromHours(2)); + + // After expiry — both still returned (TTL is best-effort, domain decides) + var afterResult = await queryStore.QueryAsync( + EntityType, Query.All(), SortParameter.Empty, DataRange.FromPage(1, 100), _ct); + afterResult.Items.ShouldContain(item => item.Value.Value == value1.Value); + afterResult.Items.ShouldContain(item => item.Value.Value == value2.Value); + } + + [Fact] + public async Task BatchUpdateWithTtlShouldStillBeFoundAfterExpiryAsync() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 6, 1, 0, 0, 0, TimeSpan.Zero)); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var id = UuidV7.New(); + var value = new TestDso($"batch-{Guid.NewGuid()}"); + + (await store.CreateAsync(id, value, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var version = (await store.TryReadAsync(EntityType, id, _ct)).Version.ShouldNotBeNull(); + + var updatedValue = new TestDso($"batch-updated-{Guid.NewGuid()}"); + var operations = new IStoreOperation[] + { + UpdateOperation.For(id, updatedValue, version, [], SearchFieldCollection.Empty, + Expiration.InRelative(TimeSpan.FromHours(1))) + }; + + var result = await store.ExecuteBatchAsync(operations, [], _ct); + result.Success.ShouldBeTrue(); + + tp.Advance(TimeSpan.FromHours(2)); + + // After expiry — still found (TTL is best-effort) + (await store.TryReadAsync(EntityType, id, _ct)).Found.ShouldBeTrue(); + } + + + private async Task CreateProviderAsync(FakeTimeProvider tp) => + await FixtureFactory.CreateAsync(_ct, services => + { + _ = services.AddSingleton(tp); + _ = services.AddSingleton(tp); + services.AddDsoRegistration(); + services.AddDsoRegistration(); + }); +} diff --git a/storage/test/SharedIntegrationTests/Stores.cs b/storage/test/SharedIntegrationTests/Stores.cs new file mode 100644 index 000000000..e2b1658b3 --- /dev/null +++ b/storage/test/SharedIntegrationTests/Stores.cs @@ -0,0 +1,495 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.IntegrationTests; + +public partial class Stores +{ + + private static readonly EntityType EntityType = TestDso.DsoVersion.EntityType; + + private readonly Ct _ct = TestContext.Current.CancellationToken; + private readonly UuidV7 _id = UuidV7.New(); + private readonly UuidV7 _id2 = UuidV7.New(); + private readonly TestJsonKeyDsk _jKey = new($"{nameof(_jKey)}-{Guid.NewGuid()}"); + private readonly TestJsonKeyDsk _jKey2 = new($"{nameof(_jKey2)}-{Guid.NewGuid()}"); + private readonly TestDso _testValue = new($"{nameof(_testValue)}-{Guid.NewGuid()}"); + private readonly TestDso _testValue2 = new($"{nameof(_testValue2)}-{Guid.NewGuid()}"); + private readonly TestUuidV4KeyDsk _uuidV4Key = new(Guid.NewGuid()); + private readonly TestUuidV4KeyDsk _uuidV4Key2 = new(Guid.NewGuid()); + private readonly TestUuidV7KeyDsk _uuidV7Key = new(Guid.CreateVersion7()); + private readonly TestUuidV7KeyDsk _uuidV7Key2 = new(Guid.CreateVersion7()); + + private static void ShouldBeFound(StoreGetResult result, IDataStorageObject expectedDso, Guid expectedId, int expectedVersion) + { + result.Found.ShouldBeTrue(); + result.Dso.ShouldBe((object)expectedDso); + result.Id.ShouldBe(expectedId); + result.Version.ShouldBe(expectedVersion); + result.CreatedAt.ShouldNotBe(default); + result.LastUpdatedAt.ShouldNotBe(default); + } + + [Fact] + public async Task Can_create() + { + await using var fixture = await CreateProviderAsync(); + + var store = fixture.Store; + + var result = await store.CreateAsync( + _id, + _testValue, + [DataStorageKey.Create(_jKey), DataStorageKey.Create(_uuidV7Key), DataStorageKey.Create(_uuidV4Key)], + [], + Expiration.NoExpiration, + [], + _ct); + + result.ShouldBe(CreateResult.Success); + ShouldBeFound(await store.TryReadAsync(EntityType, _id, _ct), _testValue, _id.Value, 1); + ShouldBeFound(await store.TryReadAsync(EntityType, DataStorageKey.Create(_jKey), _ct), _testValue, _id.Value, 1); + ShouldBeFound(await store.TryReadAsync(EntityType, DataStorageKey.Create(_uuidV7Key), _ct), _testValue, _id.Value, 1); + ShouldBeFound(await store.TryReadAsync(EntityType, DataStorageKey.Create(_uuidV4Key), _ct), _testValue, _id.Value, 1); + } + + [Fact] + public async Task CannotCreateWhenAlreadyExistsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync(_id, _testValue, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + + var result = await store.CreateAsync(_id, _testValue, [], [], Expiration.NoExpiration, [], _ct); + + result.ShouldBe(CreateResult.AlreadyExists); + } + + [Fact] + public async Task ConcurrentCreateReturnsAlreadyExistsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var concurrencyLevel = Math.Min(Environment.ProcessorCount, 10); + var tasks = new Task[concurrencyLevel]; + for (var i = 0; i < tasks.Length; i++) + { + tasks[i] = Task.Run(() => store.CreateAsync(_id, _testValue, [], [], Expiration.NoExpiration, [], _ct), _ct); + } + + var results = await Task.WhenAll(tasks); + + var successCount = results.Count(r => r == CreateResult.Success); + var alreadyExistsCount = results.Count(r => r == CreateResult.AlreadyExists); + + successCount.ShouldBe(1); + alreadyExistsCount.ShouldBe(concurrencyLevel - 1); + + } + + [Fact] + public async Task CannotCreateWhenJsonKeyAlreadyExistsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync(_id, _testValue, [DataStorageKey.Create(_jKey)], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(CreateResult.Success); + + var result = await store.CreateAsync(_id2, _testValue, [DataStorageKey.Create(_jKey)], [], Expiration.NoExpiration, [], _ct); + + result.ShouldBe(CreateResult.KeyConflict); + } + + [Fact] + public async Task CannotCreateWhenUuidV7KeyAlreadyExistsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync(_id, _testValue, [DataStorageKey.Create(_uuidV7Key)], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(CreateResult.Success); + + var result = await store.CreateAsync(_id2, _testValue, [DataStorageKey.Create(_uuidV7Key)], [], Expiration.NoExpiration, [], _ct); + + result.ShouldBe(CreateResult.KeyConflict); + } + + [Fact] + public async Task CannotCreateWhenUuidV4KeyAlreadyExistsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync(_id, _testValue, [DataStorageKey.Create(_uuidV4Key)], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(CreateResult.Success); + + var result = await store.CreateAsync(_id2, _testValue, [DataStorageKey.Create(_uuidV4Key)], [], Expiration.NoExpiration, [], _ct); + + result.ShouldBe(CreateResult.KeyConflict); + } + + [Fact] + public async Task Can_update() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync( + _id, + _testValue, + [DataStorageKey.Create(_jKey), DataStorageKey.Create(_uuidV7Key), DataStorageKey.Create(_uuidV4Key)], + [], + Expiration.NoExpiration, + [], + _ct)).ShouldBe(CreateResult.Success); + var valueVersion = (await store.TryReadAsync(EntityType, _id, _ct)).Version.ShouldNotBeNull(); + + var result = await store.UpdateAsync( + _id, + _testValue2, + valueVersion, + [DataStorageKey.Create(_jKey2), DataStorageKey.Create(_uuidV7Key2), DataStorageKey.Create(_uuidV4Key2)], + [], + expiration: null, + [], + _ct); + + result.ShouldBe(UpdateResult.Success); + ShouldBeFound(await store.TryReadAsync(EntityType, _id, _ct), _testValue2, _id.Value, valueVersion + 1); + ShouldBeFound(await store.TryReadAsync(EntityType, DataStorageKey.Create(_jKey2), _ct), _testValue2, _id.Value, valueVersion + 1); + ShouldBeFound(await store.TryReadAsync(EntityType, DataStorageKey.Create(_uuidV7Key2), _ct), _testValue2, _id.Value, valueVersion + 1); + ShouldBeFound(await store.TryReadAsync(EntityType, DataStorageKey.Create(_uuidV4Key2), _ct), _testValue2, _id.Value, valueVersion + 1); + } + + [Fact] + public async Task CannotUpdateWhenDoesNotExistAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var result = await store.UpdateAsync(_id, _testValue2, 1, [], [], expiration: null, [], _ct); + + result.ShouldBe(UpdateResult.DoesNotExist); + } + + [Fact] + public async Task CannotUpdateWithUnexpectedVersionAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync(_id, _testValue, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var valueVersion = (await store.TryReadAsync(EntityType, _id, _ct)).Version.ShouldNotBeNull(); + (await store.UpdateAsync(_id, _testValue, valueVersion, [], [], expiration: null, [], _ct)) + .ShouldBe(UpdateResult.Success); + + var result = await store.UpdateAsync(_id, _testValue, valueVersion, [], [], expiration: null, [], _ct); + + result.ShouldBe(UpdateResult.UnexpectedVersion); + } + + [Fact] + public async Task ConcurrentUpdateReturnsUnexpectedVersionAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync(_id, _testValue, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var valueVersion = (await store.TryReadAsync(EntityType, _id, _ct)).Version.ShouldNotBeNull(); + + var concurrencyLevel = Math.Min(Environment.ProcessorCount, 10); + var tasks = new Task[concurrencyLevel]; + for (var i = 0; i < tasks.Length; i++) + { + tasks[i] = Task.Run(() => store.UpdateAsync(_id, _testValue2, valueVersion, [], [], expiration: null, [], _ct), _ct); + } + + var results = await Task.WhenAll(tasks); + + var successCount = results.Count(r => r == UpdateResult.Success); + var unexpectedVersionCount = results.Count(r => r == UpdateResult.UnexpectedVersion); + + successCount.ShouldBe(1); + unexpectedVersionCount.ShouldBe(concurrencyLevel - 1); + } + + [Fact] + public async Task WhenKeyConflictJsonKeysAreNotUpdatedAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync(_id, _testValue, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var valueVersion = (await store.TryReadAsync(EntityType, _id, _ct)).Version.ShouldNotBeNull(); + (await store.UpdateAsync(_id, _testValue, valueVersion, [], [], expiration: null, [], _ct)) + .ShouldBe(UpdateResult.Success); + + var result = await store.UpdateAsync(_id, _testValue, valueVersion, [DataStorageKey.Create(_jKey)], [], expiration: null, [], _ct); + + result.ShouldBe(UpdateResult.UnexpectedVersion); + + var getResult = await store.TryReadAsync(EntityType, DataStorageKey.Create(_jKey), _ct); + getResult.Found.ShouldBe(false); + } + + [Fact] + public async Task CannotUpdateWhenJsonKeyAlreadyExistsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync(_id, _testValue, [DataStorageKey.Create(_jKey)], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(CreateResult.Success); + var valueVersion = (await store.TryReadAsync(EntityType, _id, _ct)).Version.ShouldNotBeNull(); + (await store.CreateAsync(_id2, _testValue2, [DataStorageKey.Create(_jKey2)], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(CreateResult.Success); + + var result = await store.UpdateAsync(_id, _testValue, valueVersion, [DataStorageKey.Create(_jKey2)], [], expiration: null, [], _ct); + + result.ShouldBe(UpdateResult.KeyConflict); + } + + [Fact] + public async Task CannotUpdateWhenUuidV7KeyAlreadyExistsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync(_id, _testValue, [DataStorageKey.Create(_uuidV7Key)], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(CreateResult.Success); + var valueVersion = (await store.TryReadAsync(EntityType, _id, _ct)).Version.ShouldNotBeNull(); + (await store.CreateAsync(_id2, _testValue2, [DataStorageKey.Create(_uuidV7Key2)], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(CreateResult.Success); + + var result = await store.UpdateAsync(_id, _testValue, valueVersion, [DataStorageKey.Create(_uuidV7Key2)], [], expiration: null, [], _ct); + + result.ShouldBe(UpdateResult.KeyConflict); + } + + [Fact] + public async Task CannotUpdateWhenUuidV4KeyAlreadyExistsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync(_id, _testValue, [DataStorageKey.Create(_uuidV4Key)], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(CreateResult.Success); + var valueVersion = (await store.TryReadAsync(EntityType, _id, _ct)).Version.ShouldNotBeNull(); + (await store.CreateAsync(_id2, _testValue2, [DataStorageKey.Create(_uuidV4Key2)], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(CreateResult.Success); + + var result = await store.UpdateAsync(_id, _testValue, valueVersion, [DataStorageKey.Create(_uuidV4Key2)], [], expiration: null, [], _ct); + + result.ShouldBe(UpdateResult.KeyConflict); + } + + [Fact] + public async Task Can_delete_by_id() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync( + _id, + _testValue, + [DataStorageKey.Create(_jKey), DataStorageKey.Create(_uuidV7Key), DataStorageKey.Create(_uuidV4Key)], + [], + Expiration.NoExpiration, + [], + _ct)).ShouldBe(CreateResult.Success); + + var result = await store.DeleteAsync(EntityType, _id, [], _ct); + + result.ShouldBe(DeleteResult.Success); + (await store.TryReadAsync(EntityType, _id, _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(_jKey), _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(_uuidV7Key), _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(_uuidV4Key), _ct)).Found.ShouldBeFalse(); + } + + [Fact] + public async Task Can_delete_by_json_key() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync( + _id, + _testValue, + [DataStorageKey.Create(_jKey), DataStorageKey.Create(_uuidV7Key), DataStorageKey.Create(_uuidV4Key)], + [], + Expiration.NoExpiration, + [], + _ct)).ShouldBe(CreateResult.Success); + + var result = await store.DeleteAsync(EntityType, DataStorageKey.Create(_jKey), [], _ct); + + result.ShouldBe(DeleteResult.Success); + (await store.TryReadAsync(EntityType, _id, _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(_jKey), _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(_uuidV7Key), _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(_uuidV4Key), _ct)).Found.ShouldBeFalse(); + } + + [Fact] + public async Task Can_delete_by_UuidV7_key() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync( + _id, + _testValue, + [DataStorageKey.Create(_jKey), DataStorageKey.Create(_uuidV7Key), DataStorageKey.Create(_uuidV4Key)], + [], + Expiration.NoExpiration, + [], + _ct)).ShouldBe(CreateResult.Success); + + var result = await store.DeleteAsync(EntityType, DataStorageKey.Create(_uuidV7Key), [], _ct); + + result.ShouldBe(DeleteResult.Success); + (await store.TryReadAsync(EntityType, _id, _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(_jKey), _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(_uuidV7Key), _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(_uuidV4Key), _ct)).Found.ShouldBeFalse(); + } + + [Fact] + public async Task Can_delete_by_UuidV4_key() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync( + _id, + _testValue, + [DataStorageKey.Create(_jKey), DataStorageKey.Create(_uuidV7Key), DataStorageKey.Create(_uuidV4Key)], + [], + Expiration.NoExpiration, + [], + _ct)).ShouldBe(CreateResult.Success); + + var result = await store.DeleteAsync(EntityType, DataStorageKey.Create(_uuidV4Key), [], _ct); + + result.ShouldBe(DeleteResult.Success); + (await store.TryReadAsync(EntityType, _id, _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(_jKey), _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(_uuidV7Key), _ct)).Found.ShouldBeFalse(); + (await store.TryReadAsync(EntityType, DataStorageKey.Create(_uuidV4Key), _ct)).Found.ShouldBeFalse(); + } + + [Fact] + public async Task TryReadManyReturnsAllExistingEntitiesAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync(_id, _testValue, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + (await store.CreateAsync(_id2, _testValue2, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + + var results = await store.TryReadManyAsync(EntityType, new HashSet { _id, _id2 }, 100, _ct); + results.ShouldContain(r => r.Found && r.Id == _id2.Value && r.Dso.Equals(_testValue2)); + } + + [Fact] + public async Task TryReadManySkipsMissingIdsAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + (await store.CreateAsync(_id, _testValue, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var missingId = UuidV7.New(); + + var results = await store.TryReadManyAsync(EntityType, new HashSet { _id, missingId }, 100, _ct); + + results.Count.ShouldBe(1); + results.ShouldContain(r => r.Found && r.Id == _id.Value); + } + + [Fact] + public async Task TryReadManyReturnsEmptyListWhenNoIdsExistAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var missingId1 = UuidV7.New(); + var missingId2 = UuidV7.New(); + + var results = await store.TryReadManyAsync(EntityType, new HashSet { missingId1, missingId2 }, 100, _ct); + + results.ShouldBeEmpty(); + } + + [Fact] + public async Task TryReadManyReturnsEmptyListForEmptyInputAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + + var results = await store.TryReadManyAsync(EntityType, new HashSet(), 100, _ct); + + results.ShouldBeEmpty(); + } + + [Fact] + public async Task TryReadManyThrowsWhenExceedingMaximumAsync() + { + await using var fixture = await CreateProviderAsync(); + var store = fixture.Store; + var ids = new HashSet { _id, _id2 }; + + _ = await Should.ThrowAsync( + () => store.TryReadManyAsync(EntityType, ids, 1, _ct)); + } + + [Fact] + public async Task TryReadManyDoesNotReturnEntitiesFromDifferentEntityTypeAsync() + { + await using var fixture = await FixtureFactory.CreateAsync(_ct, services => + { + services.AddDsoRegistration(); + services.AddDsoRegistration(); + }); + var store = fixture.Store; + var dso2 = new TestDso2($"value-{Guid.NewGuid()}"); + (await store.CreateAsync(_id, _testValue, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + (await store.CreateAsync(_id2, dso2, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + + var results = await store.TryReadManyAsync(EntityType, new HashSet { _id, _id2 }, 100, _ct); + + results.Count.ShouldBe(1); + results.ShouldContain(r => r.Found && r.Id == _id.Value); + } + + private async Task CreateProviderAsync() => + await FixtureFactory.CreateAsync(_ct, services => + { + services.AddDsoRegistration(); + }); + + private async Task CreateProviderAsync(FakeTimeProvider tp) => + await FixtureFactory.CreateAsync(_ct, services => + { + _ = services.AddSingleton(tp); + _ = services.AddSingleton(tp); + services.AddDsoRegistration(); + }); + + [Fact] + public async Task Created_is_stable_and_LastUpdated_advances_on_update() + { + var createTime = new DateTimeOffset(2025, 3, 1, 10, 0, 0, TimeSpan.Zero); + var updateTime = new DateTimeOffset(2025, 3, 1, 12, 0, 0, TimeSpan.Zero); + var tp = new FakeTimeProvider(createTime); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + + // Create entity at createTime + (await store.CreateAsync(_id, _testValue, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(CreateResult.Success); + var afterCreate = await store.TryReadAsync(EntityType, _id, _ct); + afterCreate.Found.ShouldBeTrue(); + afterCreate.CreatedAt.ShouldBe(createTime); + afterCreate.LastUpdatedAt.ShouldBe(createTime); + + // Advance time and update + tp.SetUtcNow(updateTime); + var updatedDso = new TestDso($"updated-{Guid.NewGuid()}"); + (await store.UpdateAsync(_id, updatedDso, afterCreate.Version!.Value, [], [], Expiration.NoExpiration, [], _ct)).ShouldBe(UpdateResult.Success); + + var afterUpdate = await store.TryReadAsync(EntityType, _id, _ct); + afterUpdate.Found.ShouldBeTrue(); + afterUpdate.CreatedAt.ShouldBe(createTime); + afterUpdate.LastUpdatedAt.ShouldBe(updateTime); + } +} diff --git a/storage/test/SharedIntegrationTests/SystemTimestampQueryTests.cs b/storage/test/SharedIntegrationTests/SystemTimestampQueryTests.cs new file mode 100644 index 000000000..8042950d4 --- /dev/null +++ b/storage/test/SharedIntegrationTests/SystemTimestampQueryTests.cs @@ -0,0 +1,385 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Duende.Storage.Internal.Builder; +using Duende.Storage.Internal.Operations; +using Duende.Storage.Internal.Querying; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Pagination; +using Microsoft.Extensions.DependencyInjection; +using SortDirection = Duende.Storage.Querying.SortDirection; +using SortParameter = Duende.Storage.Internal.Querying.Sorting.SortParameter; + +namespace Duende.Storage.IntegrationTests; + +/// +/// Cross-store integration tests for filtering and sorting by the system timestamp +/// fields (created, last_updated). These fields bypass the search_values EAV path +/// and use entity-level columns directly. +/// +public partial class SystemTimestampQueryTests +{ + + private readonly EntityType _testEntityType = new(3, "TestEntity"); + private readonly Ct _ct = TestContext.Current.CancellationToken; + + private async Task CreateProviderAsync(FakeTimeProvider tp) => + await FixtureFactory.CreateAsync(_ct, services => + { + _ = services.AddSingleton(tp); + _ = services.AddSingleton(tp); + services.AddDsoRegistration(); + }); + + private static async Task CreateEntityAsync(IStore store, string name, Ct ct) + { + var id = UuidV7.New(); + var dso = new TestEntityDso { Name = name }; + (await store.CreateAsync(id, dso, [], [], Expiration.NoExpiration, [], ct)).ShouldBe(CreateResult.Success); + return id; + } + + // ── Filtering by created ──────────────────────────────────────────── + + [Fact] + public async Task Filter_by_created_gt_returns_entities_created_after_cutoff() + { + // Arrange + var jan = new DateTimeOffset(2025, 1, 15, 0, 0, 0, TimeSpan.Zero); + var jun = new DateTimeOffset(2025, 6, 15, 0, 0, 0, TimeSpan.Zero); + var dec = new DateTimeOffset(2025, 12, 15, 0, 0, 0, TimeSpan.Zero); + + var tp = new FakeTimeProvider(jan); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var queryStore = fixture.Store; + + _ = await CreateEntityAsync(store, "Jan", _ct); + tp.SetUtcNow(jun); + _ = await CreateEntityAsync(store, "Jun", _ct); + tp.SetUtcNow(dec); + _ = await CreateEntityAsync(store, "Dec", _ct); + + var filter = SystemFields.CreatedAtField.GreaterThan(jun); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await queryStore.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Dec"); + } + + [Fact] + public async Task Filter_by_created_between_returns_entities_in_range() + { + // Arrange + var jan = new DateTimeOffset(2025, 1, 15, 0, 0, 0, TimeSpan.Zero); + var mar = new DateTimeOffset(2025, 3, 15, 0, 0, 0, TimeSpan.Zero); + var sep = new DateTimeOffset(2025, 9, 15, 0, 0, 0, TimeSpan.Zero); + var dec = new DateTimeOffset(2025, 12, 15, 0, 0, 0, TimeSpan.Zero); + + var feb = new DateTimeOffset(2025, 2, 1, 0, 0, 0, TimeSpan.Zero); + var oct = new DateTimeOffset(2025, 10, 1, 0, 0, 0, TimeSpan.Zero); + + var tp = new FakeTimeProvider(jan); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var queryStore = fixture.Store; + + _ = await CreateEntityAsync(store, "Jan", _ct); + tp.SetUtcNow(mar); + _ = await CreateEntityAsync(store, "Mar", _ct); + tp.SetUtcNow(sep); + _ = await CreateEntityAsync(store, "Sep", _ct); + tp.SetUtcNow(dec); + _ = await CreateEntityAsync(store, "Dec", _ct); + + var filter = SystemFields.CreatedAtField.Between(feb, oct); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await queryStore.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(2); + result.Items.ShouldContain(x => x.Value.Name == "Mar"); + result.Items.ShouldContain(x => x.Value.Name == "Sep"); + } + + [Fact] + public async Task Filter_by_created_eq_returns_exact_match() + { + // Arrange + var t1 = new DateTimeOffset(2025, 1, 15, 0, 0, 0, TimeSpan.Zero); + var t2 = new DateTimeOffset(2025, 6, 15, 0, 0, 0, TimeSpan.Zero); + var t3 = new DateTimeOffset(2025, 12, 15, 0, 0, 0, TimeSpan.Zero); + + var tp = new FakeTimeProvider(t1); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var queryStore = fixture.Store; + + _ = await CreateEntityAsync(store, "First", _ct); + tp.SetUtcNow(t2); + _ = await CreateEntityAsync(store, "Second", _ct); + tp.SetUtcNow(t3); + _ = await CreateEntityAsync(store, "Third", _ct); + + var filter = SystemFields.CreatedAtField.Equals(t2); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await queryStore.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("Second"); + } + + // ── Filtering by last_updated ─────────────────────────────────────── + + [Fact] + public async Task Filter_by_last_updated_gt_returns_recently_updated_entities() + { + // Arrange + var t1 = new DateTimeOffset(2025, 3, 1, 10, 0, 0, TimeSpan.Zero); + var midpoint = new DateTimeOffset(2025, 3, 1, 11, 0, 0, TimeSpan.Zero); + var t2 = new DateTimeOffset(2025, 3, 1, 12, 0, 0, TimeSpan.Zero); + + var tp = new FakeTimeProvider(t1); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var queryStore = fixture.Store; + + var idA = await CreateEntityAsync(store, "EntityA", _ct); + var idB = await CreateEntityAsync(store, "EntityB", _ct); + + // Advance clock and update only EntityB + tp.SetUtcNow(t2); + var readResult = await store.TryReadAsync(_testEntityType, idB, _ct); + readResult.Found.ShouldBeTrue(); + var updatedDso = new TestEntityDso { Name = "EntityB-updated" }; + (await store.UpdateAsync(idB, updatedDso, readResult.Version!.Value, [], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(UpdateResult.Success); + + var filter = SystemFields.LastUpdatedAtField.GreaterThan(midpoint); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await queryStore.QueryAsync(_testEntityType, filter, SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + result.Items[0].Value.Name.ShouldBe("EntityB-updated"); + } + + // ── Sorting by created ────────────────────────────────────────────── + + [Fact] + public async Task Sort_by_created_ascending_returns_oldest_first() + { + // Arrange — create in chronological order but with non-alphabetical names + var t1 = new DateTimeOffset(2025, 1, 15, 0, 0, 0, TimeSpan.Zero); + var t2 = new DateTimeOffset(2025, 6, 15, 0, 0, 0, TimeSpan.Zero); + var t3 = new DateTimeOffset(2025, 12, 15, 0, 0, 0, TimeSpan.Zero); + + var tp = new FakeTimeProvider(t1); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var queryStore = fixture.Store; + + _ = await CreateEntityAsync(store, "Charlie", _ct); + tp.SetUtcNow(t2); + _ = await CreateEntityAsync(store, "Alpha", _ct); + tp.SetUtcNow(t3); + _ = await CreateEntityAsync(store, "Bravo", _ct); + + var sort = new SortParameter(SystemFields.CreatedAtField); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await queryStore.QueryAsync(_testEntityType, Query.All(), sort, page, Ct.None); + + // Assert — oldest first: Charlie(t1), Alpha(t2), Bravo(t3) + result.Items.Count.ShouldBe(3); + result.Items[0].Value.Name.ShouldBe("Charlie"); + result.Items[1].Value.Name.ShouldBe("Alpha"); + result.Items[2].Value.Name.ShouldBe("Bravo"); + } + + [Fact] + public async Task Sort_by_created_descending_returns_newest_first() + { + // Arrange — create in chronological order but with non-alphabetical names + var t1 = new DateTimeOffset(2025, 1, 15, 0, 0, 0, TimeSpan.Zero); + var t2 = new DateTimeOffset(2025, 6, 15, 0, 0, 0, TimeSpan.Zero); + var t3 = new DateTimeOffset(2025, 12, 15, 0, 0, 0, TimeSpan.Zero); + + var tp = new FakeTimeProvider(t1); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var queryStore = fixture.Store; + + _ = await CreateEntityAsync(store, "Charlie", _ct); + tp.SetUtcNow(t2); + _ = await CreateEntityAsync(store, "Alpha", _ct); + tp.SetUtcNow(t3); + _ = await CreateEntityAsync(store, "Bravo", _ct); + + var sort = new SortParameter(SystemFields.CreatedAtField, SortDirection.Descending); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await queryStore.QueryAsync(_testEntityType, Query.All(), sort, page, Ct.None); + + // Assert — newest first: Bravo(t3), Alpha(t2), Charlie(t1) + result.Items.Count.ShouldBe(3); + result.Items[0].Value.Name.ShouldBe("Bravo"); + result.Items[1].Value.Name.ShouldBe("Alpha"); + result.Items[2].Value.Name.ShouldBe("Charlie"); + } + + // ── Sorting by last_updated ───────────────────────────────────────── + + [Fact] + public async Task Sort_by_last_updated_ascending_returns_least_recently_updated_first() + { + // Arrange: create 3 entities at t0, then update them in order C, A, B + var t0 = new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero); + var t1 = new DateTimeOffset(2025, 2, 1, 0, 0, 0, TimeSpan.Zero); + var t2 = new DateTimeOffset(2025, 3, 1, 0, 0, 0, TimeSpan.Zero); + var t3 = new DateTimeOffset(2025, 4, 1, 0, 0, 0, TimeSpan.Zero); + + var tp = new FakeTimeProvider(t0); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var queryStore = fixture.Store; + + var idA = await CreateEntityAsync(store, "A", _ct); + var idB = await CreateEntityAsync(store, "B", _ct); + var idC = await CreateEntityAsync(store, "C", _ct); + + // Update C at t1 + tp.SetUtcNow(t1); + var readC = await store.TryReadAsync(_testEntityType, idC, _ct); + (await store.UpdateAsync(idC, new TestEntityDso { Name = "C" }, readC.Version!.Value, [], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(UpdateResult.Success); + + // Update A at t2 + tp.SetUtcNow(t2); + var readA = await store.TryReadAsync(_testEntityType, idA, _ct); + (await store.UpdateAsync(idA, new TestEntityDso { Name = "A" }, readA.Version!.Value, [], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(UpdateResult.Success); + + // Update B at t3 + tp.SetUtcNow(t3); + var readB = await store.TryReadAsync(_testEntityType, idB, _ct); + (await store.UpdateAsync(idB, new TestEntityDso { Name = "B" }, readB.Version!.Value, [], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(UpdateResult.Success); + + var sort = new SortParameter(SystemFields.LastUpdatedAtField); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await queryStore.QueryAsync(_testEntityType, Query.All(), sort, page, Ct.None); + + // Assert: C(t1) < A(t2) < B(t3) + result.Items.Count.ShouldBe(3); + result.Items[0].Value.Name.ShouldBe("C"); + result.Items[1].Value.Name.ShouldBe("A"); + result.Items[2].Value.Name.ShouldBe("B"); + } + + // ── Combined filter + sort ────────────────────────────────────────── + + [Fact] + public async Task Filter_by_created_gte_and_sort_by_created_descending() + { + // Arrange + var jan = new DateTimeOffset(2025, 1, 15, 0, 0, 0, TimeSpan.Zero); + var mar = new DateTimeOffset(2025, 3, 15, 0, 0, 0, TimeSpan.Zero); + var sep = new DateTimeOffset(2025, 9, 15, 0, 0, 0, TimeSpan.Zero); + var dec = new DateTimeOffset(2025, 12, 15, 0, 0, 0, TimeSpan.Zero); + + var tp = new FakeTimeProvider(jan); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var queryStore = fixture.Store; + + _ = await CreateEntityAsync(store, "Jan", _ct); + tp.SetUtcNow(mar); + _ = await CreateEntityAsync(store, "Mar", _ct); + tp.SetUtcNow(sep); + _ = await CreateEntityAsync(store, "Sep", _ct); + tp.SetUtcNow(dec); + _ = await CreateEntityAsync(store, "Dec", _ct); + + // Filter: created >= Mar, Sort: descending + var filter = SystemFields.CreatedAtField.GreaterOrEqual(mar); + var sort = new SortParameter(SystemFields.CreatedAtField, SortDirection.Descending); + var page = DataRange.FromPage(1, 10); + + // Act + var result = await queryStore.QueryAsync(_testEntityType, filter, sort, page, Ct.None); + + // Assert: Dec, Sep, Mar (all >= mar, newest first) + result.Items.Count.ShouldBe(3); + result.Items[0].Value.Name.ShouldBe("Dec"); + result.Items[1].Value.Name.ShouldBe("Sep"); + result.Items[2].Value.Name.ShouldBe("Mar"); + } + + // ── Projection of system fields ───────────────────────────────────── + + [Fact] + public async Task Project_created_and_last_updated_via_public_alias() + { + // Arrange — use the public alias form ("created_at", "last_updated_at") to verify + // that Field.Path uppercasing doesn't break projection lookups. + var t1 = new DateTimeOffset(2025, 3, 1, 10, 0, 0, TimeSpan.Zero); + var t2 = new DateTimeOffset(2025, 3, 1, 12, 0, 0, TimeSpan.Zero); + + var tp = new FakeTimeProvider(t1); + await using var fixture = await CreateProviderAsync(tp); + var store = fixture.Store; + var queryStore = fixture.Store; + + var id = await CreateEntityAsync(store, "Entity1", _ct); + + // Update to advance last_updated + tp.SetUtcNow(t2); + var readResult = await store.TryReadAsync(_testEntityType, id, _ct); + readResult.Found.ShouldBeTrue(); + (await store.UpdateAsync(id, new TestEntityDso { Name = "Entity1-updated" }, readResult.Version!.Value, [], [], Expiration.NoExpiration, [], _ct)) + .ShouldBe(UpdateResult.Success); + + // Project using public alias names + var fields = new Field[] + { + new DateTimeField(SystemFields.CreatedAttributeName), + new DateTimeField(SystemFields.LastUpdatedAttributeName) + }; + var page = DataRange.FromPage(1, 10); + + // Act + var result = await queryStore.QueryFieldsAsync(_testEntityType, fields, Query.All(), SortParameter.Empty, page, Ct.None); + + // Assert + result.Items.Count.ShouldBe(1); + var projected = result.Items[0]; + var createdKey = fields[0].Path; // uppercased by Field.Path + var lastUpdatedKey = fields[1].Path; + + projected.Fields.ShouldContainKey(createdKey); + projected.Fields.ShouldContainKey(lastUpdatedKey); + + var createdValue = projected.Fields[createdKey].ShouldBeOfType(); + var lastUpdatedValue = projected.Fields[lastUpdatedKey].ShouldBeOfType(); + + createdValue.ShouldBe(t1); + lastUpdatedValue.ShouldBe(t2); + } +} diff --git a/storage/test/SharedIntegrationTests/TestDsos.cs b/storage/test/SharedIntegrationTests/TestDsos.cs new file mode 100644 index 000000000..af7e0de6d --- /dev/null +++ b/storage/test/SharedIntegrationTests/TestDsos.cs @@ -0,0 +1,174 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; + +namespace Duende.Storage.IntegrationTests; + +public sealed record TestDso(string Value) : IDataStorageObject +{ + public static DataStorageObjectVersion DsoVersion { get; } = new(new EntityType(99, nameof(TestDso)), 1); +} + +public enum TestKeys +{ + JsonKey = 1, + UuidV4Key = 2, + UuidV7Key = 3 +} + +public sealed record TestJsonKeyDsk(string JsonValue) : IDataStorageKey +{ + public static DataStorageKeyVersion DskVersion { get; } = new(TestKeys.JsonKey, 1); +} + +public sealed record TestUuidV4KeyDsk(Guid Value) : IGuidDataStorageKey +{ + public static DataStorageKeyVersion DskVersion { get; } = new(TestKeys.UuidV4Key, 1); +} + +public sealed record TestUuidV7KeyDsk(Guid Value) : IGuidDataStorageKey +{ + public static DataStorageKeyVersion DskVersion { get; } = new(TestKeys.UuidV7Key, 1); +} + +/// +/// Test DSO for batch operations testing with a different entity type. +/// +public sealed record TestDso2(string Value) : IDataStorageObject +{ + public static DataStorageObjectVersion DsoVersion { get; } = new(new EntityType(100, nameof(TestDso2)), 1); +} + +/// +/// Test DSO representing a user with email addresses. +/// +public sealed record TestUserDso : IDataStorageObject +{ + public static DataStorageObjectVersion DsoVersion { get; } = new(new EntityType(2, "UserEntity"), 1); + + public string Name { get; init; } = string.Empty; + + public EmailAddress[]? Emails { get; init; } +} + +/// +/// Email address type for testing array filters. +/// +public sealed record EmailAddress +{ + public string Type { get; init; } = string.Empty; + + public string Value { get; init; } = string.Empty; + + public DateTimeOffset? CreatedAt { get; init; } + + public int? Priority { get; init; } +} + +/// +/// Test DSO for basic expression tests with various field types. +/// +public sealed record TestEntityDso : IDataStorageObject +{ + public static DataStorageObjectVersion DsoVersion { get; } = new(new EntityType(3, "TestEntity"), 1); + + public string Name { get; init; } = string.Empty; + public int? Score { get; init; } + public decimal? Price { get; init; } + public DateTimeOffset? CreatedAt { get; init; } + public DateTimeOffset? LastLogin { get; init; } + public bool? IsActive { get; init; } + public string? Status { get; init; } +} + +/// +/// Test DSO for sorting tests with various sortable field types. +/// +public sealed record TestSortDso : IDataStorageObject +{ + public static DataStorageObjectVersion DsoVersion { get; } = new(new EntityType(4, "SortTestEntity"), 1); + + public string Name { get; init; } = string.Empty; + public int? Rank { get; init; } + public decimal? Rating { get; init; } + public DateTimeOffset? Timestamp { get; init; } + public string? Category { get; init; } +} + +/// +/// Test DSO for cursor pagination tests. +/// +public sealed record TestCursorDso : IDataStorageObject +{ + public required string Name { get; init; } + public required int Rank { get; init; } + public required DateTimeOffset CreatedAt { get; init; } + public required bool IsActive { get; init; } + + public static DataStorageObjectVersion DsoVersion { get; } = new(new EntityType(5, nameof(TestCursorDso)), 1); +} + +/// +/// Test DSO for offset-based pagination tests. +/// +public sealed record TestPageDso : IDataStorageObject +{ + public required string Name { get; init; } + public required int Rank { get; init; } + public required DateTimeOffset CreatedAt { get; init; } + public required bool IsActive { get; init; } + + public static DataStorageObjectVersion DsoVersion { get; } = new(new EntityType(6, nameof(TestPageDso)), 1); +} + +/// +/// Test DSO for Guid and ExactMatch field tests. +/// +public sealed record TestGuidEntityDso : IDataStorageObject +{ + public static DataStorageObjectVersion DsoVersion { get; } = new(new EntityType(6, "GuidTestEntity"), 1); + + public string Name { get; init; } = string.Empty; + public Guid? ResourceId { get; init; } + public string? ApiKey { get; init; } + public string? Tag { get; init; } +} + +// Link-related DSOs for link query tests + +public sealed record UserDso(string Name) : IDataStorageObject +{ + public static DataStorageObjectVersion DsoVersion { get; } = new(new EntityType(95, nameof(UserDso)), 1); +} + +public sealed record RoleDso(string Name) : IDataStorageObject +{ + public static DataStorageObjectVersion DsoVersion { get; } = new(new EntityType(96, nameof(RoleDso)), 1); +} + +public sealed record GroupDso(string Name) : IDataStorageObject +{ + public static DataStorageObjectVersion DsoVersion { get; } = new(new EntityType(97, nameof(GroupDso)), 1); +} + +/// +/// Shared link definitions used by link operation and outbox tests. +/// Uses TestDso (entity type 99) as "left" and TestDso2 (entity type 100) as "right". +/// +public static class TestLinkData +{ + public static readonly LinkDefinition TestLink = new() + { + Left = TestDso.DsoVersion.EntityType, + Right = TestDso2.DsoVersion.EntityType, + Link = LinkTypeRegistry.MembershipRole + }; + + public static readonly LinkDefinition TestLink2 = new() + { + Left = TestDso2.DsoVersion.EntityType, + Right = TestDso.DsoVersion.EntityType, + Link = LinkTypeRegistry.GroupRole + }; +} diff --git a/storage/test/Storage.MsSql.Tests/AspireFixture.cs b/storage/test/Storage.MsSql.Tests/AspireFixture.cs new file mode 100644 index 000000000..141a7bd3e --- /dev/null +++ b/storage/test/Storage.MsSql.Tests/AspireFixture.cs @@ -0,0 +1,104 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Aspire.Hosting; +using Aspire.Hosting.Testing; +using Microsoft.Data.SqlClient; + +namespace Duende.Storage.MsSql; + +public sealed class AspireFixture : IAsyncLifetime +{ + private DistributedApplication? _app; + private string? _serverConnectionString; + private string? _databaseName; + + public string ServerConnectionString { get; private set; } = null!; + public string ConnectionString { get; private set; } = null!; + + /// + /// Pool of reusable databases shared across all test classes in this collection. + /// + internal MsSqlDatabasePool Pool { get; private set; } = null!; + + public async ValueTask InitializeAsync() + { + var ct = TestContext.Current.CancellationToken; + Environment.SetEnvironmentVariable("TESTAPPHOST_RESOURCES", "sqlserver"); + var builder = await DistributedApplicationTestingBuilder.CreateAsync(ct); + _app = await builder.BuildAsync(ct); + await _app.StartAsync(ct); + + // WaitForResourceHealthyAsync can hang in CI when the persistent container + // (started by warmup) is already healthy before the test app subscribes to + // notifications. Use a timeout and fall through — the container is ready. + using (var healthCts = CancellationTokenSource.CreateLinkedTokenSource(ct)) + { + healthCts.CancelAfter(TimeSpan.FromSeconds(60)); + try + { + _ = await _app.ResourceNotifications.WaitForResourceHealthyAsync("sqlserver", healthCts.Token); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + // Timeout waiting for health notification — container is likely already healthy from warmup. + Console.WriteLine("WaitForResourceHealthyAsync timed out; proceeding with connection string."); + } + } + + _serverConnectionString = (await _app.GetConnectionStringAsync("sqlserver", ct))!; + ServerConnectionString = _serverConnectionString; + + Pool = new MsSqlDatabasePool(_serverConnectionString); + + // Create a dedicated database for MsSqlStoreTests (smoke tests that + // need a persistent connection string rather than a pooled one). + _databaseName = $"test_{Guid.NewGuid():N}"; + await using var connection = new SqlConnection(_serverConnectionString); + await connection.OpenAsync(ct); + await using var cmd = connection.CreateCommand(); + cmd.CommandText = $"CREATE DATABASE [{_databaseName}]"; + _ = await cmd.ExecuteNonQueryAsync(ct); + + var csb = new SqlConnectionStringBuilder(_serverConnectionString) + { + InitialCatalog = _databaseName + }; + ConnectionString = csb.ConnectionString; + } + + public async ValueTask DisposeAsync() + { + if (_app != null && _serverConnectionString != null && _databaseName != null) + { + // Drop all pooled databases. + if (Pool != null) + { + await Pool.DropAllAsync(); + } + + // Drop the dedicated smoke-test database. + try + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var ct = cts.Token; + await using var connection = new SqlConnection(_serverConnectionString); + await connection.OpenAsync(ct); + await using var cmd = connection.CreateCommand(); + cmd.CommandText = $""" + ALTER DATABASE [{_databaseName}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; + DROP DATABASE [{_databaseName}]; + """; + _ = await cmd.ExecuteNonQueryAsync(ct); + } +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 + { + Console.WriteLine($"Failed to drop test database {_databaseName}: {ex.Message}"); + } + + await _app.DisposeAsync(); + } + } +} diff --git a/storage/test/Storage.MsSql.Tests/Internal/SqlServerGuidConverterTests.cs b/storage/test/Storage.MsSql.Tests/Internal/SqlServerGuidConverterTests.cs new file mode 100644 index 000000000..cec270d35 --- /dev/null +++ b/storage/test/Storage.MsSql.Tests/Internal/SqlServerGuidConverterTests.cs @@ -0,0 +1,123 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Data.SqlTypes; + +namespace Duende.Storage.MsSql.Internal; + +public sealed class SqlServerGuidConverterTests +{ + [Fact] + public void Round_trip_preserves_original_value() + { + var original = Guid.CreateVersion7(); + + var sqlServer = SqlServerGuidConverter.ToSqlServer(original); + var restored = SqlServerGuidConverter.ToUuidV7(sqlServer); + + restored.ShouldBe(original); + } + + [Fact] + public void Round_trip_from_sql_server_preserves_value() + { + var original = Guid.CreateVersion7(); + var sqlServer = SqlServerGuidConverter.ToSqlServer(original); + + var restored = SqlServerGuidConverter.ToUuidV7(sqlServer); + var backToSqlServer = SqlServerGuidConverter.ToSqlServer(restored); + + backToSqlServer.ShouldBe(sqlServer); + } + + [Fact] + public void Converted_value_differs_from_original() + { + // Use a known UUIDv7 with distinct byte groups to guarantee the swap produces a different value. + var original = Guid.Parse("019d774c-58e8-78fa-b7d0-6f4c1a4ae935"); + + var sqlServer = SqlServerGuidConverter.ToSqlServer(original); + + sqlServer.ShouldNotBe(original); + } + + [Fact] + public void Chronological_order_preserved_in_sql_server_format() + { + // Generate UUIDv7 values with deterministic, increasing timestamps. + // SqlGuid implements SQL Server's actual UNIQUEIDENTIFIER comparison semantics, + // so we use it as the authoritative sort-order reference. + var baseTime = DateTimeOffset.UtcNow; + var ids = new List<(Guid Original, Guid SqlServer)>(); + for (var i = 0; i < 50; i++) + { + var timestamp = baseTime.AddMilliseconds(i); + var original = Guid.CreateVersion7(timestamp); + ids.Add((original, SqlServerGuidConverter.ToSqlServer(original))); + } + + // Verify that SqlGuid comparison preserves chronological order. + for (var i = 1; i < ids.Count; i++) + { + var previous = new SqlGuid(ids[i - 1].SqlServer); + var current = new SqlGuid(ids[i].SqlServer); + + var comparison = previous.CompareTo(current); + comparison.ShouldBeLessThan(0, + $"Expected GUID at index {i - 1} ({ids[i - 1].Original}) to sort before index {i} ({ids[i].Original}) " + + $"in SQL Server order. SqlServer values: {previous} vs {current}"); + } + } + + [Fact] + public void Multiple_round_trips_produce_same_result() + { + var original = Guid.CreateVersion7(); + + var sqlServer1 = SqlServerGuidConverter.ToSqlServer(original); + var restored1 = SqlServerGuidConverter.ToUuidV7(sqlServer1); + var sqlServer2 = SqlServerGuidConverter.ToSqlServer(restored1); + var restored2 = SqlServerGuidConverter.ToUuidV7(sqlServer2); + + restored1.ShouldBe(original); + restored2.ShouldBe(original); + sqlServer1.ShouldBe(sqlServer2); + } + + [Fact] + public void Batch_round_trip_all_values_preserved() + { + var originals = Enumerable.Range(0, 1000) + .Select(_ => Guid.CreateVersion7()) + .ToList(); + + var roundTripped = originals + .Select(SqlServerGuidConverter.ToSqlServer) + .Select(SqlServerGuidConverter.ToUuidV7) + .ToList(); + + for (var i = 0; i < originals.Count; i++) + { + roundTripped[i].ShouldBe(originals[i], $"Mismatch at index {i}"); + } + } + + [Fact] + public void Timestamp_bytes_moved_to_high_priority_position() + { + var original = Guid.CreateVersion7(); + Span originalBytes = stackalloc byte[16]; + _ = original.TryWriteBytes(originalBytes, bigEndian: true, out _); + + var sqlServer = SqlServerGuidConverter.ToSqlServer(original); + Span sqlBytes = stackalloc byte[16]; + _ = sqlServer.TryWriteBytes(sqlBytes, bigEndian: true, out _); + + // Timestamp (original bytes 0-5) should now be at bytes 10-15 + for (var i = 0; i < 6; i++) + { + sqlBytes[10 + i].ShouldBe(originalBytes[i], + $"Timestamp byte {i} not correctly placed at SQL Server position {10 + i}"); + } + } +} diff --git a/storage/test/Storage.MsSql.Tests/MigrationTests.cs b/storage/test/Storage.MsSql.Tests/MigrationTests.cs new file mode 100644 index 000000000..e0df4477e --- /dev/null +++ b/storage/test/Storage.MsSql.Tests/MigrationTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.MsSql; + +namespace Duende.Storage.IntegrationTests; + +[Collection("MsSqlIntegration")] +public partial class MigrationTests(AspireFixture fixture) +{ + private IMigrationFixtureFactory MigrationFixtureFactory { get; } = new MsSqlMigrationFixtureFactory(fixture); +} diff --git a/storage/test/Storage.MsSql.Tests/MsSqlDatabasePool.cs b/storage/test/Storage.MsSql.Tests/MsSqlDatabasePool.cs new file mode 100644 index 000000000..7209d0f4d --- /dev/null +++ b/storage/test/Storage.MsSql.Tests/MsSqlDatabasePool.cs @@ -0,0 +1,107 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Collections.Concurrent; +using Microsoft.Data.SqlClient; + +namespace Duende.Storage.MsSql; + +/// +/// Pools reusable SQL Server databases across integration tests. Instead of +/// creating and dropping a database per test, each test checks out a database, +/// runs, then returns it. On return the database tables are cleared so the +/// next test starts clean. +/// +internal sealed class MsSqlDatabasePool(string serverConnectionString) +{ + private readonly ConcurrentQueue _available = new(); + private readonly ConcurrentBag _all = new(); + + /// + /// Returns a connection string for a ready-to-use database. The caller is + /// responsible for applying the schema. Creates a new database if none are + /// available in the pool. + /// + public async Task GetConnectionStringAsync(CancellationToken ct) + { + if (_available.TryDequeue(out var connectionString)) + { + return connectionString; + } + + var dbName = $"pool_{Guid.NewGuid():N}"; + await using var connection = new SqlConnection(serverConnectionString); + await connection.OpenAsync(ct); + await using var cmd = connection.CreateCommand(); + cmd.CommandText = $"CREATE DATABASE [{dbName}]"; + _ = await cmd.ExecuteNonQueryAsync(ct); + + var csb = new SqlConnectionStringBuilder(serverConnectionString) { InitialCatalog = dbName }; + var cs = csb.ConnectionString; + _all.Add(cs); + return cs; + } + + /// + /// Returns a database to the pool after clearing all test data. + /// Deletes in FK-dependency order since SQL Server has no TRUNCATE CASCADE. + /// + public async Task ReturnAsync(string connectionString) + { + try + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + var ct = cts.Token; + await using var connection = new SqlConnection(connectionString); + await connection.OpenAsync(ct); + await using var cmd = connection.CreateCommand(); + cmd.CommandText = """ + DELETE FROM [dbo].[outbox_subscriber_queue]; + DELETE FROM [dbo].[entity_links]; + DELETE FROM [dbo].[search_values]; + DELETE FROM [dbo].[entity_keys]; + DELETE FROM [dbo].[entities]; + """; + _ = await cmd.ExecuteNonQueryAsync(ct); + _available.Enqueue(connectionString); + } +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 + { + // If cleanup fails, don't return to pool — leave it out until DropAllAsync. + Console.WriteLine($"Failed to clean pooled database; it will not be reused: {ex.Message}"); + } + } + + /// + /// Drops all databases that were created by this pool. Called during + /// test suite teardown. + /// + public async Task DropAllAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var ct = cts.Token; + foreach (var connectionString in _all) + { + var dbName = new SqlConnectionStringBuilder(connectionString).InitialCatalog; + try + { + await using var conn = new SqlConnection(serverConnectionString); + await conn.OpenAsync(ct); + await using var cmd = conn.CreateCommand(); + cmd.CommandText = $""" + ALTER DATABASE [{dbName}] SET SINGLE_USER WITH ROLLBACK IMMEDIATE; + DROP DATABASE [{dbName}]; + """; + _ = await cmd.ExecuteNonQueryAsync(ct); + } +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 + { + Console.WriteLine($"Failed to drop pooled database {dbName}: {ex.Message}"); + } + } + } +} diff --git a/storage/test/Storage.MsSql.Tests/MsSqlIntegrationGroup.cs b/storage/test/Storage.MsSql.Tests/MsSqlIntegrationGroup.cs new file mode 100644 index 000000000..6039d5d32 --- /dev/null +++ b/storage/test/Storage.MsSql.Tests/MsSqlIntegrationGroup.cs @@ -0,0 +1,7 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.MsSql; + +[CollectionDefinition("MsSqlIntegration")] +public sealed class MsSqlIntegrationGroup : ICollectionFixture; diff --git a/storage/test/Storage.MsSql.Tests/MsSqlIntegrationTests.cs b/storage/test/Storage.MsSql.Tests/MsSqlIntegrationTests.cs new file mode 100644 index 000000000..566134068 --- /dev/null +++ b/storage/test/Storage.MsSql.Tests/MsSqlIntegrationTests.cs @@ -0,0 +1,109 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.MsSql; + +namespace Duende.Storage.IntegrationTests; + +[Collection("MsSqlIntegration")] +public partial class Stores(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class StoreBatchOperations(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class StoreLinkOperations(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class StoreLinkQueryTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class StoreOutboxOperations(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class StoreTtlTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class StoreTryReadManyTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class PurgeExpiredTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class FilterTranslatorIntegrationTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class QueryStoreArrayFilterTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class QueryStoreBasicExpressionTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class QueryStoreCountTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class QueryStoreCursorPagingTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class QueryStoreGuidFieldTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class QueryStorePagingTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class QueryStoreSortTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + +[Collection("MsSqlIntegration")] +public partial class SystemTimestampQueryTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new MsSqlStoreFixtureFactory(fixture); +} + diff --git a/storage/test/Storage.MsSql.Tests/MsSqlMigrationFixtureFactory.cs b/storage/test/Storage.MsSql.Tests/MsSqlMigrationFixtureFactory.cs new file mode 100644 index 000000000..5aec58467 --- /dev/null +++ b/storage/test/Storage.MsSql.Tests/MsSqlMigrationFixtureFactory.cs @@ -0,0 +1,90 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Globalization; +using Duende.Storage.IntegrationTests; +using Duende.Storage.Internal; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.MsSql; + +internal sealed class MsSqlMigrationFixtureFactory(AspireFixture aspire) : IMigrationFixtureFactory +{ + public async Task CreateAsync(CancellationToken ct) + { + var connectionString = await aspire.Pool.GetConnectionStringAsync(ct); + var schemaName = "s_" + DateTime.Now.Ticks.ToString(CultureInfo.InvariantCulture); + + var services = new ServiceCollection(); + _ = services.AddLogging(); + _ = services.AddKeyedSingleton("migration-test", () => new SqlConnection(connectionString)); + _ = services.AddStorageInternal(storage => storage.AddMsSqlStore("migration-test", o => o.SchemaName = schemaName)); + var provider = services.BuildServiceProvider(); + + var schema = provider.GetRequiredKeyedService("migration-test"); + return new MsSqlMigrationFixture(provider, schemaName, schema, connectionString); + } +} + +internal sealed class MsSqlMigrationFixture( + ServiceProvider provider, + string schemaName, + IDatabaseSchema schema, + string connectionString) : IMigrationFixture +{ + public IDatabaseSchema Schema => schema; + + public async Task ExecuteSqlAsync(string sql, CancellationToken ct) + { + await using var connection = new SqlConnection(connectionString); + await connection.OpenAsync(ct); + await using var cmd = connection.CreateCommand(); + cmd.CommandText = sql; + _ = await cmd.ExecuteNonQueryAsync(ct); + } + + public async ValueTask DisposeAsync() + { + await provider.DisposeAsync(); + + await using var connection = new SqlConnection(connectionString); + await connection.OpenAsync(); + await using var cmd = connection.CreateCommand(); + cmd.CommandText = $""" + DECLARE @sql NVARCHAR(MAX) = N''; + + -- Drop all foreign keys + SELECT @sql += 'ALTER TABLE ' + QUOTENAME(s.name) + '.' + QUOTENAME(t.name) + ' DROP CONSTRAINT ' + QUOTENAME(fk.name) + ';' + CHAR(10) + FROM sys.foreign_keys fk + INNER JOIN sys.tables t ON fk.parent_object_id = t.object_id + INNER JOIN sys.schemas s ON t.schema_id = s.schema_id + WHERE s.name = N'{schemaName}'; + EXEC sp_executesql @sql; + + -- Drop all types + SET @sql = N''; + SELECT @sql += 'DROP TYPE ' + QUOTENAME(s.name) + '.' + QUOTENAME(t.name) + ';' + CHAR(10) + FROM sys.types t + INNER JOIN sys.schemas s ON t.schema_id = s.schema_id + WHERE s.name = N'{schemaName}' AND t.is_user_defined = 1; + EXEC sp_executesql @sql; + + -- Drop all tables + SET @sql = N''; + SELECT @sql += 'DROP TABLE ' + QUOTENAME(s.name) + '.' + QUOTENAME(t.name) + ';' + CHAR(10) + FROM sys.tables t + INNER JOIN sys.schemas s ON t.schema_id = s.schema_id + WHERE s.name = N'{schemaName}'; + EXEC sp_executesql @sql; + + -- Drop extended properties on the schema + IF EXISTS (SELECT 1 FROM sys.extended_properties WHERE class = 3 AND major_id = SCHEMA_ID(N'{schemaName}')) + EXEC sys.sp_dropextendedproperty @name = N'SchemaVersion', @level0type = N'SCHEMA', @level0name = N'{schemaName}'; + + -- Drop the schema + DROP SCHEMA IF EXISTS [{schemaName}]; + """; + _ = await cmd.ExecuteNonQueryAsync(); + } +} diff --git a/storage/test/Storage.MsSql.Tests/MsSqlStoreFixture.cs b/storage/test/Storage.MsSql.Tests/MsSqlStoreFixture.cs new file mode 100644 index 000000000..1d0311bd1 --- /dev/null +++ b/storage/test/Storage.MsSql.Tests/MsSqlStoreFixture.cs @@ -0,0 +1,23 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.IntegrationTests; +using Duende.Storage.Internal; +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.MsSql; + +internal sealed class MsSqlStoreFixture( + ServiceProvider provider, + IStore store, + MsSqlDatabasePool pool, + string connectionString) : IStoreFixture +{ + public IStore Store { get; } = store; + + public async ValueTask DisposeAsync() + { + await provider.DisposeAsync(); + await pool.ReturnAsync(connectionString); + } +} diff --git a/storage/test/Storage.MsSql.Tests/MsSqlStoreFixtureFactory.cs b/storage/test/Storage.MsSql.Tests/MsSqlStoreFixtureFactory.cs new file mode 100644 index 000000000..8939bf2b8 --- /dev/null +++ b/storage/test/Storage.MsSql.Tests/MsSqlStoreFixtureFactory.cs @@ -0,0 +1,34 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.IntegrationTests; +using Duende.Storage.Internal; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.MsSql; + +internal sealed class MsSqlStoreFixtureFactory(AspireFixture aspire) : IStoreFixtureFactory +{ + private const string ServiceKey = "test"; + + public async Task CreateAsync(Ct ct, Action? configure = null) + { + var connectionString = await aspire.Pool.GetConnectionStringAsync(ct); + + var services = new ServiceCollection(); + _ = services.AddLogging(); + configure?.Invoke(services); + _ = services.AddKeyedSingleton(ServiceKey, () => new SqlConnection(connectionString)); + _ = services.AddStorageInternal(storage => storage.AddMsSqlStore(ServiceKey, _ => { })); + var provider = services.BuildServiceProvider(); + + var schema = provider.GetRequiredKeyedService(ServiceKey); + await schema.MigrateAsync(ct); + + var pooledStore = provider.GetRequiredKeyedService(ServiceKey); + var store = pooledStore.OpenPool(1); + + return new MsSqlStoreFixture(provider, store, aspire.Pool, connectionString); + } +} diff --git a/storage/test/Storage.MsSql.Tests/MsSqlStoreTests.cs b/storage/test/Storage.MsSql.Tests/MsSqlStoreTests.cs new file mode 100644 index 000000000..f5208ab22 --- /dev/null +++ b/storage/test/Storage.MsSql.Tests/MsSqlStoreTests.cs @@ -0,0 +1,53 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Microsoft.Data.SqlClient; +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.MsSql; + +public class MsSqlStoreTests(AspireFixture fixture) : IClassFixture +{ + private readonly Ct _ct = TestContext.Current.CancellationToken; + private const string ServiceKey = "my-mssql-store"; + + private ServiceProvider CreateServiceProvider() + { + var serviceCollection = new ServiceCollection(); + _ = serviceCollection.AddLogging(); + var connectionString = fixture.ConnectionString; + _ = serviceCollection.AddKeyedSingleton(ServiceKey, () => new SqlConnection(connectionString)); + _ = serviceCollection.AddStorageInternal(storage => storage.AddMsSqlStore(ServiceKey, _ => { })); + return serviceCollection.BuildServiceProvider(); + } + + [Fact] + public void Can_resolve_store() + { + var serviceProvider = CreateServiceProvider(); + + var pooledStore = serviceProvider.GetRequiredKeyedService(ServiceKey); + + var store = pooledStore.OpenPool(1); + + _ = store.ShouldNotBeNull(); + } + + [Fact] + public async Task Can_create_schema() + { + var serviceProvider = CreateServiceProvider(); + + var pooledStore = serviceProvider.GetRequiredKeyedService(ServiceKey); + + var schemaVersionResult = await pooledStore.CheckVersionAsync(_ct); + schemaVersionResult.CurrentVersion.ShouldBe(0u); + schemaVersionResult.IsCompatible.ShouldBeFalse(); + schemaVersionResult.RequiredVersion.ShouldBe(1u); + + await pooledStore.MigrateAsync(_ct); + schemaVersionResult = await pooledStore.CheckVersionAsync(_ct); + schemaVersionResult.CurrentVersion.ShouldBe(1u); + } +} diff --git a/storage/test/Storage.MsSql.Tests/Storage.MsSql.Tests.csproj b/storage/test/Storage.MsSql.Tests/Storage.MsSql.Tests.csproj new file mode 100644 index 000000000..aa61f5417 --- /dev/null +++ b/storage/test/Storage.MsSql.Tests/Storage.MsSql.Tests.csproj @@ -0,0 +1,17 @@ + + + + Duende.Storage.MsSql + + + + + + + + + + + + + diff --git a/storage/test/Storage.PostgreSql.Tests/AspireFixture.cs b/storage/test/Storage.PostgreSql.Tests/AspireFixture.cs new file mode 100644 index 000000000..a936c0d27 --- /dev/null +++ b/storage/test/Storage.PostgreSql.Tests/AspireFixture.cs @@ -0,0 +1,104 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Aspire.Hosting; +using Aspire.Hosting.Testing; +using Npgsql; + +namespace Duende.Storage.PostgreSql; + +public sealed class AspireFixture : IAsyncLifetime +{ + private DistributedApplication? _app; + private string? _serverConnectionString; + private string? _databaseName; + + public string ServerConnectionString { get; private set; } = null!; + public string ConnectionString { get; private set; } = null!; + + /// + /// Pool of reusable databases shared across all test classes in this collection. + /// + internal PostgreSqlDatabasePool Pool { get; private set; } = null!; + + public async ValueTask InitializeAsync() + { + var ct = TestContext.Current.CancellationToken; + Environment.SetEnvironmentVariable("TESTAPPHOST_RESOURCES", "postgresql"); + var builder = await DistributedApplicationTestingBuilder.CreateAsync(ct); + _app = await builder.BuildAsync(ct); + await _app.StartAsync(ct); + + // WaitForResourceHealthyAsync can hang in CI when the persistent container + // (started by warmup) is already healthy before the test app subscribes to + // notifications. Use a timeout and fall through — the container is ready. + using (var healthCts = CancellationTokenSource.CreateLinkedTokenSource(ct)) + { + healthCts.CancelAfter(TimeSpan.FromSeconds(60)); + try + { + _ = await _app.ResourceNotifications.WaitForResourceHealthyAsync("postgresql", healthCts.Token); + } + catch (OperationCanceledException) when (!ct.IsCancellationRequested) + { + // Timeout waiting for health notification — container is likely already healthy from warmup. + Console.WriteLine("WaitForResourceHealthyAsync timed out; proceeding with connection string."); + } + } + + _serverConnectionString = (await _app.GetConnectionStringAsync("postgresql", ct))!; + ServerConnectionString = _serverConnectionString; + + Pool = new PostgreSqlDatabasePool(_serverConnectionString); + + // Create a dedicated database for PostgreSqlStoreTests (smoke tests that + // need a persistent connection string rather than a pooled one). + _databaseName = $"test_{Guid.NewGuid():N}"; + await using var dataSource = NpgsqlDataSource.Create(_serverConnectionString); + await using var cmd = dataSource.CreateCommand($"CREATE DATABASE \"{_databaseName}\""); + _ = await cmd.ExecuteNonQueryAsync(ct); + + var csb = new NpgsqlConnectionStringBuilder(_serverConnectionString) + { + Database = _databaseName + }; + ConnectionString = csb.ConnectionString; + } + + public async ValueTask DisposeAsync() + { + if (_app != null && _serverConnectionString != null && _databaseName != null) + { + // Drop all pooled databases. + if (Pool != null) + { + await Pool.DropAllAsync(); + } + + // Drop the dedicated smoke-test database. + try + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var ct = cts.Token; + await using var dataSource = NpgsqlDataSource.Create(_serverConnectionString); + await using var terminateCmd = dataSource.CreateCommand( + """ + SELECT pg_terminate_backend(pid) FROM pg_stat_activity + WHERE datname = $1 AND pid <> pg_backend_pid() + """); + _ = terminateCmd.Parameters.Add(new NpgsqlParameter { Value = _databaseName }); + _ = await terminateCmd.ExecuteNonQueryAsync(ct); + await using var dropCmd = dataSource.CreateCommand($"DROP DATABASE \"{_databaseName}\""); + _ = await dropCmd.ExecuteNonQueryAsync(ct); + } +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 + { + Console.WriteLine($"Failed to drop test database {_databaseName}: {ex.Message}"); + } + + await _app.DisposeAsync(); + } + } +} diff --git a/storage/test/Storage.PostgreSql.Tests/MigrationTests.cs b/storage/test/Storage.PostgreSql.Tests/MigrationTests.cs new file mode 100644 index 000000000..739aa551c --- /dev/null +++ b/storage/test/Storage.PostgreSql.Tests/MigrationTests.cs @@ -0,0 +1,12 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.PostgreSql; + +namespace Duende.Storage.IntegrationTests; + +[Collection("PostgreSqlIntegration")] +public partial class MigrationTests(AspireFixture fixture) +{ + private IMigrationFixtureFactory MigrationFixtureFactory { get; } = new PostgreSqlMigrationFixtureFactory(fixture); +} diff --git a/storage/test/Storage.PostgreSql.Tests/PostgreSqlDatabasePool.cs b/storage/test/Storage.PostgreSql.Tests/PostgreSqlDatabasePool.cs new file mode 100644 index 000000000..4eb68502e --- /dev/null +++ b/storage/test/Storage.PostgreSql.Tests/PostgreSqlDatabasePool.cs @@ -0,0 +1,101 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Collections.Concurrent; +using Npgsql; + +namespace Duende.Storage.PostgreSql; + +/// +/// Pools reusable PostgreSQL databases across integration tests. Instead of +/// creating and dropping a database per test, each test checks out a database, +/// runs, then returns it. On return the database tables are truncated so the +/// next test starts clean. +/// +internal sealed class PostgreSqlDatabasePool(string serverConnectionString) +{ + private readonly ConcurrentQueue _available = new(); + private readonly ConcurrentBag _all = []; + + /// + /// Returns a connection string for a ready-to-use database. The caller is + /// responsible for applying the schema. Creates a new database if none are + /// available in the pool. + /// + public async Task GetConnectionStringAsync(CancellationToken ct) + { + if (_available.TryDequeue(out var connectionString)) + { + return connectionString; + } + + var dbName = $"pool_{Guid.NewGuid():N}"; + await using var ds = NpgsqlDataSource.Create(serverConnectionString); + await using var cmd = ds.CreateCommand($"CREATE DATABASE \"{dbName}\""); + _ = await cmd.ExecuteNonQueryAsync(ct); + + var csb = new NpgsqlConnectionStringBuilder(serverConnectionString) { Database = dbName }; + var cs = csb.ConnectionString; + _all.Add(cs); + return cs; + } + + /// + /// Returns a database to the pool after truncating all test data. + /// + public async Task ReturnAsync(string connectionString) + { + try + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(10)); + await using var ds = NpgsqlDataSource.Create(connectionString); + await using var cmd = ds.CreateCommand(""" + TRUNCATE TABLE public.outbox_subscriber_queue; + TRUNCATE TABLE public.entity_links; + TRUNCATE TABLE public.entities CASCADE; + """); + _ = await cmd.ExecuteNonQueryAsync(cts.Token); + _available.Enqueue(connectionString); + } +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 + { + // If truncation fails, don't return the database to the pool — + // it stays poisoned until DropAllAsync cleans it up. + Console.WriteLine($"Failed to truncate pooled database; it will not be reused: {ex.Message}"); + } + } + + /// + /// Drops all databases that were created by this pool. Called during + /// test suite teardown. + /// + public async Task DropAllAsync() + { + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + var ct = cts.Token; + await using var ds = NpgsqlDataSource.Create(serverConnectionString); + foreach (var connectionString in _all) + { + var dbName = new NpgsqlConnectionStringBuilder(connectionString).Database!; + try + { + await using var terminateCmd = ds.CreateCommand(""" + SELECT pg_terminate_backend(pid) FROM pg_stat_activity + WHERE datname = $1 AND pid <> pg_backend_pid(); + """); + _ = terminateCmd.Parameters.Add(new NpgsqlParameter { Value = dbName }); + _ = await terminateCmd.ExecuteNonQueryAsync(ct); + await using var dropCmd = ds.CreateCommand($"DROP DATABASE \"{dbName}\""); + _ = await dropCmd.ExecuteNonQueryAsync(ct); + } +#pragma warning disable CA1031 + catch (Exception ex) +#pragma warning restore CA1031 + { + Console.WriteLine($"Failed to drop pooled database {dbName}: {ex.Message}"); + } + } + } +} diff --git a/storage/test/Storage.PostgreSql.Tests/PostgreSqlIntegrationGroup.cs b/storage/test/Storage.PostgreSql.Tests/PostgreSqlIntegrationGroup.cs new file mode 100644 index 000000000..58a3addf2 --- /dev/null +++ b/storage/test/Storage.PostgreSql.Tests/PostgreSqlIntegrationGroup.cs @@ -0,0 +1,7 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.PostgreSql; + +[CollectionDefinition("PostgreSqlIntegration")] +public sealed class PostgreSqlIntegrationGroup : ICollectionFixture; diff --git a/storage/test/Storage.PostgreSql.Tests/PostgreSqlIntegrationTests.cs b/storage/test/Storage.PostgreSql.Tests/PostgreSqlIntegrationTests.cs new file mode 100644 index 000000000..0792ce72c --- /dev/null +++ b/storage/test/Storage.PostgreSql.Tests/PostgreSqlIntegrationTests.cs @@ -0,0 +1,109 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.PostgreSql; + +namespace Duende.Storage.IntegrationTests; + +[Collection("PostgreSqlIntegration")] +public partial class Stores(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class StoreBatchOperations(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class StoreLinkOperations(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class StoreLinkQueryTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class StoreOutboxOperations(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class StoreTtlTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class StoreTryReadManyTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class PurgeExpiredTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class FilterTranslatorIntegrationTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class QueryStoreArrayFilterTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class QueryStoreBasicExpressionTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class QueryStoreCountTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class QueryStoreCursorPagingTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class QueryStoreGuidFieldTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class QueryStorePagingTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class QueryStoreSortTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + +[Collection("PostgreSqlIntegration")] +public partial class SystemTimestampQueryTests(AspireFixture fixture) +{ + private IStoreFixtureFactory FixtureFactory { get; } = new PostgreSqlStoreFixtureFactory(fixture); +} + diff --git a/storage/test/Storage.PostgreSql.Tests/PostgreSqlMigrationFixtureFactory.cs b/storage/test/Storage.PostgreSql.Tests/PostgreSqlMigrationFixtureFactory.cs new file mode 100644 index 000000000..2ed26d0fd --- /dev/null +++ b/storage/test/Storage.PostgreSql.Tests/PostgreSqlMigrationFixtureFactory.cs @@ -0,0 +1,53 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Globalization; +using Duende.Storage.IntegrationTests; +using Duende.Storage.Internal; +using Microsoft.Extensions.DependencyInjection; +using Npgsql; + +namespace Duende.Storage.PostgreSql; + +internal sealed class PostgreSqlMigrationFixtureFactory(AspireFixture aspire) : IMigrationFixtureFactory +{ + public async Task CreateAsync(CancellationToken ct) + { + var connectionString = await aspire.Pool.GetConnectionStringAsync(ct); + var schemaName = "s_" + DateTime.Now.Ticks.ToString(CultureInfo.InvariantCulture); + var services = new ServiceCollection(); + _ = services.AddLogging(); + _ = services.AddNpgsqlDataSource(connectionString, serviceKey: "migration-test"); + _ = services.AddStorageInternal(storage => storage.AddPostgreSqlStore("migration-test", o => o.SchemaName = schemaName)); + var provider = services.BuildServiceProvider(); + + var schema = provider.GetRequiredKeyedService("migration-test"); + return new PostgreSqlMigrationFixture(provider, schemaName, schema, connectionString); + } +} + +internal sealed class PostgreSqlMigrationFixture( + ServiceProvider provider, + string schemaName, + IDatabaseSchema schema, + string connectionString) : IMigrationFixture +{ + private NpgsqlDataSource _dataSource = NpgsqlDataSource.Create(connectionString); + public IDatabaseSchema Schema => schema; + + public async Task ExecuteSqlAsync(string sql, CancellationToken ct) + { + await using var cmd = _dataSource.CreateCommand(sql); + _ = await cmd.ExecuteNonQueryAsync(ct); + } + + public async ValueTask DisposeAsync() + { + await provider.DisposeAsync(); + + var dropCommand = _dataSource.CreateCommand("DROP SCHEMA IF EXISTS \"" + schemaName + "\" CASCADE"); + _ = await dropCommand.ExecuteNonQueryAsync(); + + await _dataSource.DisposeAsync(); + } +} diff --git a/storage/test/Storage.PostgreSql.Tests/PostgreSqlStoreFixture.cs b/storage/test/Storage.PostgreSql.Tests/PostgreSqlStoreFixture.cs new file mode 100644 index 000000000..160a5cc4c --- /dev/null +++ b/storage/test/Storage.PostgreSql.Tests/PostgreSqlStoreFixture.cs @@ -0,0 +1,23 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.IntegrationTests; +using Duende.Storage.Internal; +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.PostgreSql; + +internal sealed class PostgreSqlStoreFixture( + ServiceProvider provider, + IStore store, + PostgreSqlDatabasePool pool, + string connectionString) : IStoreFixture +{ + public IStore Store { get; } = store; + + public async ValueTask DisposeAsync() + { + await provider.DisposeAsync(); + await pool.ReturnAsync(connectionString); + } +} diff --git a/storage/test/Storage.PostgreSql.Tests/PostgreSqlStoreFixtureFactory.cs b/storage/test/Storage.PostgreSql.Tests/PostgreSqlStoreFixtureFactory.cs new file mode 100644 index 000000000..98a05ebd0 --- /dev/null +++ b/storage/test/Storage.PostgreSql.Tests/PostgreSqlStoreFixtureFactory.cs @@ -0,0 +1,33 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.IntegrationTests; +using Duende.Storage.Internal; +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.PostgreSql; + +internal sealed class PostgreSqlStoreFixtureFactory(AspireFixture aspire) : IStoreFixtureFactory +{ + private const string ServiceKey = "test"; + + public async Task CreateAsync(Ct ct, Action? configure = null) + { + var connectionString = await aspire.Pool.GetConnectionStringAsync(ct); + + var services = new ServiceCollection(); + _ = services.AddLogging(); + configure?.Invoke(services); + _ = services.AddNpgsqlDataSource(connectionString, serviceKey: ServiceKey); + _ = services.AddStorageInternal(storage => storage.AddPostgreSqlStore(ServiceKey, _ => { })); + var provider = services.BuildServiceProvider(); + + var schema = provider.GetRequiredKeyedService(ServiceKey); + await schema.MigrateAsync(ct); + + var pooledStore = provider.GetRequiredKeyedService(ServiceKey); + var store = pooledStore.OpenPool(1); + + return new PostgreSqlStoreFixture(provider, store, aspire.Pool, connectionString); + } +} diff --git a/storage/test/Storage.PostgreSql.Tests/PostgreSqlStoreTests.cs b/storage/test/Storage.PostgreSql.Tests/PostgreSqlStoreTests.cs new file mode 100644 index 000000000..1c60b7de2 --- /dev/null +++ b/storage/test/Storage.PostgreSql.Tests/PostgreSqlStoreTests.cs @@ -0,0 +1,51 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.PostgreSql; + +public class PostgreSqlStoreTests(AspireFixture fixture) : IClassFixture +{ + private readonly Ct _ct = TestContext.Current.CancellationToken; + private const string ServiceKey = "my-postgresql-store"; + + private ServiceProvider CreateServiceProvider() + { + var serviceCollection = new ServiceCollection(); + _ = serviceCollection.AddLogging(); + _ = serviceCollection.AddNpgsqlDataSource(fixture.ConnectionString, serviceKey: ServiceKey); + _ = serviceCollection.AddStorageInternal(storage => storage.AddPostgreSqlStore(ServiceKey, opt => { })); + return serviceCollection.BuildServiceProvider(); + } + + [Fact] + public void Can_resolve_store() + { + var serviceProvider = CreateServiceProvider(); + + var pooledStore = serviceProvider.GetRequiredKeyedService(ServiceKey); + + var store = pooledStore.OpenPool(1); + + _ = store.ShouldNotBeNull(); + } + + [Fact] + public async Task Can_create_schema() + { + var serviceProvider = CreateServiceProvider(); + + var pooledStore = serviceProvider.GetRequiredKeyedService(ServiceKey); + + var schemaVersionResult = await pooledStore.CheckVersionAsync(_ct); + schemaVersionResult.CurrentVersion.ShouldBe(0u); + schemaVersionResult.IsCompatible.ShouldBeFalse(); + schemaVersionResult.RequiredVersion.ShouldBe(1u); + + await pooledStore.MigrateAsync(_ct); + schemaVersionResult = await pooledStore.CheckVersionAsync(_ct); + schemaVersionResult.CurrentVersion.ShouldBe(1u); + } +} diff --git a/storage/test/Storage.PostgreSql.Tests/Storage.PostgreSql.Tests.csproj b/storage/test/Storage.PostgreSql.Tests/Storage.PostgreSql.Tests.csproj new file mode 100644 index 000000000..cc5d8202e --- /dev/null +++ b/storage/test/Storage.PostgreSql.Tests/Storage.PostgreSql.Tests.csproj @@ -0,0 +1,17 @@ + + + + Duende.Storage.PostgreSql + + + + + + + + + + + + + diff --git a/storage/test/Storage.Sqlite.Tests/MigrationTests.cs b/storage/test/Storage.Sqlite.Tests/MigrationTests.cs new file mode 100644 index 000000000..6ca7aa3f9 --- /dev/null +++ b/storage/test/Storage.Sqlite.Tests/MigrationTests.cs @@ -0,0 +1,11 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Sqlite; + +namespace Duende.Storage.IntegrationTests; + +public partial class MigrationTests +{ + private IMigrationFixtureFactory MigrationFixtureFactory { get; } = new SqliteMigrationFixtureFactory(); +} diff --git a/storage/test/Storage.Sqlite.Tests/SqliteIntegrationTests.cs b/storage/test/Storage.Sqlite.Tests/SqliteIntegrationTests.cs new file mode 100644 index 000000000..f1867028c --- /dev/null +++ b/storage/test/Storage.Sqlite.Tests/SqliteIntegrationTests.cs @@ -0,0 +1,92 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Sqlite; + +namespace Duende.Storage.IntegrationTests; + +public partial class Stores +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class StoreBatchOperations +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class StoreLinkOperations +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class StoreLinkQueryTests +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class StoreOutboxOperations +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class StoreTtlTests +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class StoreTryReadManyTests +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class PurgeExpiredTests +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class FilterTranslatorIntegrationTests +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class QueryStoreArrayFilterTests +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class QueryStoreBasicExpressionTests +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class QueryStoreCountTests +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class QueryStoreCursorPagingTests +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class QueryStoreGuidFieldTests +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class QueryStorePagingTests +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class QueryStoreSortTests +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + +public partial class SystemTimestampQueryTests +{ + private IStoreFixtureFactory FixtureFactory { get; } = new SqliteStoreFixtureFactory(); +} + diff --git a/storage/test/Storage.Sqlite.Tests/SqliteMigrationFixtureFactory.cs b/storage/test/Storage.Sqlite.Tests/SqliteMigrationFixtureFactory.cs new file mode 100644 index 000000000..7d4031ce3 --- /dev/null +++ b/storage/test/Storage.Sqlite.Tests/SqliteMigrationFixtureFactory.cs @@ -0,0 +1,46 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.IntegrationTests; +using Duende.Storage.Internal; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.Sqlite; + +internal sealed class SqliteMigrationFixtureFactory : IMigrationFixtureFactory +{ + public Task CreateAsync(CancellationToken ct) + { + var dbName = $"migration_test_{Guid.NewGuid():N}"; + var connectionString = $"Data Source={dbName};Mode=Memory;Cache=Shared"; + + var services = new ServiceCollection(); + _ = services.AddLogging(); + _ = services.AddStorageInternal(storage => storage.AddSqliteStore("migration-test", opt => opt.ConnectionString = connectionString)); + var provider = services.BuildServiceProvider(); + + var schema = provider.GetRequiredKeyedService("migration-test"); + IMigrationFixture fixture = new SqliteMigrationFixture(provider, schema, connectionString); + return Task.FromResult(fixture); + } +} + +internal sealed class SqliteMigrationFixture( + ServiceProvider provider, + IDatabaseSchema schema, + string connectionString) : IMigrationFixture +{ + public IDatabaseSchema Schema => schema; + + public async Task ExecuteSqlAsync(string sql, CancellationToken ct) + { + await using var connection = new SqliteConnection(connectionString); + await connection.OpenAsync(ct); + await using var cmd = connection.CreateCommand(); + cmd.CommandText = sql; + _ = await cmd.ExecuteNonQueryAsync(ct); + } + + public async ValueTask DisposeAsync() => await provider.DisposeAsync(); +} diff --git a/storage/test/Storage.Sqlite.Tests/SqliteStoreFixtureFactory.cs b/storage/test/Storage.Sqlite.Tests/SqliteStoreFixtureFactory.cs new file mode 100644 index 000000000..f5c0dc096 --- /dev/null +++ b/storage/test/Storage.Sqlite.Tests/SqliteStoreFixtureFactory.cs @@ -0,0 +1,13 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.IntegrationTests; +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.Sqlite; + +internal sealed class SqliteStoreFixtureFactory : IStoreFixtureFactory +{ + public async Task CreateAsync(Ct ct, Action? configure = null) => + await StoreFixture.CreateAsync(ct, configure); +} diff --git a/storage/test/Storage.Sqlite.Tests/SqliteStoreTests.cs b/storage/test/Storage.Sqlite.Tests/SqliteStoreTests.cs new file mode 100644 index 000000000..8a47f8781 --- /dev/null +++ b/storage/test/Storage.Sqlite.Tests/SqliteStoreTests.cs @@ -0,0 +1,181 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + + +using Duende.Storage.Internal; +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.Sqlite; + +public class SqliteStoreTests +{ + private readonly Ct _ct = TestContext.Current.CancellationToken; + private readonly IServiceProvider _serviceProvider; + private readonly string _connectionString; + private const string ServiceKey = "my-sqlite-store"; + + public SqliteStoreTests() + { + var serviceCollection = new ServiceCollection(); + _ = serviceCollection.AddLogging(); + var dbName = $"test_{Guid.NewGuid():N}"; + _connectionString = $"Data Source={dbName};Mode=Memory;Cache=Shared"; + _ = serviceCollection.AddStorageInternal(storage => storage.AddSqliteStore(ServiceKey, opt => opt.ConnectionString = _connectionString)); + _serviceProvider = serviceCollection.BuildServiceProvider(); + } + + [Fact] + public void Can_resolve_store() + { + var pooledStore = _serviceProvider.GetRequiredKeyedService(ServiceKey); + + var store = pooledStore.OpenPool(1); + + _ = store.ShouldNotBeNull(); + } + + [Fact] + public async Task Can_create_schema() + { + var databaseSchema = _serviceProvider.GetRequiredKeyedService(ServiceKey); + + var schemaVersionResult = await databaseSchema.CheckVersionAsync(_ct); + schemaVersionResult.CurrentVersion.ShouldBe(0u); + schemaVersionResult.IsCompatible.ShouldBeFalse(); + schemaVersionResult.RequiredVersion.ShouldBe(1u); + + await databaseSchema.MigrateAsync(_ct); + schemaVersionResult = await databaseSchema.CheckVersionAsync(_ct); + schemaVersionResult.CurrentVersion.ShouldBe(1u); + } + + [Fact] + public async Task Create_schema_twice_succeeds() + { + var databaseSchema = _serviceProvider.GetRequiredKeyedService(ServiceKey); + + await databaseSchema.MigrateAsync(_ct); + await databaseSchema.MigrateAsync(_ct); + } + + [Fact] + public async Task migrate_async_on_fresh_db_succeeds_and_version_is_current() + { + var databaseSchema = _serviceProvider.GetRequiredKeyedService(ServiceKey); + + await databaseSchema.MigrateAsync(_ct); + + var result = await databaseSchema.CheckVersionAsync(_ct); + result.CurrentVersion.ShouldBe(result.RequiredVersion); + result.IsCompatible.ShouldBeTrue(); + } + + [Fact] + public async Task migrate_async_twice_is_idempotent() + { + var databaseSchema = _serviceProvider.GetRequiredKeyedService(ServiceKey); + + await databaseSchema.MigrateAsync(_ct); + var versionAfterFirst = (await databaseSchema.CheckVersionAsync(_ct)).CurrentVersion; + + await databaseSchema.MigrateAsync(_ct); + var versionAfterSecond = (await databaseSchema.CheckVersionAsync(_ct)).CurrentVersion; + + versionAfterSecond.ShouldBe(versionAfterFirst); + } + + [Fact] + public async Task build_migration_script_from_zero_returns_non_empty_script_with_expected_tables() + { + var databaseSchema = _serviceProvider.GetRequiredKeyedService(ServiceKey); + + var script = databaseSchema.BuildMigrationScript(DatabaseSchemaVersion.Zero); + + script.ShouldNotBeNullOrWhiteSpace(); + script.ShouldContain("entities"); + script.ShouldContain("entity_keys"); + script.ShouldContain("search_values"); + script.ShouldContain("entity_links"); + script.ShouldContain("outbox_subscriber_queue"); + } + + [Fact] + public async Task build_migration_script_from_current_version_returns_empty_string() + { + var databaseSchema = _serviceProvider.GetRequiredKeyedService(ServiceKey); + + await databaseSchema.MigrateAsync(_ct); + var currentVersion = (await databaseSchema.CheckVersionAsync(_ct)).CurrentVersion; + + var script = databaseSchema.BuildMigrationScript(new DatabaseSchemaVersion((int)currentVersion)); + + script.Trim().ShouldBeEmpty(); + } + + [Fact] + public async Task verify_schema_async_on_valid_schema_returns_no_errors() + { + var databaseSchema = _serviceProvider.GetRequiredKeyedService(ServiceKey); + + await databaseSchema.MigrateAsync(_ct); + + var result = await databaseSchema.VerifySchemaAsync(_ct); + + result.IsValid.ShouldBeTrue(); + result.Errors.ShouldBeEmpty(); + } + + [Fact] + public async Task verify_schema_async_detects_dropped_column() + { + var databaseSchema = _serviceProvider.GetRequiredKeyedService(ServiceKey); + + await databaseSchema.MigrateAsync(_ct); + + await using var cnn = new SqliteConnection(_connectionString); + await cnn.OpenAsync(_ct); + + // Drop the partial index that references expires_at before dropping the column. + await using var dropIdxCmd = cnn.CreateCommand(); + dropIdxCmd.CommandText = "DROP INDEX entities_expires_at_index"; + _ = await dropIdxCmd.ExecuteNonQueryAsync(_ct); + + await using var cmd = cnn.CreateCommand(); + cmd.CommandText = "ALTER TABLE entities DROP COLUMN expires_at"; + _ = await cmd.ExecuteNonQueryAsync(_ct); + + var result = await databaseSchema.VerifySchemaAsync(_ct); + + result.IsValid.ShouldBeFalse(); + result.Errors.ShouldContain(e => + e.Kind == SchemaVerificationErrorKind.MissingColumn && + e.Table == "entities" && + e.Column == "expires_at"); + } + + [Fact] + public async Task migrate_async_throws_when_schema_verification_fails() + { + var databaseSchema = _serviceProvider.GetRequiredKeyedService(ServiceKey); + + // Migrate first so the schema exists and all steps are already applied. + await databaseSchema.MigrateAsync(_ct); + + // Corrupt the schema by dropping a column. + await using var cnn = new SqliteConnection(_connectionString); + await cnn.OpenAsync(_ct); + + // Drop the partial index that references expires_at before dropping the column. + await using var dropIdxCmd = cnn.CreateCommand(); + dropIdxCmd.CommandText = "DROP INDEX entities_expires_at_index"; + _ = await dropIdxCmd.ExecuteNonQueryAsync(_ct); + + await using var cmd = cnn.CreateCommand(); + cmd.CommandText = "ALTER TABLE entities DROP COLUMN expires_at"; + _ = await cmd.ExecuteNonQueryAsync(_ct); + + // Second MigrateAsync: no migration steps to run, but VerifySchemaAsync detects the missing column. + _ = await Should.ThrowAsync(() => databaseSchema.MigrateAsync(_ct)); + } +} diff --git a/storage/test/Storage.Sqlite.Tests/Storage.Sqlite.Tests.csproj b/storage/test/Storage.Sqlite.Tests/Storage.Sqlite.Tests.csproj new file mode 100644 index 000000000..2352ab68f --- /dev/null +++ b/storage/test/Storage.Sqlite.Tests/Storage.Sqlite.Tests.csproj @@ -0,0 +1,20 @@ + + + + Duende.Storage.Sqlite + + + + + + + + + + + + + + + + diff --git a/storage/test/Storage.Sqlite.Tests/StorageMetricsTests.cs b/storage/test/Storage.Sqlite.Tests/StorageMetricsTests.cs new file mode 100644 index 000000000..0edfeb382 --- /dev/null +++ b/storage/test/Storage.Sqlite.Tests/StorageMetricsTests.cs @@ -0,0 +1,151 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Diagnostics.Metrics; +using Duende.Storage.Internal.Telemetry; + +namespace Duende.Storage.Sqlite; + +public class StorageMetricsTests : IDisposable +{ + private readonly StorageMetrics _metrics = new(); + private readonly MeterListener _listener; + private readonly List<(string Name, object? Value, KeyValuePair[] Tags)> _recordedMetrics = []; + + public StorageMetricsTests() + { + _listener = new MeterListener(); + _listener.InstrumentPublished = (instrument, listener) => + { + if (instrument.Meter.Name == StorageTelemetryConstants.MeterName) + { + listener.EnableMeasurementEvents(instrument); + } + }; + _listener.SetMeasurementEventCallback(OnMeasurement); + _listener.SetMeasurementEventCallback(OnMeasurementDouble); + _listener.Start(); + } + + private void OnMeasurement(Instrument instrument, long measurement, ReadOnlySpan> tags, object? state) => + _recordedMetrics.Add((instrument.Name, measurement, tags.ToArray())); + + private void OnMeasurementDouble(Instrument instrument, double measurement, ReadOnlySpan> tags, object? state) => + _recordedMetrics.Add((instrument.Name, measurement, tags.ToArray())); + + public void Dispose() + { + _listener.Dispose(); + _metrics.Dispose(); + GC.SuppressFinalize(this); + } + + [Fact] + public void meter_name_should_be_correct() => + StorageTelemetryConstants.MeterName.ShouldBe("Duende.Storage"); + + [Fact] + public void instrument_names_should_match_spec() + { + StorageTelemetryConstants.Instruments.OperationCount.ShouldBe("duende.storage.operation.count"); + StorageTelemetryConstants.Instruments.OperationDuration.ShouldBe("duende.storage.operation.duration"); + } + + [Fact] + public void tag_names_should_match_spec() + { + StorageTelemetryConstants.Tags.Operation.ShouldBe("duende.storage.operation"); + StorageTelemetryConstants.Tags.DbSystem.ShouldBe("db.system"); + StorageTelemetryConstants.Tags.EntityType.ShouldBe("duende.storage.entity_type"); + StorageTelemetryConstants.Tags.Result.ShouldBe("duende.storage.result"); + StorageTelemetryConstants.Tags.ErrorType.ShouldBe("error.type"); + } + + [Fact] + public void record_success_should_increment_counter_with_correct_tags() + { + _metrics.RecordSuccess("create", "mssql", "client"); + + _recordedMetrics.ShouldContain(m => + m.Name == StorageTelemetryConstants.Instruments.OperationCount && + (long)m.Value! == 1 && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.Operation && (string?)t.Value == "create") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.DbSystem && (string?)t.Value == "mssql") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.EntityType && (string?)t.Value == "client") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.Result && (string?)t.Value == "success")); + } + + [Fact] + public void record_success_without_entity_type_should_omit_entity_type_tag() + { + _metrics.RecordSuccess("read", "postgresql", null); + + _recordedMetrics.ShouldContain(m => + m.Name == StorageTelemetryConstants.Instruments.OperationCount && + (long)m.Value! == 1 && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.Operation && (string?)t.Value == "read") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.DbSystem && (string?)t.Value == "postgresql") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.Result && (string?)t.Value == "success") && + !m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.EntityType)); + } + + [Fact] + public void record_error_should_increment_counter_with_error_tags() + { + var exception = new InvalidOperationException("test error"); + + _metrics.RecordError("update", "sqlite", exception, "session"); + + _recordedMetrics.ShouldContain(m => + m.Name == StorageTelemetryConstants.Instruments.OperationCount && + (long)m.Value! == 1 && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.Operation && (string?)t.Value == "update") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.DbSystem && (string?)t.Value == "sqlite") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.EntityType && (string?)t.Value == "session") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.Result && (string?)t.Value == "error") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.ErrorType && (string?)t.Value == "InvalidOperationException")); + } + + [Fact] + public void record_error_without_entity_type_should_omit_entity_type_tag() + { + var exception = new TimeoutException(); + + _metrics.RecordError("delete", "in_memory", exception, null); + + _recordedMetrics.ShouldContain(m => + m.Name == StorageTelemetryConstants.Instruments.OperationCount && + (long)m.Value! == 1 && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.Result && (string?)t.Value == "error") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.ErrorType && (string?)t.Value == "TimeoutException") + && m.Tags.All(t => t.Key != StorageTelemetryConstants.Tags.EntityType)); + } + + [Fact] + public void record_duration_should_record_histogram_with_correct_tags() + { + _metrics.RecordDuration("query", 1.234, "postgresql", "success", "token"); + + _recordedMetrics.ShouldContain(m => + m.Name == StorageTelemetryConstants.Instruments.OperationDuration && + (double)m.Value! == 1.234 && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.Operation && (string?)t.Value == "query") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.DbSystem && (string?)t.Value == "postgresql") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.EntityType && (string?)t.Value == "token") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.Result && (string?)t.Value == "success")); + } + + [Fact] + public void record_duration_without_entity_type_should_omit_entity_type_tag() + { + _metrics.RecordDuration("batch", 0.567, "mssql", "error", null); + + _recordedMetrics.ShouldContain(m => + m.Name == StorageTelemetryConstants.Instruments.OperationDuration && + (double)m.Value! == 0.567 && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.Operation && (string?)t.Value == "batch") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.DbSystem && (string?)t.Value == "mssql") && + m.Tags.Any(t => t.Key == StorageTelemetryConstants.Tags.Result && (string?)t.Value == "error") + && m.Tags.All(t => t.Key != StorageTelemetryConstants.Tags.EntityType)); + } +} diff --git a/storage/test/Storage.Sqlite.Tests/StoreFixture.cs b/storage/test/Storage.Sqlite.Tests/StoreFixture.cs new file mode 100644 index 000000000..1d6f4b2e1 --- /dev/null +++ b/storage/test/Storage.Sqlite.Tests/StoreFixture.cs @@ -0,0 +1,46 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.IntegrationTests; +using Duende.Storage.Internal; +using Microsoft.Extensions.DependencyInjection; + +namespace Duende.Storage.Sqlite; + +internal sealed class StoreFixture : IStoreFixture +{ + private const string ServiceKey = "test"; + private readonly ServiceProvider _provider; + + public IStore Store { get; } + + private StoreFixture(ServiceProvider provider, IStore store) + { + _provider = provider; + Store = store; + } + + public static async Task CreateAsync( + Ct ct, + Action? configure = null) + { + var services = new ServiceCollection(); + _ = services.AddLogging(); + configure?.Invoke(services); + + var dbName = $"test_{Guid.NewGuid():N}"; + _ = services.AddStorageInternal(storage => storage.AddSqliteStore(ServiceKey, opt => + opt.ConnectionString = $"Data Source={dbName};Mode=Memory;Cache=Shared")); + + var provider = services.BuildServiceProvider(); + + var schema = provider.GetRequiredKeyedService(ServiceKey); + await schema.MigrateAsync(ct); + + var pooledStore = provider.GetRequiredKeyedService(ServiceKey); + var store = pooledStore.OpenPool(1); + return new StoreFixture(provider, store); + } + + public async ValueTask DisposeAsync() => await _provider.DisposeAsync(); +} diff --git a/storage/test/Storage.Tests/EntityAttributeValue/AttributeCodes.cs b/storage/test/Storage.Tests/EntityAttributeValue/AttributeCodes.cs new file mode 100644 index 000000000..0a3c5d994 --- /dev/null +++ b/storage/test/Storage.Tests/EntityAttributeValue/AttributeCodes.cs @@ -0,0 +1,122 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +public static class AttributeCodes +{ + public static TheoryData InvalidInputs { get; } = + [ + "", + " ", + "a b", + "*", + "*a", + "a*", + "a*b", + "-", + "-a", + "a-", + "1", + "1a", + "a_", + new string('x', 101) + ]; + + [Theory] + [MemberData(nameof(InvalidInputs))] + public static void CannotParseInvalidInputs(string input) + { + var ex = Record.Exception(() => _ = AttributeCode.Create(input)); + + _ = ex.ShouldBeOfType(); + } + + [Fact] + public static void String_is_input() + { + const string Input = "schema_attribute_name"; + var instance = AttributeCode.Create(Input); + + var @string = instance.ToString(); + + @string.ShouldBe(Input); + } + + [Theory] + [InlineData("givenName")] + [InlineData("FamilyName")] + [InlineData("A")] + [InlineData("Aa")] + [InlineData("aA")] + [InlineData("aAb")] + public static void TryParse_accepts_mixed_case(string input) + { + var result = AttributeCode.TryCreate(input, out var name); + + result.ShouldBeTrue(); + name!.Value.ShouldBe(input, "Original casing should be preserved"); + } + + [Fact] + public static void Preserves_original_casing() + { + var name = AttributeCode.Create("givenName"); + + name.Value.ShouldBe("givenName"); + name.ToString().ShouldBe("givenName"); + } + + [Theory] + [InlineData("givenName", "givenname")] + [InlineData("givenName", "GIVENNAME")] + [InlineData("givenName", "GivenName")] + [InlineData("name", "NAME")] + public static void Equals_is_case_insensitive(string left, string right) + { + var a = AttributeCode.Create(left); + var b = AttributeCode.Create(right); + + a.ShouldBe(b); + b.ShouldBe(a); + } + + [Fact] + public static void GetHashCode_is_case_insensitive() + { + var a = AttributeCode.Create("givenName"); + var b = AttributeCode.Create("GIVENNAME"); + var c = AttributeCode.Create("givenname"); + + a.GetHashCode().ShouldBe(b.GetHashCode()); + a.GetHashCode().ShouldBe(c.GetHashCode()); + } + + [Fact] + public static void Works_as_case_insensitive_dictionary_key() + { + var dict = new Dictionary + { + [AttributeCode.Create("givenName")] = "Alice" + }; + + dict.TryGetValue(AttributeCode.Create("GIVENNAME"), out var value).ShouldBeTrue(); + value.ShouldBe("Alice"); + + dict.TryGetValue(AttributeCode.Create("givenname"), out value).ShouldBeTrue(); + value.ShouldBe("Alice"); + } + + [Fact] + public static void Dictionary_rejects_duplicate_casing_variants() + { + var dict = new Dictionary + { + [AttributeCode.Create("name")] = "first" + }; + + var ex = Record.Exception(() => dict.Add(AttributeCode.Create("NAME"), "second")); + + _ = ex.ShouldNotBeNull(); + } +} diff --git a/storage/test/Storage.Tests/EntityAttributeValue/AttributeDefinitions.cs b/storage/test/Storage.Tests/EntityAttributeValue/AttributeDefinitions.cs new file mode 100644 index 000000000..c6f707bc4 --- /dev/null +++ b/storage/test/Storage.Tests/EntityAttributeValue/AttributeDefinitions.cs @@ -0,0 +1,128 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +public static class AttributeDefinitions +{ + private static readonly AttributeCode TestName = AttributeCode.Create("test_attr"); + private static readonly AttributeDescription TestDescription = AttributeDescription.Create("A test attribute"); + + [Theory] + [InlineData(ScalarDataType.Boolean)] + [InlineData(ScalarDataType.Date)] + [InlineData(ScalarDataType.DateTime)] + [InlineData(ScalarDataType.Decimal)] + [InlineData(ScalarDataType.Integer)] + [InlineData(ScalarDataType.String)] + public static void existing_scalar_constructor_still_works(ScalarDataType dataType) + { + var definition = new AttributeDefinition(TestName, dataType, TestDescription); + + _ = definition.AttributeType.ShouldBeOfType(); + ((ScalarAttributeType)definition.AttributeType).DataType.ShouldBe(dataType); + } + + [Fact] + public static void new_constructor_with_complex_type_works() + { + var complexType = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }); + var definition = new AttributeDefinition(TestName, complexType, TestDescription); + + definition.AttributeType.ShouldBe(complexType); + } + + [Fact] + public static void is_unique_true_with_complex_type_throws() + { + var complexType = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }); + + var ex = Record.Exception(() => + _ = new AttributeDefinition(TestName, complexType, TestDescription, IsUnique: true, Tags: null)); + + _ = ex.ShouldBeOfType(); + } + + [Fact] + public static void is_unique_true_with_list_type_throws() + { + var listType = new ListAttributeType(new ScalarAttributeType(ScalarDataType.String)); + + var ex = Record.Exception(() => + _ = new AttributeDefinition(TestName, listType, TestDescription, IsUnique: true, Tags: null)); + + _ = ex.ShouldBeOfType(); + } + + [Fact] + public static void tags_default_to_empty_collection() + { + var definition = new AttributeDefinition(TestName, ScalarDataType.String, TestDescription); + + definition.Tags.ShouldBeEmpty(); + } + + [Fact] + public static void scalar_attribute_can_have_group_name() + { + var groupName = AttributeGroupCode.Create("personal_info"); + var definition = new AttributeDefinition(TestName, ScalarDataType.String, TestDescription, false, null, groupName, 0); + + definition.GroupCode.ShouldBe(groupName); + } + + [Fact] + public static void complex_attribute_can_have_group_name() + { + var complexType = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }); + var groupName = AttributeGroupCode.Create("personal_info"); + + var definition = new AttributeDefinition(TestName, complexType, TestDescription, false, null, groupName, 0); + + definition.GroupCode.ShouldBe(groupName); + } + + [Fact] + public static void list_attribute_can_have_group_name() + { + var listType = new ListAttributeType(new ScalarAttributeType(ScalarDataType.String)); + var groupName = AttributeGroupCode.Create("personal_info"); + + var definition = new AttributeDefinition(TestName, listType, TestDescription, false, null, groupName, 0); + + definition.GroupCode.ShouldBe(groupName); + } + + [Fact] + public static void group_name_defaults_to_null() + { + var definition = new AttributeDefinition(TestName, ScalarDataType.String, TestDescription); + + definition.GroupCode.ShouldBeNull(); + } + + [Fact] + public static void order_defaults_to_zero() + { + var definition = new AttributeDefinition(TestName, ScalarDataType.String, TestDescription); + + definition.Order.ShouldBe(0); + } + + [Fact] + public static void order_is_preserved() + { + var definition = new AttributeDefinition(TestName, ScalarDataType.String, TestDescription, false, null, null, 42); + + definition.Order.ShouldBe(42); + } +} diff --git a/storage/test/Storage.Tests/EntityAttributeValue/AttributeDescriptions.cs b/storage/test/Storage.Tests/EntityAttributeValue/AttributeDescriptions.cs new file mode 100644 index 000000000..b3452cc2d --- /dev/null +++ b/storage/test/Storage.Tests/EntityAttributeValue/AttributeDescriptions.cs @@ -0,0 +1,29 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +public static class AttributeDescriptions +{ + public static TheoryData InvalidInputs { get; } = ["", " ", new string('x', 201)]; + + [Theory] + [MemberData(nameof(InvalidInputs))] + public static void CannotParseInvalidInputs(string input) + { + var ex = Record.Exception(() => _ = AttributeDescription.Create(input)); + + _ = ex.ShouldBeOfType(); + } + + [Fact] + public static void String_is_input() + { + const string Input = $"{nameof(AttributeDescription)}1"; + var instance = AttributeDescription.Create(Input); + + var @string = instance.ToString(); + + @string.ShouldBe(Input); + } +} diff --git a/storage/test/Storage.Tests/EntityAttributeValue/AttributeDisplayNames.cs b/storage/test/Storage.Tests/EntityAttributeValue/AttributeDisplayNames.cs new file mode 100644 index 000000000..6d38e5e45 --- /dev/null +++ b/storage/test/Storage.Tests/EntityAttributeValue/AttributeDisplayNames.cs @@ -0,0 +1,75 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +public sealed class AttributeDisplayNames +{ + [Fact] + public void valid_input_is_accepted() + { + var instance = AttributeDisplayName.Create("Personal Information"); + + instance.Value.ShouldBe("Personal Information"); + } + + [Fact] + public void empty_string_is_rejected() + { + var ex = Record.Exception(() => _ = AttributeDisplayName.Create("")); + + _ = ex.ShouldBeOfType(); + } + + [Fact] + public void whitespace_only_is_rejected() + { + var ex = Record.Exception(() => _ = AttributeDisplayName.Create(" ")); + + _ = ex.ShouldBeOfType(); + } + + [Fact] + public void over_max_length_is_rejected() + { + var input = new string('x', 201); + + var ex = Record.Exception(() => _ = AttributeDisplayName.Create(input)); + + _ = ex.ShouldBeOfType(); + } + + [Fact] + public void max_length_is_accepted() + { + var input = new string('x', 200); + var instance = AttributeDisplayName.Create(input); + + instance.Value.ShouldBe(input); + } + + [Fact] + public void input_is_trimmed() + { + var instance = AttributeDisplayName.Create(" Personal Info "); + + instance.Value.ShouldBe("Personal Info"); + } + + [Fact] + public void try_parse_returns_false_for_empty() + { + var result = AttributeDisplayName.TryCreate("", out var parsed); + + result.ShouldBeFalse(); + parsed.ShouldBe(default(AttributeDisplayName)); + } + + [Fact] + public void to_string_returns_value() + { + var instance = AttributeDisplayName.Create("Contact Details"); + + instance.ToString().ShouldBe("Contact Details"); + } +} diff --git a/storage/test/Storage.Tests/EntityAttributeValue/AttributeGroupCodes.cs b/storage/test/Storage.Tests/EntityAttributeValue/AttributeGroupCodes.cs new file mode 100644 index 000000000..2ed3f74fc --- /dev/null +++ b/storage/test/Storage.Tests/EntityAttributeValue/AttributeGroupCodes.cs @@ -0,0 +1,106 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +public sealed class AttributeGroupNames +{ + public static TheoryData InvalidInputs { get; } = + [ + "", + " ", + "a b", + "*", + "*a", + "a*", + "a*b", + "hello world", + "name@group", + new string('x', 101) + ]; + + public static TheoryData ValidInputs { get; } = + [ + "a", + "abc", + "personal_info", + "my-group", + "123", + "1a", + "a_", + "-a", + "a-", + "A_B-C" + ]; + + [Theory] + [MemberData(nameof(InvalidInputs))] + public void cannot_parse_invalid_inputs(string input) + { + var ex = Record.Exception(() => _ = AttributeGroupCode.Create(input)); + + _ = ex.ShouldBeOfType(); + } + + [Theory] + [MemberData(nameof(ValidInputs))] + public void can_parse_valid_inputs(string input) + { + var instance = AttributeGroupCode.Create(input); + + instance.Value.ShouldBe(input); + } + + [Fact] + public void string_is_preserved() + { + const string Input = "personal_info"; + var instance = AttributeGroupCode.Create(Input); + + instance.Value.ShouldBe(Input); + } + + [Fact] + public void equality_is_case_insensitive() + { + var a = AttributeGroupCode.Create("PersonalInfo"); + var b = AttributeGroupCode.Create("personalinfo"); + + a.ShouldBe(b); + } + + [Fact] + public void hash_code_is_case_insensitive() + { + var a = AttributeGroupCode.Create("PersonalInfo"); + var b = AttributeGroupCode.Create("personalinfo"); + + a.GetHashCode().ShouldBe(b.GetHashCode()); + } + + [Fact] + public void max_length_is_accepted() + { + var input = new string('x', 100); + var instance = AttributeGroupCode.Create(input); + + instance.Value.ShouldBe(input); + } + + [Fact] + public void try_parse_returns_false_for_invalid() + { + var result = AttributeGroupCode.TryCreate("not valid!", out var parsed); + + result.ShouldBeFalse(); + parsed.ShouldBe(default(AttributeGroupCode)); + } + + [Fact] + public void to_string_returns_value() + { + var instance = AttributeGroupCode.Create("my_group"); + + instance.ToString().ShouldBe("my_group"); + } +} diff --git a/storage/test/Storage.Tests/EntityAttributeValue/AttributeSchemaCreationTests.cs b/storage/test/Storage.Tests/EntityAttributeValue/AttributeSchemaCreationTests.cs new file mode 100644 index 000000000..561c0b24a --- /dev/null +++ b/storage/test/Storage.Tests/EntityAttributeValue/AttributeSchemaCreationTests.cs @@ -0,0 +1,329 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Collections.ObjectModel; +using Duende.Storage.EntityAttributeValue.Internal; + +namespace Duende.Storage.EntityAttributeValue; + +public sealed class AttributeSchemaCreationTests +{ + private static readonly AttributeDescription Desc = AttributeDescription.Create("test"); + + private static AttributeSchema SchemaWith(AttributeDefinition definition) + { + var schema = new AttributeSchema(); + _ = schema.AddAttributeDefinition(definition); + return schema; + } + + [Fact] + public void complex_matching_properties_succeeds() + { + var name = AttributeCode.Create("address"); + var complexType = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String), + [AttributeCode.Create("zip")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }); + var schema = SchemaWith(new AttributeDefinition(name, complexType, Desc)); + + var value = new Dictionary { ["city"] = "Seattle", ["zip"] = "98101" }; + var attr = schema.CreateAttribute(name, value); + + _ = attr.UntypedValue.ShouldBeOfType>(); + } + + [Fact] + public void complex_partial_object_is_allowed() + { + // Properties are optional — partial objects are valid + var name = AttributeCode.Create("address"); + var complexType = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String), + [AttributeCode.Create("zip")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }); + var schema = SchemaWith(new AttributeDefinition(name, complexType, Desc)); + + var value = new Dictionary { ["city"] = "Seattle" }; // zip omitted + var result = schema.TryCreateAttribute(name, value, out _); + + result.ShouldBeTrue(); + } + + [Fact] + public void complex_extra_unknown_property_fails() + { + var name = AttributeCode.Create("address"); + var complexType = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }); + var schema = SchemaWith(new AttributeDefinition(name, complexType, Desc)); + + var value = new Dictionary { ["city"] = "Seattle", ["country"] = "US" }; + var result = schema.TryCreateAttribute(name, value, out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void complex_wrong_sub_property_type_fails() + { + var name = AttributeCode.Create("address"); + var complexType = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("zip")] = ComplexAttributeProperty.Of(ScalarDataType.Integer) + }); + var schema = SchemaWith(new AttributeDefinition(name, complexType, Desc)); + + var value = new Dictionary { ["zip"] = "not-an-int" }; // string instead of int + var result = schema.TryCreateAttribute(name, value, out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void list_of_strings_succeeds() + { + var name = AttributeCode.Create("tags"); + var listType = new ListAttributeType(new ScalarAttributeType(ScalarDataType.String)); + var schema = SchemaWith(new AttributeDefinition(name, listType, Desc)); + + var value = new List { "admin", "power" }; + var attr = schema.CreateAttribute(name, value); + + _ = attr.UntypedValue.ShouldBeOfType>(); + } + + [Fact] + public void list_of_complex_succeeds() + { + var name = AttributeCode.Create("phones"); + var listType = new ListAttributeType(new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("number")] = ComplexAttributeProperty.Of(ScalarDataType.String), + [AttributeCode.Create("type")] = ComplexAttributeProperty.Of(ScalarDataType.String) + })); + var schema = SchemaWith(new AttributeDefinition(name, listType, Desc)); + + var value = new List + { + new Dictionary { ["number"] = "555-1234", ["type"] = "mobile" }, + new Dictionary { ["number"] = "555-9999", ["type"] = "home"}, + }; + var attr = schema.CreateAttribute(name, value); + + _ = attr.UntypedValue.ShouldBeOfType>(); + + attr.TypedValue.Count.ShouldBe(2); + ((Dictionary)attr.TypedValue[0])["number"].ShouldBe("555-1234"); + ((Dictionary)attr.TypedValue[^1])["type"].ShouldBe("home"); + } + + [Fact] + public void empty_list_succeeds() + { + var name = AttributeCode.Create("tags"); + var listType = new ListAttributeType(new ScalarAttributeType(ScalarDataType.String)); + var schema = SchemaWith(new AttributeDefinition(name, listType, Desc)); + + var value = new List(); + var result = schema.TryCreateAttribute(name, value, out _); + + result.ShouldBeTrue(); + } + + // --- Negative / edge-case tests --- + + [Fact] + public void unknown_attribute_name_fails_for_bool() + { + var schema = SchemaWith(new AttributeDefinition( + AttributeCode.Create("active"), new ScalarAttributeType(ScalarDataType.Boolean), Desc)); + + var result = schema.TryCreateAttribute(AttributeCode.Create("missing"), true, out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void unknown_attribute_name_fails_for_string() + { + var schema = SchemaWith(new AttributeDefinition( + AttributeCode.Create("color"), new ScalarAttributeType(ScalarDataType.String), Desc)); + + var result = schema.TryCreateAttribute(AttributeCode.Create("missing"), "value", out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void unknown_attribute_name_fails_for_int() + { + var schema = SchemaWith(new AttributeDefinition( + AttributeCode.Create("age"), new ScalarAttributeType(ScalarDataType.Integer), Desc)); + + var result = schema.TryCreateAttribute(AttributeCode.Create("missing"), 42, out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void unknown_attribute_name_fails_for_decimal() + { + var schema = SchemaWith(new AttributeDefinition( + AttributeCode.Create("balance"), new ScalarAttributeType(ScalarDataType.Decimal), Desc)); + + var result = schema.TryCreateAttribute(AttributeCode.Create("missing"), 3.14m, out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void unknown_attribute_name_fails_for_date() + { + var schema = SchemaWith(new AttributeDefinition( + AttributeCode.Create("birthdate"), new ScalarAttributeType(ScalarDataType.Date), Desc)); + + var result = schema.TryCreateAttribute(AttributeCode.Create("missing"), new DateOnly(2000, 1, 1), out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void unknown_attribute_name_fails_for_date_time() + { + var schema = SchemaWith(new AttributeDefinition( + AttributeCode.Create("recordedat"), new ScalarAttributeType(ScalarDataType.DateTime), Desc)); + + var result = schema.TryCreateAttribute(AttributeCode.Create("missing"), DateTimeOffset.UtcNow, out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void unknown_attribute_name_fails_for_complex() + { + var schema = SchemaWith(new AttributeDefinition( + AttributeCode.Create("address"), + new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }), + Desc)); + + var value = new Dictionary { ["city"] = "Seattle" }; + var result = schema.TryCreateAttribute(AttributeCode.Create("missing"), value, out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void unknown_attribute_name_fails_for_list() + { + var schema = SchemaWith(new AttributeDefinition( + AttributeCode.Create("tags"), + new ListAttributeType(new ScalarAttributeType(ScalarDataType.String)), + Desc)); + + var value = new List { "a", "b" }; + var result = schema.TryCreateAttribute(AttributeCode.Create("missing"), value, out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void wrong_type_string_instead_of_bool_fails() + { + var name = AttributeCode.Create("active"); + var schema = SchemaWith(new AttributeDefinition(name, new ScalarAttributeType(ScalarDataType.Boolean), Desc)); + + // Attempt to create a string attribute for a boolean definition + var result = schema.TryCreateAttribute(name, "true", out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void wrong_type_bool_instead_of_string_fails() + { + var name = AttributeCode.Create("color"); + var schema = SchemaWith(new AttributeDefinition(name, new ScalarAttributeType(ScalarDataType.String), Desc)); + + // Attempt to create a bool attribute for a string definition + var result = schema.TryCreateAttribute(name, true, out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void wrong_type_int_instead_of_decimal_fails() + { + var name = AttributeCode.Create("balance"); + var schema = SchemaWith(new AttributeDefinition(name, new ScalarAttributeType(ScalarDataType.Decimal), Desc)); + + // Attempt to create an int attribute for a decimal definition + var result = schema.TryCreateAttribute(name, 42, out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void wrong_type_scalar_instead_of_list_fails() + { + var name = AttributeCode.Create("tags"); + var schema = SchemaWith(new AttributeDefinition( + name, new ListAttributeType(new ScalarAttributeType(ScalarDataType.String)), Desc)); + + // Attempt to create a string attribute for a list definition + var result = schema.TryCreateAttribute(name, "single", out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void wrong_type_scalar_instead_of_complex_fails() + { + var name = AttributeCode.Create("address"); + var schema = SchemaWith(new AttributeDefinition( + name, + new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }), + Desc)); + + // Attempt to create a string attribute for a complex definition + var result = schema.TryCreateAttribute(name, "not-a-complex", out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void list_with_wrong_element_type_fails() + { + var name = AttributeCode.Create("tags"); + var schema = SchemaWith(new AttributeDefinition( + name, new ListAttributeType(new ScalarAttributeType(ScalarDataType.String)), Desc)); + + // Provide ints instead of strings as list elements + var value = new List { 1, 2, 3 }; + var result = schema.TryCreateAttribute(name, value, out _); + + result.ShouldBeFalse(); + } + + [Fact] + public void create_attribute_throws_for_invalid_value() + { + var name = AttributeCode.Create("active"); + var schema = SchemaWith(new AttributeDefinition(name, new ScalarAttributeType(ScalarDataType.Boolean), Desc)); + + // CreateAttribute (non-Try) should throw for wrong type + var ex = Record.Exception(() => schema.CreateAttribute(name, "not-a-bool")); + + _ = ex.ShouldBeOfType(); + } +} diff --git a/storage/test/Storage.Tests/EntityAttributeValue/AttributeSchemaGroupTests.cs b/storage/test/Storage.Tests/EntityAttributeValue/AttributeSchemaGroupTests.cs new file mode 100644 index 000000000..42b5663c6 --- /dev/null +++ b/storage/test/Storage.Tests/EntityAttributeValue/AttributeSchemaGroupTests.cs @@ -0,0 +1,189 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.EntityAttributeValue.Internal; + +namespace Duende.Storage.EntityAttributeValue; + +public sealed class AttributeSchemaGroupTests +{ + private static readonly AttributeDescription Desc = AttributeDescription.Create("test"); + + [Fact] + public void add_group_stores_code_and_display_name() + { + var schema = new AttributeSchema(); + var group = new AttributeGroup( + AttributeGroupCode.Create("personal_info"), + AttributeDisplayName.Create("Personal Information"), + null, + 0); + + schema.AddGroup(group).ShouldBeTrue(); + + schema.Groups.ShouldContainKey(group.Code); + schema.Groups[group.Code].DisplayName.ShouldBe(group.DisplayName); + } + + [Fact] + public void add_group_stores_description() + { + var schema = new AttributeSchema(); + var group = new AttributeGroup( + AttributeGroupCode.Create("personal_info"), + AttributeDisplayName.Create("Personal Information"), + AttributeDescription.Create("Personal details"), + 0); + + schema.AddGroup(group).ShouldBeTrue(); + + schema.Groups[group.Code].Description.ShouldBe(group.Description); + } + + [Fact] + public void add_group_stores_order() + { + var schema = new AttributeSchema(); + var group = new AttributeGroup( + AttributeGroupCode.Create("personal_info"), + null, + null, + 42); + + schema.AddGroup(group).ShouldBeTrue(); + + schema.Groups[group.Code].Order.ShouldBe(42); + } + + [Fact] + public void add_group_with_null_display_name_and_description() + { + var schema = new AttributeSchema(); + var group = new AttributeGroup( + AttributeGroupCode.Create("personal_info"), + null, + null, + 0); + + schema.AddGroup(group).ShouldBeTrue(); + + schema.Groups[group.Code].DisplayName.ShouldBeNull(); + schema.Groups[group.Code].Description.ShouldBeNull(); + } + + [Fact] + public void cannot_add_duplicate_group() + { + var schema = new AttributeSchema(); + var group = new AttributeGroup( + AttributeGroupCode.Create("personal_info"), + AttributeDisplayName.Create("Personal Information"), + null, + 0); + + schema.AddGroup(group).ShouldBeTrue(); + schema.AddGroup(group).ShouldBeFalse(); + } + + [Fact] + public void remove_group_returns_true_when_exists() + { + var schema = new AttributeSchema(); + var code = AttributeGroupCode.Create("personal_info"); + var group = new AttributeGroup(code, null, null, 0); + _ = schema.AddGroup(group); + + schema.RemoveGroup(code).ShouldBeTrue(); + + schema.Groups.ShouldBeEmpty(); + } + + [Fact] + public void remove_group_returns_false_when_not_found() + { + var schema = new AttributeSchema(); + + schema.RemoveGroup(AttributeGroupCode.Create("nonexistent")).ShouldBeFalse(); + } + + [Fact] + public void remove_group_ungroups_member_attributes() + { + var schema = new AttributeSchema(); + var groupCode = AttributeGroupCode.Create("personal_info"); + var group = new AttributeGroup(groupCode, AttributeDisplayName.Create("Personal"), null, 0); + _ = schema.AddGroup(group); + + var definition = new AttributeDefinition( + AttributeCode.Create("first_name"), + ScalarDataType.String, + Desc, + false, + null, + groupCode, + 0); + _ = schema.AddAttributeDefinition(definition); + + schema.RemoveGroup(groupCode).ShouldBeTrue(); + + schema.AttributeDefinitions[definition].GroupCode.ShouldBeNull(); + } + + [Fact] + public void remove_group_preserves_ungrouped_attributes() + { + var schema = new AttributeSchema(); + var groupCode = AttributeGroupCode.Create("personal_info"); + var group = new AttributeGroup(groupCode, null, null, 0); + _ = schema.AddGroup(group); + + var grouped = new AttributeDefinition( + AttributeCode.Create("first_name"), + ScalarDataType.String, + Desc, + false, + null, + groupCode, + 0); + var ungrouped = new AttributeDefinition( + AttributeCode.Create("email"), + ScalarDataType.String, + Desc); + _ = schema.AddAttributeDefinition(grouped); + _ = schema.AddAttributeDefinition(ungrouped); + + _ = schema.RemoveGroup(groupCode); + + schema.AttributeDefinitions[ungrouped].GroupCode.ShouldBeNull(); + schema.AttributeDefinitions.Count.ShouldBe(2); + } + + [Fact] + public void group_dso_round_trip_preserves_display_name_and_description() + { + var schema = new AttributeSchema(); + var group = new AttributeGroup( + AttributeGroupCode.Create("contact"), + AttributeDisplayName.Create("Contact Info"), + AttributeDescription.Create("Contact details"), + 5); + _ = schema.AddGroup(group); + + var definition = new AttributeDefinition( + AttributeCode.Create("phone"), + ScalarDataType.String, + Desc, + false, + null, + group.Code, + 1); + _ = schema.AddAttributeDefinition(definition); + + // Verify the in-memory state is correct + var loaded = schema.Groups[group.Code]; + loaded.Code.ShouldBe(group.Code); + loaded.DisplayName.ShouldBe(group.DisplayName); + loaded.Description.ShouldBe(group.Description); + loaded.Order.ShouldBe(5); + } +} diff --git a/storage/test/Storage.Tests/EntityAttributeValue/AttributeSchemaValidationErrorTests.cs b/storage/test/Storage.Tests/EntityAttributeValue/AttributeSchemaValidationErrorTests.cs new file mode 100644 index 000000000..e428e89a2 --- /dev/null +++ b/storage/test/Storage.Tests/EntityAttributeValue/AttributeSchemaValidationErrorTests.cs @@ -0,0 +1,223 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.EntityAttributeValue.Internal; + +namespace Duende.Storage.EntityAttributeValue; + +public sealed class AttributeSchemaValidationErrorTests +{ + private static readonly AttributeDescription Desc = AttributeDescription.Create("test"); + + private static AttributeSchema SchemaWith(AttributeDefinition definition) + { + var schema = new AttributeSchema(); + _ = schema.AddAttributeDefinition(definition); + return schema; + } + + // --- Scalar --- + + [Fact] + public void undefined_attribute_code_returns_not_defined_error() + { + var schema = SchemaWith(new AttributeDefinition( + AttributeCode.Create("active"), new ScalarAttributeType(ScalarDataType.Boolean), Desc)); + + var result = schema.TryCreateAttribute(AttributeCode.Create("missing"), true, out _, out var errors); + + result.ShouldBeFalse(); + _ = errors.ShouldNotBeNull(); + errors.ShouldContain(e => e.Contains("is not defined in the schema")); + } + + [Fact] + public void type_mismatch_returns_defined_as_error() + { + var name = AttributeCode.Create("active"); + var schema = SchemaWith(new AttributeDefinition(name, new ScalarAttributeType(ScalarDataType.Boolean), Desc)); + + var result = schema.TryCreateAttribute(name, "true", out _, out var errors); + + result.ShouldBeFalse(); + _ = errors.ShouldNotBeNull(); + errors.ShouldContain(e => e.Contains("is defined as")); + } + + [Fact] + public void success_returns_null_errors() + { + var name = AttributeCode.Create("active"); + var schema = SchemaWith(new AttributeDefinition(name, new ScalarAttributeType(ScalarDataType.Boolean), Desc)); + + var result = schema.TryCreateAttribute(name, true, out _, out var errors); + + result.ShouldBeTrue(); + errors.ShouldBeNull(); + } + + // --- Complex --- + + [Fact] + public void unknown_property_returns_not_defined_error() + { + var name = AttributeCode.Create("address"); + var complexType = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }); + var schema = SchemaWith(new AttributeDefinition(name, complexType, Desc)); + + var value = new Dictionary { ["city"] = "Seattle", ["country"] = "US" }; + var result = schema.TryCreateAttribute(name, (IReadOnlyDictionary)value, out _, out var errors); + + result.ShouldBeFalse(); + _ = errors.ShouldNotBeNull(); + errors.ShouldContain(e => e.Contains("is not defined in complex attribute")); + } + + [Fact] + public void property_type_mismatch_returns_expects_type_error() + { + var name = AttributeCode.Create("address"); + var complexType = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("zip")] = ComplexAttributeProperty.Of(ScalarDataType.Integer) + }); + var schema = SchemaWith(new AttributeDefinition(name, complexType, Desc)); + + var value = new Dictionary { ["zip"] = "not-an-int" }; + var result = schema.TryCreateAttribute(name, (IReadOnlyDictionary)value, out _, out var errors); + + result.ShouldBeFalse(); + _ = errors.ShouldNotBeNull(); + errors.ShouldContain(e => e.Contains("expects type")); + } + + [Fact] + public void multiple_errors_accumulated() + { + var name = AttributeCode.Create("address"); + var complexType = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }); + var schema = SchemaWith(new AttributeDefinition(name, complexType, Desc)); + + // Two unknown properties → two errors + var value = new Dictionary { ["country"] = "US", ["region"] = "WA" }; + var result = schema.TryCreateAttribute(name, (IReadOnlyDictionary)value, out _, out var errors); + + result.ShouldBeFalse(); + _ = errors.ShouldNotBeNull(); + errors.Count.ShouldBe(2); + } + + [Fact] + public void not_a_complex_type_returns_error() + { + var name = AttributeCode.Create("active"); + var schema = SchemaWith(new AttributeDefinition(name, new ScalarAttributeType(ScalarDataType.Boolean), Desc)); + + var value = new Dictionary { ["key"] = "value" }; + var result = schema.TryCreateAttribute(name, (IReadOnlyDictionary)value, out _, out var errors); + + result.ShouldBeFalse(); + _ = errors.ShouldNotBeNull(); + errors.ShouldContain(e => e.Contains("is not a complex type")); + } + + // --- List --- + + [Fact] + public void element_type_mismatch_returns_index_error() + { + var name = AttributeCode.Create("tags"); + var schema = SchemaWith(new AttributeDefinition( + name, new ListAttributeType(new ScalarAttributeType(ScalarDataType.String)), Desc)); + + var value = new List { "valid", 42 }; + var result = schema.TryCreateAttribute(name, (IReadOnlyList)value, out _, out var errors); + + result.ShouldBeFalse(); + _ = errors.ShouldNotBeNull(); + errors.ShouldContain(e => e.Contains("Element at index")); + } + + [Fact] + public void multiple_list_errors_accumulated() + { + var name = AttributeCode.Create("tags"); + var schema = SchemaWith(new AttributeDefinition( + name, new ListAttributeType(new ScalarAttributeType(ScalarDataType.String)), Desc)); + + // Two wrong-type elements → two errors + var value = new List { 1, 2 }; + var result = schema.TryCreateAttribute(name, (IReadOnlyList)value, out _, out var errors); + + result.ShouldBeFalse(); + _ = errors.ShouldNotBeNull(); + errors.Count.ShouldBe(2); + } + + [Fact] + public void not_a_list_type_returns_error() + { + var name = AttributeCode.Create("active"); + var schema = SchemaWith(new AttributeDefinition(name, new ScalarAttributeType(ScalarDataType.Boolean), Desc)); + + var value = new List { "a", "b" }; + var result = schema.TryCreateAttribute(name, (IReadOnlyList)value, out _, out var errors); + + result.ShouldBeFalse(); + _ = errors.ShouldNotBeNull(); + errors.ShouldContain(e => e.Contains("is not a list type")); + } + + // --- List of Complex (SCIM emails) --- + + private static AttributeSchema EmailsSchema() + { + var emailsType = new ListAttributeType(new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("value")] = ComplexAttributeProperty.Of(ScalarDataType.String), + [AttributeCode.Create("type")] = ComplexAttributeProperty.Of(ScalarDataType.String), + [AttributeCode.Create("primary")] = ComplexAttributeProperty.Of(ScalarDataType.Boolean) + })); + return SchemaWith(new AttributeDefinition(AttributeCode.Create("emails"), emailsType, Desc)); + } + + [Fact] + public void list_of_complex_property_type_mismatch_returns_error() + { + var schema = EmailsSchema(); + + // "primary" should be a bool, not a string + var value = new List + { + new Dictionary { ["value"] = "a@b.com", ["type"] = "work", ["primary"] = "yes" } + }; + var result = schema.TryCreateAttribute(AttributeCode.Create("emails"), (IReadOnlyList)value, out _, out var errors); + + result.ShouldBeFalse(); + _ = errors.ShouldNotBeNull(); + errors.ShouldContain(e => e.Contains("Element at index 0") && e.Contains("primary") && e.Contains("expects type")); + } + + [Fact] + public void list_of_complex_unknown_property_returns_error() + { + var schema = EmailsSchema(); + + // "display" is not a defined property in the emails complex type + var value = new List + { + new Dictionary { ["value"] = "a@b.com", ["display"] = "Work Email" } + }; + var result = schema.TryCreateAttribute(AttributeCode.Create("emails"), (IReadOnlyList)value, out _, out var errors); + + result.ShouldBeFalse(); + _ = errors.ShouldNotBeNull(); + errors.ShouldContain(e => e.Contains("Element at index 0") && e.Contains("display") && e.Contains("is not defined in complex attribute")); + } +} diff --git a/storage/test/Storage.Tests/EntityAttributeValue/AttributeTypes.cs b/storage/test/Storage.Tests/EntityAttributeValue/AttributeTypes.cs new file mode 100644 index 000000000..8a7aff200 --- /dev/null +++ b/storage/test/Storage.Tests/EntityAttributeValue/AttributeTypes.cs @@ -0,0 +1,153 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +namespace Duende.Storage.EntityAttributeValue; + +public static class AttributeTypes +{ + [Theory] + [InlineData(ScalarDataType.Boolean)] + [InlineData(ScalarDataType.Date)] + [InlineData(ScalarDataType.DateTime)] + [InlineData(ScalarDataType.Decimal)] + [InlineData(ScalarDataType.Integer)] + [InlineData(ScalarDataType.String)] + public static void ScalarCanBeConstructedForEachDataType(ScalarDataType dataType) + { + var type = new ScalarAttributeType(dataType); + + type.DataType.ShouldBe(dataType); + } + + [Fact] + public static void ScalarRejectsInvalidDataType() + { + var ex = Record.Exception(() => _ = new ScalarAttributeType((ScalarDataType)999)); + + _ = ex.ShouldBeOfType(); + } + + [Fact] + public static void ComplexCanBeConstructedWithProperties() + { + var type = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String), + [AttributeCode.Create("zip")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }); + + type.Properties.Count.ShouldBe(2); + } + + [Fact] + public static void ComplexRejectsEmptyProperties() + { + var ex = Record.Exception(() => _ = new ComplexAttributeType(new Dictionary())); + + _ = ex.ShouldBeOfType(); + } + + [Fact] + public static void ComplexAcceptsMixedCasePropertyNames() + { + var type = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("givenName")] = ComplexAttributeProperty.Of(ScalarDataType.String), + [AttributeCode.Create("familyName")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }); + + type.Properties.Count.ShouldBe(2); + } + + [Fact] + public static void ComplexProperties_lookup_is_case_insensitive() + { + var type = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("givenName")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }); + + type.TryGetProperty("givenname", out _, out _).ShouldBeTrue(); + type.TryGetProperty("GIVENNAME", out _, out _).ShouldBeTrue(); + type.TryGetProperty("GivenName", out _, out _).ShouldBeTrue(); + } + + [Fact] + public static void ListOfScalarIsValid() + { + var type = new ListAttributeType(new ScalarAttributeType(ScalarDataType.String)); + + _ = type.ElementType.ShouldBeOfType(); + } + + [Fact] + public static void ListOfComplexIsValid() + { + var type = new ListAttributeType(new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("name")] = ComplexAttributeProperty.Of(ScalarDataType.String) + })); + + _ = type.ElementType.ShouldBeOfType(); + } + + [Fact] + public static void DirectListInListThrows() + { + var innerList = new ListAttributeType(new ScalarAttributeType(ScalarDataType.String)); + + var ex = Record.Exception(() => _ = new ListAttributeType(innerList)); + + _ = ex.ShouldBeOfType(); + } + + [Fact] + public static void ComplexContainingListContainingComplexIsValid() + { + // address { phones: List<{ number: string }> } + var type = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("phones")] = ComplexAttributeProperty.Of( + new ListAttributeType(new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("number")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }))) + }); + + _ = type.ShouldNotBeNull(); + } + + [Fact] + public static void ComplexContainingListContainingComplexContainingListThrows() + { + // Outer complex → list → inner complex → list (transitive list-in-list at depth 2) + var ex = Record.Exception(() => _ = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("outer")] = ComplexAttributeProperty.Of( + new ListAttributeType(new ComplexAttributeType(new Dictionary + { + // This inner list would be inside an outer list → invalid + [AttributeCode.Create("inner")] = ComplexAttributeProperty.Of( + new ListAttributeType(new ScalarAttributeType(ScalarDataType.String))) + }))) + })); + + _ = ex.ShouldBeOfType(); + } + + [Fact] + public static void ListOfComplexContainingComplexIsValid() + { + // List<{ address: { city: string } }> — no nested list + var type = new ListAttributeType(new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("address")] = ComplexAttributeProperty.Of( + new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String) + })) + })); + + _ = type.ShouldNotBeNull(); + } +} diff --git a/storage/test/Storage.Tests/EntityAttributeValue/AttributeValueCollectionTests.cs b/storage/test/Storage.Tests/EntityAttributeValue/AttributeValueCollectionTests.cs new file mode 100644 index 000000000..328b65a14 --- /dev/null +++ b/storage/test/Storage.Tests/EntityAttributeValue/AttributeValueCollectionTests.cs @@ -0,0 +1,277 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.EntityAttributeValue.Internal; + +namespace Duende.Storage.EntityAttributeValue; + +public static class AttributeValueCollectionTests +{ + private static readonly AttributeDescription Desc = AttributeDescription.Create("test"); + + private static AttributeSchema SchemaWith(params AttributeDefinition[] definitions) + { + var schema = new AttributeSchema(); + foreach (var def in definitions) + { + _ = schema.AddAttributeDefinition(def); + } + return schema; + } + + private static AttributeDefinition StringDef(string name) => + new(AttributeCode.Create(name), new ScalarAttributeType(ScalarDataType.String), Desc); + + // --- Set --- + + [Fact] + public static void set_adds_new_attribute() + { + var schema = SchemaWith(StringDef("color")); + var attr = schema.CreateAttribute(AttributeCode.Create("color"), "red"); + var collection = new AttributeValueCollection(); + + collection.Set(attr); + + collection.Count.ShouldBe(1); + } + + [Fact] + public static void set_replaces_existing_attribute() + { + var schema = SchemaWith(StringDef("color")); + var name = AttributeCode.Create("color"); + var collection = new AttributeValueCollection(); + + collection.Set(schema.CreateAttribute(name, "red")); + collection.Set(schema.CreateAttribute(name, "blue")); + + collection.Count.ShouldBe(1); + collection[name].UntypedValue.ShouldBe("blue"); + } + + [Fact] + public static void set_throws_for_null() + { + var collection = new AttributeValueCollection(); + + var ex = Record.Exception(() => collection.Set(null!)); + + _ = ex.ShouldBeOfType(); + } + + // --- Remove --- + + [Fact] + public static void remove_returns_true_when_present() + { + var schema = SchemaWith(StringDef("color")); + var name = AttributeCode.Create("color"); + var collection = new AttributeValueCollection(); + collection.Set(schema.CreateAttribute(name, "red")); + + var result = collection.Remove(name); + + result.ShouldBeTrue(); + collection.Count.ShouldBe(0); + } + + [Fact] + public static void remove_returns_false_when_absent() + { + var collection = new AttributeValueCollection(); + + var result = collection.Remove(AttributeCode.Create("missing")); + + result.ShouldBeFalse(); + } + + // --- Contains --- + + [Fact] + public static void contains_returns_true_when_present() + { + var schema = SchemaWith(StringDef("color")); + var name = AttributeCode.Create("color"); + var collection = new AttributeValueCollection(); + collection.Set(schema.CreateAttribute(name, "red")); + + collection.Contains(name).ShouldBeTrue(); + } + + [Fact] + public static void contains_returns_false_when_absent() + { + var collection = new AttributeValueCollection(); + + collection.Contains(AttributeCode.Create("missing")).ShouldBeFalse(); + } + + // --- TryGet --- + + [Fact] + public static void try_get_returns_true_and_value_when_present() + { + var schema = SchemaWith(StringDef("color")); + var name = AttributeCode.Create("color"); + var collection = new AttributeValueCollection(); + collection.Set(schema.CreateAttribute(name, "red")); + + var found = collection.TryGet(name, out var attribute); + + found.ShouldBeTrue(); + _ = attribute.ShouldNotBeNull(); + attribute.UntypedValue.ShouldBe("red"); + } + + [Fact] + public static void try_get_returns_false_when_absent() + { + var collection = new AttributeValueCollection(); + + var found = collection.TryGet(AttributeCode.Create("missing"), out var attribute); + + found.ShouldBeFalse(); + attribute.ShouldBeNull(); + } + + // --- Indexer --- + + [Fact] + public static void indexer_returns_attribute_when_present() + { + var schema = SchemaWith(StringDef("color")); + var name = AttributeCode.Create("color"); + var collection = new AttributeValueCollection(); + collection.Set(schema.CreateAttribute(name, "red")); + + var attribute = collection[name]; + + attribute.UntypedValue.ShouldBe("red"); + } + + [Fact] + public static void indexer_throws_when_absent() + { + var collection = new AttributeValueCollection(); + + var ex = Record.Exception(() => _ = collection[AttributeCode.Create("missing")]); + + _ = ex.ShouldBeOfType(); + } + + // --- Count --- + + [Fact] + public static void empty_collection_has_zero_count() + { + var collection = new AttributeValueCollection(); + + collection.Count.ShouldBe(0); + } + + // --- GetEnumerator --- + + [Fact] + public static void enumerator_yields_all_attributes() + { + var schema = SchemaWith(StringDef("color"), StringDef("size")); + var collection = new AttributeValueCollection(); + collection.Set(schema.CreateAttribute(AttributeCode.Create("color"), "red")); + collection.Set(schema.CreateAttribute(AttributeCode.Create("size"), "large")); + + var items = collection.ToList(); + + items.Count.ShouldBe(2); + } + + // --- Constructor duplicate rejection --- + + [Fact] + public static void constructor_rejects_duplicate_names() + { + var schema = SchemaWith(StringDef("color")); + var name = AttributeCode.Create("color"); + var attr1 = schema.CreateAttribute(name, "red"); + var attr2 = schema.CreateAttribute(name, "blue"); + + var ex = Record.Exception(() => schema.CreateAttributes([attr1, attr2])); + + _ = ex.ShouldBeOfType(); + ex.Message.ShouldContain("color"); + } + + [Fact] + public static void constructor_rejects_duplicate_names_different_casing() + { + var schema = SchemaWith(StringDef("color")); + var attr1 = schema.CreateAttribute(AttributeCode.Create("color"), "red"); + var attr2 = schema.CreateAttribute(AttributeCode.Create("Color"), "blue"); + + var ex = Record.Exception(() => schema.CreateAttributes([attr1, attr2])); + + _ = ex.ShouldBeOfType(); + } + + // --- Case-insensitive lookups --- + + [Fact] + public static void try_get_finds_attribute_with_different_casing() + { + var schema = SchemaWith(StringDef("givenName")); + var collection = new AttributeValueCollection(); + collection.Set(schema.CreateAttribute(AttributeCode.Create("givenName"), "Alice")); + + collection.TryGet(AttributeCode.Create("givenname"), out var attr).ShouldBeTrue(); + _ = attr.ShouldNotBeNull(); + attr.UntypedValue.ShouldBe("Alice"); + } + + [Fact] + public static void contains_matches_case_insensitively() + { + var schema = SchemaWith(StringDef("givenName")); + var collection = new AttributeValueCollection(); + collection.Set(schema.CreateAttribute(AttributeCode.Create("givenName"), "Alice")); + + collection.Contains(AttributeCode.Create("GIVENNAME")).ShouldBeTrue(); + collection.Contains(AttributeCode.Create("givenname")).ShouldBeTrue(); + collection.Contains(AttributeCode.Create("GivenName")).ShouldBeTrue(); + } + + [Fact] + public static void set_replaces_attribute_with_different_casing() + { + var schema = SchemaWith(StringDef("givenName")); + var collection = new AttributeValueCollection(); + collection.Set(schema.CreateAttribute(AttributeCode.Create("givenName"), "Alice")); + collection.Set(schema.CreateAttribute(AttributeCode.Create("GIVENNAME"), "Bob")); + + collection.Count.ShouldBe(1); + collection.TryGet(AttributeCode.Create("givenName"), out var attr).ShouldBeTrue(); + attr!.UntypedValue.ShouldBe("Bob"); + } + + [Fact] + public static void remove_works_with_different_casing() + { + var schema = SchemaWith(StringDef("givenName")); + var collection = new AttributeValueCollection(); + collection.Set(schema.CreateAttribute(AttributeCode.Create("givenName"), "Alice")); + + collection.Remove(AttributeCode.Create("GIVENNAME")).ShouldBeTrue(); + collection.Count.ShouldBe(0); + } + + [Fact] + public static void stored_attribute_preserves_original_casing() + { + var schema = SchemaWith(StringDef("givenName")); + var collection = new AttributeValueCollection(); + collection.Set(schema.CreateAttribute(AttributeCode.Create("givenName"), "Alice")); + + var attr = collection[AttributeCode.Create("givenname")]; + + attr.Code.Value.ShouldBe("givenName", "Original casing should be preserved in storage"); + } +} diff --git a/storage/test/Storage.Tests/EntityAttributeValue/Storage/AttributeTypeDsoRoundTripTests.cs b/storage/test/Storage.Tests/EntityAttributeValue/Storage/AttributeTypeDsoRoundTripTests.cs new file mode 100644 index 000000000..aef7a100a --- /dev/null +++ b/storage/test/Storage.Tests/EntityAttributeValue/Storage/AttributeTypeDsoRoundTripTests.cs @@ -0,0 +1,130 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using System.Text.Json; +using Duende.Storage.EntityAttributeValue.Internal.Storage; + +namespace Duende.Storage.EntityAttributeValue.Storage; + +/// +/// Verifies that all subtypes survive a round-trip +/// through the layer (and JSON serialization). +/// +public static class AttributeTypeDsoRoundTripTests +{ + private static AttributeTypeDso ToTypeDso(AttributeType type) => + type switch + { + ScalarAttributeType scalar => new AttributeTypeDso( + "Scalar", scalar.DataType.ToString(), null, null, null, null), + ComplexAttributeType complex => new AttributeTypeDso( + "Complex", null, null, null, + complex.Properties.ToDictionary(kvp => kvp.Key.Value, kvp => new ComplexPropertyDso(ToTypeDso(kvp.Value.Type), kvp.Value.DisplayName?.Value, kvp.Value.Description?.Value)), + null), + ListAttributeType list => new AttributeTypeDso( + "List", null, null, null, null, ToTypeDso(list.ElementType)), + _ => throw new InvalidOperationException() + }; + + private static AttributeType ToTypeValueObject(AttributeTypeDso dso) => + dso.Kind switch + { + "Scalar" => new ScalarAttributeType(Enum.Parse(dso.ScalarDataType!)), + "Complex" => new ComplexAttributeType( + (dso.Properties ?? []).ToDictionary( + kvp => AttributeCode.Create(kvp.Key), + kvp => ComplexAttributeProperty.Of( + ToTypeValueObject(kvp.Value.Type), + kvp.Value.DisplayName is not null ? AttributeDisplayName.Create(kvp.Value.DisplayName) : (AttributeDisplayName?)null, + kvp.Value.Description is not null ? AttributeDescription.Create(kvp.Value.Description) : (AttributeDescription?)null))), + "List" => new ListAttributeType(ToTypeValueObject(dso.ElementType!)), + _ => throw new InvalidOperationException() + }; + + private static AttributeType RoundTrip(AttributeType original) + { + var dso = ToTypeDso(original); + // Simulate JSON serialization round-trip (what the store would do) + var json = JsonSerializer.Serialize(dso); + var deserialized = JsonSerializer.Deserialize(json); + return ToTypeValueObject(deserialized!); + } + + [Fact] + public static void scalar_type_round_trips() + { + var original = new ScalarAttributeType(ScalarDataType.String); + + var result = RoundTrip(original); + + result.ShouldBe(original); + } + + [Fact] + public static void complex_type_round_trips() + { + var original = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String), + [AttributeCode.Create("zip")] = ComplexAttributeProperty.Of(ScalarDataType.Integer) + }); + + var result = RoundTrip(original); + + _ = result.ShouldBeOfType(); + var resultComplex = (ComplexAttributeType)result; + resultComplex.Properties.Count.ShouldBe(2); + _ = resultComplex.Properties[AttributeCode.Create("city")].Type.ShouldBeOfType(); + _ = resultComplex.Properties[AttributeCode.Create("zip")].Type.ShouldBeOfType(); + } + + [Fact] + public static void list_of_complex_type_round_trips() + { + var original = new ListAttributeType(new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("number")] = ComplexAttributeProperty.Of(ScalarDataType.String), + [AttributeCode.Create("type")] = ComplexAttributeProperty.Of(ScalarDataType.String) + })); + + var result = RoundTrip(original); + + _ = result.ShouldBeOfType(); + var resultList = (ListAttributeType)result; + _ = resultList.ElementType.ShouldBeOfType(); + var resultElement = (ComplexAttributeType)resultList.ElementType; + resultElement.Properties.Count.ShouldBe(2); + } + + [Fact] + public static void complex_property_metadata_round_trips() + { + var original = new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of( + new ScalarAttributeType(ScalarDataType.String), + AttributeDisplayName.Create("City"), + AttributeDescription.Create("The city name")), + [AttributeCode.Create("zip")] = ComplexAttributeProperty.Of( + new ScalarAttributeType(ScalarDataType.Integer), + AttributeDisplayName.Create("ZIP Code"), + null) + }); + + var result = RoundTrip(original); + + _ = result.ShouldBeOfType(); + var resultComplex = (ComplexAttributeType)result; + resultComplex.Properties.Count.ShouldBe(2); + + var city = resultComplex.Properties[AttributeCode.Create("city")]; + _ = city.Type.ShouldBeOfType(); + city.DisplayName.ShouldBe(AttributeDisplayName.Create("City")); + city.Description.ShouldBe(AttributeDescription.Create("The city name")); + + var zip = resultComplex.Properties[AttributeCode.Create("zip")]; + _ = zip.Type.ShouldBeOfType(); + zip.DisplayName.ShouldBe(AttributeDisplayName.Create("ZIP Code")); + zip.Description.ShouldBeNull(); + } +} diff --git a/storage/test/Storage.Tests/EntityAttributeValue/Storage/AttributeTypeResolverTests.cs b/storage/test/Storage.Tests/EntityAttributeValue/Storage/AttributeTypeResolverTests.cs new file mode 100644 index 000000000..1d060eb03 --- /dev/null +++ b/storage/test/Storage.Tests/EntityAttributeValue/Storage/AttributeTypeResolverTests.cs @@ -0,0 +1,316 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.EntityAttributeValue.Internal.Storage; +using Duende.Storage.Internal.Querying.Fields; + +namespace Duende.Storage.EntityAttributeValue.Storage; + +public static class AttributeTypeResolverTests +{ + private static readonly AttributeDescription Desc = AttributeDescription.Create("test"); + + private static AttributeTypeResolver ResolverWith(params AttributeDefinition[] definitions) + { + var dict = definitions.ToDictionary(d => d.Code, d => d); + return new AttributeTypeResolver(dict); + } + + // --- Scalar paths --- + + [Fact] + public static void scalar_string_resolves_to_string_field() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("displayname"), + new ScalarAttributeType(ScalarDataType.String), + Desc)); + + var field = resolver.ResolveField("displayname"); + + _ = field.ShouldBeOfType(); + field.Path.ShouldBe("DISPLAYNAME"); + } + + [Fact] + public static void scalar_boolean_resolves_to_boolean_field() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("active"), + new ScalarAttributeType(ScalarDataType.Boolean), + Desc)); + + var field = resolver.ResolveField("active"); + + _ = field.ShouldBeOfType(); + } + + [Fact] + public static void scalar_date_resolves_to_date_time_field() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("birthdate"), + new ScalarAttributeType(ScalarDataType.Date), + Desc)); + + var field = resolver.ResolveField("birthdate"); + + _ = field.ShouldBeOfType(); + } + + [Fact] + public static void scalar_date_time_resolves_to_date_time_field() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("recordedat"), + new ScalarAttributeType(ScalarDataType.DateTime), + Desc)); + + var field = resolver.ResolveField("recordedat"); + + _ = field.ShouldBeOfType(); + } + + [Fact] + public static void scalar_decimal_resolves_to_number_field() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("balance"), + new ScalarAttributeType(ScalarDataType.Decimal), + Desc)); + + var field = resolver.ResolveField("balance"); + + _ = field.ShouldBeOfType(); + } + + [Fact] + public static void scalar_integer_resolves_to_number_field() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("age"), + new ScalarAttributeType(ScalarDataType.Integer), + Desc)); + + var field = resolver.ResolveField("age"); + + _ = field.ShouldBeOfType(); + } + + // --- Built-in username --- + + [Fact] + public static void username_resolves_to_string_field() + { + var resolver = ResolverWith(); // no schema definitions needed + + var field = resolver.ResolveField("userName"); + + _ = field.ShouldBeOfType(); + field.Path.ShouldBe("USERNAME"); + } + + [Fact] + public static void username_case_insensitive() + { + var resolver = ResolverWith(); + + var field = resolver.ResolveField("USERNAME"); + + _ = field.ShouldBeOfType(); + field.Path.ShouldBe("USERNAME"); + } + + // --- Case normalization --- + + [Fact] + public static void mixed_case_attribute_is_normalized() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("displayname"), + new ScalarAttributeType(ScalarDataType.String), + Desc)); + + var field = resolver.ResolveField("DisplayName"); + + _ = field.ShouldBeOfType(); + field.Path.ShouldBe("DISPLAYNAME"); + } + + // --- Dotted complex paths --- + + [Fact] + public static void complex_sub_property_resolves_to_correct_field_type() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("address"), + new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String), + [AttributeCode.Create("zip")] = ComplexAttributeProperty.Of(ScalarDataType.Integer) + }), + Desc)); + + var cityField = resolver.ResolveField("address.city"); + var zipField = resolver.ResolveField("address.zip"); + + _ = cityField.ShouldBeOfType(); + _ = zipField.ShouldBeOfType(); + } + + // --- List paths --- + + [Fact] + public static void list_of_scalar_resolves_to_multi_valued_string_field() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("tags"), + new ListAttributeType(new ScalarAttributeType(ScalarDataType.String)), + Desc)); + + var field = resolver.ResolveField("tags"); + + _ = field.ShouldBeOfType(); + field.IsMultiValued.ShouldBeTrue(); + } + + [Fact] + public static void list_of_boolean_resolves_to_multi_valued_boolean_field() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("flags"), + new ListAttributeType(new ScalarAttributeType(ScalarDataType.Boolean)), + Desc)); + + var field = resolver.ResolveField("flags"); + + _ = field.ShouldBeOfType(); + field.IsMultiValued.ShouldBeTrue(); + } + + // --- List-of-complex dotted paths --- + + [Fact] + public static void list_of_complex_sub_property_resolves_to_multi_valued_string_field() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("phones"), + new ListAttributeType(new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("number")] = ComplexAttributeProperty.Of(ScalarDataType.String), + [AttributeCode.Create("type")] = ComplexAttributeProperty.Of(ScalarDataType.String) + })), + Desc)); + + var field = resolver.ResolveField("phones.number"); + + // Inside a list, scalars resolve to their native type with IsMultiValued = true + _ = field.ShouldBeOfType(); + field.IsMultiValued.ShouldBeTrue(); + } + + [Fact] + public static void list_of_complex_integer_sub_property_resolves_to_multi_valued_number_field() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("entries"), + new ListAttributeType(new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("rank")] = ComplexAttributeProperty.Of(ScalarDataType.Integer) + })), + Desc)); + + var field = resolver.ResolveField("entries.rank"); + + // Array context: integer becomes multi-valued NumberField + _ = field.ShouldBeOfType(); + field.IsMultiValued.ShouldBeTrue(); + } + + // --- Error cases --- + + [Fact] + public static void unknown_attribute_throws_not_supported() + { + var resolver = ResolverWith(); + + var ex = Record.Exception(() => resolver.ResolveField("nosuchattr")); + + _ = ex.ShouldBeOfType(); + ex.Message.ShouldContain("nosuchattr"); + } + + [Fact] + public static void unknown_sub_property_throws_not_supported() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("address"), + new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }), + Desc)); + + var ex = Record.Exception(() => resolver.ResolveField("address.country")); + + _ = ex.ShouldBeOfType(); + ex.Message.ShouldContain("country"); + } + + [Fact] + public static void direct_complex_query_throws_not_supported() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("address"), + new ComplexAttributeType(new Dictionary + { + [AttributeCode.Create("city")] = ComplexAttributeProperty.Of(ScalarDataType.String) + }), + Desc)); + + var ex = Record.Exception(() => resolver.ResolveField("address")); + + _ = ex.ShouldBeOfType(); + ex.Message.ShouldContain("complex"); + } + + [Fact] + public static void navigating_into_scalar_throws_not_supported() + { + var resolver = ResolverWith( + new AttributeDefinition( + AttributeCode.Create("displayname"), + new ScalarAttributeType(ScalarDataType.String), + Desc)); + + var ex = Record.Exception(() => resolver.ResolveField("displayname.sub")); + + _ = ex.ShouldBeOfType(); + ex.Message.ShouldContain("sub"); + } + + [Fact] + public static void undefined_attribute_throws_not_supported() + { + var resolver = ResolverWith(); + + // The field is treated as unknown because it is not defined in the schema. + var ex = Record.Exception(() => resolver.ResolveField("INVALID_NAME")); + + _ = ex.ShouldBeOfType(); + } +} diff --git a/storage/test/Storage.Tests/ExpirationTests.cs b/storage/test/Storage.Tests/ExpirationTests.cs new file mode 100644 index 000000000..3304b9f89 --- /dev/null +++ b/storage/test/Storage.Tests/ExpirationTests.cs @@ -0,0 +1,63 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal; + +namespace Duende.Storage; + +public sealed class ExpirationTests +{ + [Fact] + public void Absolute_expiration_resolve_should_return_expires_at() + { + var expiresAt = new DateTimeOffset(2030, 6, 15, 12, 0, 0, TimeSpan.Zero); + var expiration = Expiration.AtAbsolute(expiresAt); + + var resolved = expiration.Resolve(TimeProvider.System); + + resolved.ShouldBe(expiresAt); + } + + [Fact] + public void Relative_expiration_resolve_should_return_now_plus_lifetime() + { + var tp = new FakeTimeProvider(new DateTimeOffset(2025, 1, 1, 0, 0, 0, TimeSpan.Zero)); + var lifetime = TimeSpan.FromHours(2); + var expiration = Expiration.InRelative(lifetime); + + var resolved = expiration.Resolve(tp); + + resolved.ShouldBe(tp.GetUtcNow() + lifetime); + } + + [Fact] + public void Never_expiration_resolve_should_return_null() + { + var resolved = Expiration.NoExpiration.Resolve(TimeProvider.System); + + resolved.ShouldBeNull(); + } + + [Fact] + public void Absolute_expiration_non_utc_offset_should_throw_ArgumentException() + { + var nonUtc = new DateTimeOffset(2030, 6, 15, 12, 0, 0, TimeSpan.FromHours(5)); + + _ = Should.Throw(() => Expiration.AtAbsolute(nonUtc)); + } + + [Fact] + public void Relative_expiration_zero_TimeSpan_should_throw_ArgumentOutOfRangeException() => Should.Throw(() => Expiration.InRelative(TimeSpan.Zero)); + + [Fact] + public void Relative_expiration_negative_TimeSpan_should_throw_ArgumentOutOfRangeException() => Should.Throw(() => Expiration.InRelative(TimeSpan.FromMinutes(-5))); + + [Fact] + public void No_expiration_should_be_singleton() + { + var a = Expiration.NoExpiration; + var b = Expiration.NoExpiration; + + ReferenceEquals(a, b).ShouldBeTrue(); + } +} diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.all_expression.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.all_expression.verified.txt new file mode 100644 index 000000000..c21d5ed3e --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.all_expression.verified.txt @@ -0,0 +1,4 @@ +SQL: +1 + +Parameters: diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.and_expression.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.and_expression.verified.txt new file mode 100644 index 000000000..ff13817f2 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.and_expression.verified.txt @@ -0,0 +1,24 @@ +SQL: +(EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.guid_value = @value_0 +)) AND (EXISTS ( + SELECT 1 FROM main.search_values sv1 + WHERE sv1.entity_type_id = v.entity_type_id + AND sv1.pool_id = v.pool_id + AND sv1.entity_id = v.entity_id + AND sv1.field_path = @field_path_1 + AND sv1.item_index = -1 + AND sv1.number_value > @value_1 +)) + +Parameters: +@field_path_0 = 327b0827-ee9d-e8ad-28ed-d652d45461b2 +@value_0 = 3018b8b6-434a-919f-fc90-b3fe8dd35be8 +@field_path_1 = 463e2156-a7ab-74fd-11eb-0b25ecf7bce8 +@value_1 = 18 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.array_contains_expression.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.array_contains_expression.verified.txt new file mode 100644 index 000000000..b0de63b4b --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.array_contains_expression.verified.txt @@ -0,0 +1,14 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index >= 0 + AND sv0.guid_value = @value_0 +) + +Parameters: +@field_path_0 = 0b19472a-1395-657e-194f-279e53519b64 +@value_0 = 243a4abb-021d-033b-1e53-d235fc679a1b \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.array_filter_expression_and_conditions.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.array_filter_expression_and_conditions.verified.txt new file mode 100644 index 000000000..6e802be5c --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.array_filter_expression_and_conditions.verified.txt @@ -0,0 +1,23 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0_0 + INNER JOIN main.search_values sv0_1 + ON sv0_0.entity_type_id = sv0_1.entity_type_id + AND sv0_0.pool_id = sv0_1.pool_id + AND sv0_0.entity_id = sv0_1.entity_id + AND sv0_0.item_index = sv0_1.item_index + WHERE sv0_0.entity_type_id = v.entity_type_id + AND sv0_0.pool_id = v.pool_id + AND sv0_0.entity_id = v.entity_id + AND sv0_0.item_index IS NOT NULL + AND sv0_0.field_path = @array_field_0_0 + AND sv0_0.guid_value = @array_value_0_0 + AND sv0_1.field_path = @array_field_0_1 + AND sv0_1.string_value LIKE @array_value_0_1 ESCAPE '\' +) + +Parameters: +@array_field_0_0 = 60489294-2cba-2d42-60fd-0617941bf915 +@array_value_0_0 = fbf1209f-d44b-8163-44d4-f642b3159812 +@array_field_0_1 = 45aa6524-c732-d63a-ada3-56ad96a7fab6 +@array_value_0_1 = %@EXAMPLE.COM% \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.array_filter_expression_or_conditions.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.array_filter_expression_or_conditions.verified.txt new file mode 100644 index 000000000..13819997e --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.array_filter_expression_or_conditions.verified.txt @@ -0,0 +1,24 @@ +SQL: +(EXISTS ( + SELECT 1 FROM main.search_values sv0_0 + WHERE sv0_0.entity_type_id = v.entity_type_id + AND sv0_0.pool_id = v.pool_id + AND sv0_0.entity_id = v.entity_id + AND sv0_0.item_index IS NOT NULL + AND sv0_0.field_path = @array_field_0_0 + AND sv0_0.guid_value = @array_value_0_0 +)) OR (EXISTS ( + SELECT 1 FROM main.search_values sv1_0 + WHERE sv1_0.entity_type_id = v.entity_type_id + AND sv1_0.pool_id = v.pool_id + AND sv1_0.entity_id = v.entity_id + AND sv1_0.item_index IS NOT NULL + AND sv1_0.field_path = @array_field_1_0 + AND sv1_0.guid_value = @array_value_1_0 +)) + +Parameters: +@array_field_0_0 = 60489294-2cba-2d42-60fd-0617941bf915 +@array_value_0_0 = fbf1209f-d44b-8163-44d4-f642b3159812 +@array_field_1_0 = 60489294-2cba-2d42-60fd-0617941bf915 +@array_value_1_0 = dd330571-9d87-20d1-2e5c-73b27705bf02 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.array_filter_expression_single_condition.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.array_filter_expression_single_condition.verified.txt new file mode 100644 index 000000000..c636a5701 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.array_filter_expression_single_condition.verified.txt @@ -0,0 +1,14 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0_0 + WHERE sv0_0.entity_type_id = v.entity_type_id + AND sv0_0.pool_id = v.pool_id + AND sv0_0.entity_id = v.entity_id + AND sv0_0.item_index IS NOT NULL + AND sv0_0.field_path = @array_field_0_0 + AND sv0_0.guid_value = @array_value_0_0 +) + +Parameters: +@array_field_0_0 = 60489294-2cba-2d42-60fd-0617941bf915 +@array_value_0_0 = fbf1209f-d44b-8163-44d4-f642b3159812 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.between_expression.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.between_expression.verified.txt new file mode 100644 index 000000000..370b7f02c --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.between_expression.verified.txt @@ -0,0 +1,15 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.number_value BETWEEN @min_0 AND @max_0 +) + +Parameters: +@field_path_0 = 463e2156-a7ab-74fd-11eb-0b25ecf7bce8 +@min_0 = 18 +@max_0 = 65 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.between_expression_system_field.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.between_expression_system_field.verified.txt new file mode 100644 index 000000000..92af0b5a0 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.between_expression_system_field.verified.txt @@ -0,0 +1,6 @@ +SQL: +v.created_at BETWEEN @min_0 AND @max_0 + +Parameters: +@min_0 = 2024-01-01T00:00:00.0000000Z +@max_0 = 2024-12-31T00:00:00.0000000Z \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.contains_expression.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.contains_expression.verified.txt new file mode 100644 index 000000000..79ad509ce --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.contains_expression.verified.txt @@ -0,0 +1,14 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.string_value LIKE @value_0 ESCAPE '\' +) + +Parameters: +@field_path_0 = 4c7c61fb-913b-1c70-b0ff-ba5fd7c3c289 +@value_0 = %ALICE% \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.contains_expression_with_wildcards.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.contains_expression_with_wildcards.verified.txt new file mode 100644 index 000000000..1b0033f0d --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.contains_expression_with_wildcards.verified.txt @@ -0,0 +1,14 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.string_value LIKE @value_0 ESCAPE '\' +) + +Parameters: +@field_path_0 = 4c7c61fb-913b-1c70-b0ff-ba5fd7c3c289 +@value_0 = %AL\%ICE\_TEST% \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.cs b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.cs new file mode 100644 index 000000000..b1ae1a4e1 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.cs @@ -0,0 +1,341 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Duende.Storage.Internal.Querying.Expressions; +using Duende.Storage.Internal.Querying.Fields; +using Duende.Storage.Sqlite.Internal; +using Microsoft.Data.Sqlite; + +namespace Duende.Storage.Internal.Querying; + +public sealed class SqlWhereClauseBuilderTests : VerifyBase +{ + public SqlWhereClauseBuilderTests() : base() + { + } + + private static readonly ISqlDialect Dialect = new SqliteDialect(); + private const string Schema = "main"; + + private static (SqlWhereClauseBuilder Builder, SqliteCommand Command) CreateBuilder() + { + var command = new SqliteCommand(); + var builder = new SqlWhereClauseBuilder(Schema, command, Dialect); + return (builder, command); + } + + private static string FormatValue(object? value) => value switch + { + byte[] bytes when bytes.Length == 16 => new Guid(bytes).ToString(), + _ => value?.ToString() ?? "NULL" + }; + + private static string BuildAndDescribeParameters(SqlWhereClauseBuilder builder, SqliteCommand command, IQueryExpression expression) + { + var sql = builder.BuildWhereClause(expression); + var parameters = command.Parameters + .Cast() + .Select(p => $"{p.ParameterName} = {FormatValue(p.Value)}") + .ToList(); + return $"SQL:\n{sql}\n\nParameters:\n{string.Join("\n", parameters)}"; + } + + [Fact] + public async Task all_expression() + { + var (builder, command) = CreateBuilder(); + var result = BuildAndDescribeParameters(builder, command, AllExpression.Instance); + _ = await Verify(result); + } + + [Fact] + public async Task equal_expression_string_field() + { + var (builder, command) = CreateBuilder(); + var field = new StringField("userName"); + var expression = field.Equals("alice"); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task equal_expression_number_field() + { + var (builder, command) = CreateBuilder(); + var field = new NumberField("age"); + var expression = field.Equals(42m); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task equal_expression_system_field_created() + { + var (builder, command) = CreateBuilder(); + var field = new DateTimeField(SystemFields.Created); + var expression = new EqualExpression(field, new DateTime(2024, 1, 15, 0, 0, 0, DateTimeKind.Utc)); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task contains_expression() + { + var (builder, command) = CreateBuilder(); + var field = new StringField("displayName"); + var expression = field.Contains("alice"); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task contains_expression_with_wildcards() + { + var (builder, command) = CreateBuilder(); + var field = new StringField("displayName"); + var expression = field.Contains("al%ice_test"); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task starts_with_expression() + { + var (builder, command) = CreateBuilder(); + var field = new StringField("userName"); + var expression = field.StartsWith("ali"); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task ends_with_expression() + { + var (builder, command) = CreateBuilder(); + var field = new StringField("email"); + var expression = field.EndsWith("@example.com"); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task greater_than_expression() + { + var (builder, command) = CreateBuilder(); + var field = new NumberField("age"); + var expression = field.GreaterThan(18m); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task greater_than_expression_system_field() + { + var (builder, command) = CreateBuilder(); + var field = new DateTimeField(SystemFields.LastUpdated); + var expression = new GreaterThanExpression(field, new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc)); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task less_than_expression() + { + var (builder, command) = CreateBuilder(); + var field = new NumberField("score"); + var expression = field.LessThan(100m); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task greater_or_equal_expression() + { + var (builder, command) = CreateBuilder(); + var field = new NumberField("age"); + var expression = field.GreaterOrEqual(18m); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task less_or_equal_expression() + { + var (builder, command) = CreateBuilder(); + var field = new NumberField("age"); + var expression = field.LessOrEqual(65m); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task between_expression() + { + var (builder, command) = CreateBuilder(); + var field = new NumberField("age"); + var expression = field.Between(18m, 65m); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task between_expression_system_field() + { + var (builder, command) = CreateBuilder(); + var field = new DateTimeField(SystemFields.Created); + var expression = new BetweenExpression(field, + new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc), + new DateTime(2024, 12, 31, 0, 0, 0, DateTimeKind.Utc)); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task in_expression_string_field() + { + var (builder, command) = CreateBuilder(); + var field = new StringField("status"); + var expression = field.In(["active", "pending"]); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task in_expression_number_field() + { + var (builder, command) = CreateBuilder(); + var field = new NumberField("priority"); + var expression = field.In([1m, 2m, 3m]); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task in_expression_empty_list() + { + var (builder, command) = CreateBuilder(); + var field = new NumberField("priority"); + var expression = field.In([]); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task in_expression_system_field() + { + var (builder, command) = CreateBuilder(); + var field = new DateTimeField(SystemFields.Created); + var expression = new InExpression(field, new[] + { + new DateTime(2024, 1, 1, 0, 0, 0, DateTimeKind.Utc), + new DateTime(2024, 6, 1, 0, 0, 0, DateTimeKind.Utc) + }); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task present_expression_scalar_field() + { + var (builder, command) = CreateBuilder(); + var field = new StringField("email"); + var expression = new PresentExpression(field); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task present_expression_array_field() + { + var (builder, command) = CreateBuilder(); + var field = new StringArrayField("emails"); + var expression = new PresentExpression(field); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task present_expression_system_field() + { + var (builder, command) = CreateBuilder(); + var field = new DateTimeField(SystemFields.Created); + var expression = new PresentExpression(field); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task array_contains_expression() + { + var (builder, command) = CreateBuilder(); + var field = new StringArrayField("emails"); + var expression = field.Contains("alice@example.com"); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task not_expression() + { + var (builder, command) = CreateBuilder(); + var field = new StringField("status"); + var inner = field.Equals("inactive"); + var expression = new NotExpression(inner); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task and_expression() + { + var (builder, command) = CreateBuilder(); + var nameField = new StringField("userName"); + var ageField = new NumberField("age"); + var expression = new AndExpression([nameField.Equals("alice"), ageField.GreaterThan(18m)]); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task or_expression() + { + var (builder, command) = CreateBuilder(); + var field = new StringField("status"); + var expression = new OrExpression([field.Equals("active"), field.Equals("pending")]); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task array_filter_expression_single_condition() + { + var (builder, command) = CreateBuilder(); + var typeField = new StringField("type"); + var expression = new ArrayFilterExpression("emails", typeField.Equals("work")); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task array_filter_expression_and_conditions() + { + var (builder, command) = CreateBuilder(); + var typeField = new StringField("type"); + var valueField = new StringField("value"); + var filter = new AndExpression([typeField.Equals("work"), valueField.Contains("@example.com")]); + var expression = new ArrayFilterExpression("emails", filter); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } + + [Fact] + public async Task array_filter_expression_or_conditions() + { + var (builder, command) = CreateBuilder(); + var typeField = new StringField("type"); + var filter = new OrExpression([typeField.Equals("work"), typeField.Equals("home")]); + var expression = new ArrayFilterExpression("emails", filter); + var result = BuildAndDescribeParameters(builder, command, expression); + _ = await Verify(result); + } +} diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.ends_with_expression.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.ends_with_expression.verified.txt new file mode 100644 index 000000000..df06a527c --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.ends_with_expression.verified.txt @@ -0,0 +1,14 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.string_value LIKE @value_0 ESCAPE '\' +) + +Parameters: +@field_path_0 = a349a661-283f-e569-e35f-bb7aff3a80d9 +@value_0 = %@EXAMPLE.COM \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.equal_expression_number_field.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.equal_expression_number_field.verified.txt new file mode 100644 index 000000000..106f54205 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.equal_expression_number_field.verified.txt @@ -0,0 +1,14 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.number_value = @value_0 +) + +Parameters: +@field_path_0 = 463e2156-a7ab-74fd-11eb-0b25ecf7bce8 +@value_0 = 42 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.equal_expression_string_field.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.equal_expression_string_field.verified.txt new file mode 100644 index 000000000..1f692ddd4 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.equal_expression_string_field.verified.txt @@ -0,0 +1,14 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.guid_value = @value_0 +) + +Parameters: +@field_path_0 = 327b0827-ee9d-e8ad-28ed-d652d45461b2 +@value_0 = 3018b8b6-434a-919f-fc90-b3fe8dd35be8 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.equal_expression_system_field_created.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.equal_expression_system_field_created.verified.txt new file mode 100644 index 000000000..37deeaf49 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.equal_expression_system_field_created.verified.txt @@ -0,0 +1,5 @@ +SQL: +v.created_at = @value_0 + +Parameters: +@value_0 = 2024-01-15T00:00:00.0000000Z \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.greater_or_equal_expression.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.greater_or_equal_expression.verified.txt new file mode 100644 index 000000000..29a9ec0c4 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.greater_or_equal_expression.verified.txt @@ -0,0 +1,14 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.number_value >= @value_0 +) + +Parameters: +@field_path_0 = 463e2156-a7ab-74fd-11eb-0b25ecf7bce8 +@value_0 = 18 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.greater_than_expression.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.greater_than_expression.verified.txt new file mode 100644 index 000000000..c00abeb88 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.greater_than_expression.verified.txt @@ -0,0 +1,14 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.number_value > @value_0 +) + +Parameters: +@field_path_0 = 463e2156-a7ab-74fd-11eb-0b25ecf7bce8 +@value_0 = 18 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.greater_than_expression_system_field.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.greater_than_expression_system_field.verified.txt new file mode 100644 index 000000000..f8c0cec32 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.greater_than_expression_system_field.verified.txt @@ -0,0 +1,5 @@ +SQL: +v.last_updated_at > @value_0 + +Parameters: +@value_0 = 2024-01-01T00:00:00.0000000Z \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.in_expression_empty_list.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.in_expression_empty_list.verified.txt new file mode 100644 index 000000000..b4b20a29a --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.in_expression_empty_list.verified.txt @@ -0,0 +1,4 @@ +SQL: +0 + +Parameters: diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.in_expression_number_field.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.in_expression_number_field.verified.txt new file mode 100644 index 000000000..2eca04857 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.in_expression_number_field.verified.txt @@ -0,0 +1,16 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.number_value IN (@in_value_0_0, @in_value_0_1, @in_value_0_2) +) + +Parameters: +@field_path_0 = b75df5ca-492c-9e96-2ed2-70adbcd0a8f5 +@in_value_0_0 = 1 +@in_value_0_1 = 2 +@in_value_0_2 = 3 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.in_expression_string_field.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.in_expression_string_field.verified.txt new file mode 100644 index 000000000..49e103354 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.in_expression_string_field.verified.txt @@ -0,0 +1,15 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.guid_value IN (@in_value_0_0, @in_value_0_1) +) + +Parameters: +@field_path_0 = 8c1c245f-988f-3c5b-51e0-5d39cf030f4c +@in_value_0_0 = f474ff18-a43d-c510-529f-7d6fca84f115 +@in_value_0_1 = e1069fc6-b0a9-d116-3390-7b4e5f5864d2 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.in_expression_system_field.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.in_expression_system_field.verified.txt new file mode 100644 index 000000000..6b803e04c --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.in_expression_system_field.verified.txt @@ -0,0 +1,6 @@ +SQL: +v.created_at IN (@in_value_0_0, @in_value_0_1) + +Parameters: +@in_value_0_0 = 2024-01-01T00:00:00.0000000Z +@in_value_0_1 = 2024-06-01T00:00:00.0000000Z \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.less_or_equal_expression.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.less_or_equal_expression.verified.txt new file mode 100644 index 000000000..0e48b08ea --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.less_or_equal_expression.verified.txt @@ -0,0 +1,14 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.number_value <= @value_0 +) + +Parameters: +@field_path_0 = 463e2156-a7ab-74fd-11eb-0b25ecf7bce8 +@value_0 = 65 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.less_than_expression.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.less_than_expression.verified.txt new file mode 100644 index 000000000..aafc02f7b --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.less_than_expression.verified.txt @@ -0,0 +1,14 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.number_value < @value_0 +) + +Parameters: +@field_path_0 = 1f28b2c9-fda6-3840-5544-3e6555aa16a9 +@value_0 = 100 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.not_expression.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.not_expression.verified.txt new file mode 100644 index 000000000..9c6b3103f --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.not_expression.verified.txt @@ -0,0 +1,14 @@ +SQL: +NOT (EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.guid_value = @value_0 +)) + +Parameters: +@field_path_0 = 8c1c245f-988f-3c5b-51e0-5d39cf030f4c +@value_0 = 4333276b-54c4-c5f9-3dcf-c9c4ccf171d2 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.or_expression.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.or_expression.verified.txt new file mode 100644 index 000000000..8eae553b5 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.or_expression.verified.txt @@ -0,0 +1,24 @@ +SQL: +(EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.guid_value = @value_0 +)) OR (EXISTS ( + SELECT 1 FROM main.search_values sv1 + WHERE sv1.entity_type_id = v.entity_type_id + AND sv1.pool_id = v.pool_id + AND sv1.entity_id = v.entity_id + AND sv1.field_path = @field_path_1 + AND sv1.item_index = -1 + AND sv1.guid_value = @value_1 +)) + +Parameters: +@field_path_0 = 8c1c245f-988f-3c5b-51e0-5d39cf030f4c +@value_0 = f474ff18-a43d-c510-529f-7d6fca84f115 +@field_path_1 = 8c1c245f-988f-3c5b-51e0-5d39cf030f4c +@value_1 = e1069fc6-b0a9-d116-3390-7b4e5f5864d2 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.present_expression_array_field.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.present_expression_array_field.verified.txt new file mode 100644 index 000000000..0ede42ee9 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.present_expression_array_field.verified.txt @@ -0,0 +1,12 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index >= 0 +) + +Parameters: +@field_path_0 = 0b19472a-1395-657e-194f-279e53519b64 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.present_expression_scalar_field.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.present_expression_scalar_field.verified.txt new file mode 100644 index 000000000..ed6627f0f --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.present_expression_scalar_field.verified.txt @@ -0,0 +1,13 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.string_value IS NOT NULL +) + +Parameters: +@field_path_0 = a349a661-283f-e569-e35f-bb7aff3a80d9 \ No newline at end of file diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.present_expression_system_field.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.present_expression_system_field.verified.txt new file mode 100644 index 000000000..d009cae30 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.present_expression_system_field.verified.txt @@ -0,0 +1,4 @@ +SQL: +1=1 + +Parameters: diff --git a/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.starts_with_expression.verified.txt b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.starts_with_expression.verified.txt new file mode 100644 index 000000000..51b4f82c1 --- /dev/null +++ b/storage/test/Storage.Tests/Internal/Querying/SqlWhereClauseBuilderTests.starts_with_expression.verified.txt @@ -0,0 +1,14 @@ +SQL: +EXISTS ( + SELECT 1 FROM main.search_values sv0 + WHERE sv0.entity_type_id = v.entity_type_id + AND sv0.pool_id = v.pool_id + AND sv0.entity_id = v.entity_id + AND sv0.field_path = @field_path_0 + AND sv0.item_index = -1 + AND sv0.string_value LIKE @value_0 ESCAPE '\' +) + +Parameters: +@field_path_0 = 327b0827-ee9d-e8ad-28ed-d652d45461b2 +@value_0 = ALI% \ No newline at end of file diff --git a/storage/test/Storage.Tests/Storage.Tests.csproj b/storage/test/Storage.Tests/Storage.Tests.csproj new file mode 100644 index 000000000..eb223febe --- /dev/null +++ b/storage/test/Storage.Tests/Storage.Tests.csproj @@ -0,0 +1,18 @@ + + + + Duende.Storage + + + + + + + + + + + + + + diff --git a/storage/testing/Directory.Build.props b/storage/testing/Directory.Build.props new file mode 100644 index 000000000..106b12775 --- /dev/null +++ b/storage/testing/Directory.Build.props @@ -0,0 +1,3 @@ + + + diff --git a/storage/testing/TestAppHost/Program.cs b/storage/testing/TestAppHost/Program.cs new file mode 100644 index 000000000..ac828a40c --- /dev/null +++ b/storage/testing/TestAppHost/Program.cs @@ -0,0 +1,108 @@ +// Copyright (c) Duende Software. All rights reserved. +// See LICENSE in the project root for license information. + +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Diagnostics.HealthChecks; +using Microsoft.Extensions.Logging; + +var isWarmup = args.Length > 0 && args[0] == "-warmup"; + +// Determine which resources to start. Sources (in priority order): +// 1. Warmup CLI args: `-warmup postgresql` or `-warmup sqlserver` +// 2. Environment variable: TESTAPPHOST_RESOURCES=postgresql or TESTAPPHOST_RESOURCES=sqlserver +// 3. Default: start all resources +var resources = isWarmup + ? args.Skip(1).ToHashSet(StringComparer.OrdinalIgnoreCase) + : (Environment.GetEnvironmentVariable("TESTAPPHOST_RESOURCES") is { Length: > 0 } envValue + ? envValue.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .ToHashSet(StringComparer.OrdinalIgnoreCase) + : []); +var startAll = resources.Count == 0; + +var options = new DistributedApplicationOptions +{ + Args = isWarmup ? [] : args, + DisableDashboard = isWarmup, +}; + +var builder = DistributedApplication.CreateBuilder(options); + +// Suppress default health check logs — they log full stack traces at Error level on every +// failed poll while containers are starting. We replace with a custom listener below that +// logs a concise info message per failed poll instead. +builder.Services.AddLogging(logging => +{ + _ = logging.AddFilter("Microsoft.Extensions.Diagnostics.HealthChecks", LogLevel.None); + // Suppress noisy GSSAPI and checkpoint logs from the PostgreSQL container resource + _ = logging.AddFilter("Duende.TestAppHost.Resources.postgresql", LogLevel.None); +}); + +// Use fixed passwords so the warmup step and test app agree on credentials +// when reusing persistent containers. +if (startAll || resources.Contains("sqlserver")) +{ + var sqlPassword = builder.AddParameter("sqlserver-password", "DuendeTests!1"); + _ = builder.AddSqlServer("sqlserver", sqlPassword, port: 37834) + .WithLifetime(ContainerLifetime.Persistent) + .WithContainerName("duende-storage-sqlserver"); +} + +if (startAll || resources.Contains("postgresql")) +{ + var pgPassword = builder.AddParameter("postgresql-password", "DuendeTests!1"); + _ = builder.AddPostgres("postgresql", password: pgPassword, port: 37833) + .WithArgs( + "-c", "max_connections=500", // increase max connections so fast concurrent execution of tests doesn't run out of connections + "-c", "shared_buffers=512MB", // the primary cache for data (25% of total RAM is the rule of thumb) + "-c", "work_mem=16MB") // memory per sort/join operation (useful for complex queries) + .WithLifetime(ContainerLifetime.Persistent) + .WithContainerName("duende-storage-postgresql"); +} + +var app = builder.Build(); + +if (isWarmup) +{ + await app.StartAsync(); + + var logger = app.Services.GetRequiredService().CreateLogger("Warmup"); + var notificationService = app.Services.GetRequiredService(); + + if (startAll || resources.Contains("sqlserver")) + { + await WaitForHealthyAsync("sqlserver"); + } + + if (startAll || resources.Contains("postgresql")) + { + await WaitForHealthyAsync("postgresql"); + } + + await app.StopAsync(); + + async Task WaitForHealthyAsync(string resourceName) + { + using var cts = new CancellationTokenSource(TimeSpan.FromMinutes(5)); + await foreach (var notification in notificationService.WatchAsync(cts.Token)) + { + if (notification.Resource.Name != resourceName) + { + continue; + } + + if (notification.Snapshot.HealthStatus == HealthStatus.Healthy) + { + logger.LogInformation("{Resource} is healthy", resourceName); + return; + } + + var state = notification.Snapshot.State?.Text ?? "unknown"; + logger.LogInformation("{Resource} not yet healthy (state: {State}), waiting...", + resourceName, state); + } + } +} +else +{ + await app.RunAsync(); +} diff --git a/storage/testing/TestAppHost/Properties/launchSettings.json b/storage/testing/TestAppHost/Properties/launchSettings.json new file mode 100644 index 000000000..16dc83a4f --- /dev/null +++ b/storage/testing/TestAppHost/Properties/launchSettings.json @@ -0,0 +1,17 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "default": { + "commandName": "Project", + "dotnetRunMessages": true, + "launchBrowser": true, + "applicationUrl": "https://localhost:17160", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development", + "DOTNET_ENVIRONMENT": "Development", + "DOTNET_DASHBOARD_OTLP_ENDPOINT_URL": "https://localhost:21146", + "DOTNET_RESOURCE_SERVICE_ENDPOINT_URL": "https://localhost:22151" + } + } + } +} diff --git a/storage/testing/TestAppHost/TestAppHost.csproj b/storage/testing/TestAppHost/TestAppHost.csproj new file mode 100644 index 000000000..32682dd96 --- /dev/null +++ b/storage/testing/TestAppHost/TestAppHost.csproj @@ -0,0 +1,18 @@ + + + + true + Exe + storage-test-app-host + + + + $(NoWarn);CA1848;CA1873 + + + + + + + +