From 04d427cbeb41095e09af7860c338a62d2102c127 Mon Sep 17 00:00:00 2001 From: Michael Green <84688932+michael-j-green@users.noreply.github.com> Date: Thu, 26 Mar 2026 16:14:50 +1100 Subject: [PATCH] Refactor file handling to improve performance (#701) Refactor image file handling to use PhysicalFile for better performance and add ETag for caching. Optimize metadata fetching, query performance, and memory usage in various controllers. Many background tasks are now moved to a discrete process so as to not interfere with the main process. --- .vscode/launch.json | 5 +- gaseous-lib/Classes/Common.cs | 305 ++++++++++++++++++ .../Classes/Database/DatabaseMigration.cs | 14 +- gaseous-lib/Classes/Filters.cs | 103 +++--- gaseous-lib/Classes/GameLibrary.cs | 1 - gaseous-lib/Classes/HTTPComms.cs | 157 +++++---- gaseous-lib/Classes/HashObject.cs | 40 +++ gaseous-lib/Classes/ImportGames.cs | 244 +++++++++++--- gaseous-lib/Classes/Metadata/Games.cs | 117 ++++--- gaseous-lib/Classes/Metadata/Images.cs | 13 +- gaseous-lib/Classes/MetadataManagement.cs | 20 +- .../Plugins/FileSignatures/FileSignature.cs | 247 +++++++++----- .../FileSignaturePlugins/Hasheous.cs | 27 +- .../Classes/ProcessQueue/ProcessQueue.cs | 23 +- .../Classes/ProcessQueue/QueueItemStatus.cs | 9 +- .../Classes/UnknownEnumFallbackConverter.cs | 118 +++++++ gaseous-lib/gaseous-lib.csproj | 28 +- gaseous-processhost/Program.cs | 10 +- gaseous-server.Tests/HTTPCommsTests.cs | 142 +++++++- .../gaseous-server.Tests.csproj | 2 +- .../Controllers/V1.0/BiosController.cs | 3 +- .../Controllers/V1.0/GamesController.cs | 40 ++- .../Controllers/V1.0/PlatformsController.cs | 25 +- .../Controllers/V1.1/GamesController.cs | 200 ++++++------ 24 files changed, 1408 insertions(+), 485 deletions(-) create mode 100644 gaseous-lib/Classes/UnknownEnumFallbackConverter.cs diff --git a/.vscode/launch.json b/.vscode/launch.json index 2c85382..c631205 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -79,11 +79,12 @@ "program": "${workspaceFolder}/gaseous-processhost/bin/Debug/net10.0/gaseous-processhost.dll", "args": [ "--service", - "SignatureIngestor", + "MetadataRefresh", "--reportingserver", "https://localhost:5197", "--correlationid", - "00000000-0000-0000-0000-000000000000" + "00000000-0000-0000-0000-000000000000", + "--force" ], "cwd": "${workspaceFolder}/gaseous-processhost", "stopAtEntry": false diff --git a/gaseous-lib/Classes/Common.cs b/gaseous-lib/Classes/Common.cs index 7041fc5..b1d4312 100644 --- a/gaseous-lib/Classes/Common.cs +++ b/gaseous-lib/Classes/Common.cs @@ -204,6 +204,311 @@ namespace gaseous_server.Classes Country, Language } + + public class RomanNumerals + { + /// + /// Converts an integer to its Roman numeral representation. + /// + /// The integer to convert (1-3999). + /// A string containing the Roman numeral. + public static string IntToRoman(int number) + { + if (number < 1 || number > 3999) + throw new ArgumentOutOfRangeException(nameof(number), "Value must be in the range 1-3999."); + + var numerals = new[] + { + new { Value = 1000, Numeral = "M" }, + new { Value = 900, Numeral = "CM" }, + new { Value = 500, Numeral = "D" }, + new { Value = 400, Numeral = "CD" }, + new { Value = 100, Numeral = "C" }, + new { Value = 90, Numeral = "XC" }, + new { Value = 50, Numeral = "L" }, + new { Value = 40, Numeral = "XL" }, + new { Value = 10, Numeral = "X" }, + new { Value = 9, Numeral = "IX" }, + new { Value = 5, Numeral = "V" }, + new { Value = 4, Numeral = "IV" }, + new { Value = 1, Numeral = "I" } + }; + + var result = string.Empty; + foreach (var item in numerals) + { + while (number >= item.Value) + { + result += item.Numeral; + number -= item.Value; + } + } + return result; + } + + /// + /// Finds the first Roman numeral in a string. + /// + /// The input string to search. + /// The first Roman numeral found, or null if none found. + public static string? FindFirstRomanNumeral(string input) + { + if (string.IsNullOrEmpty(input)) + return null; + + // Regex for Roman numerals (1-3999, case-insensitive) + var matches = Regex.Matches(input, @"\bM{0,3}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})\b", RegexOptions.IgnoreCase); + foreach (Match match in matches) + { + if (match.Success && !string.IsNullOrEmpty(match.Value)) + return match.Value.ToUpper(); + } + + return null; + } + + /// + /// Converts a Roman numeral string to its integer representation. + /// + /// The Roman numeral string to convert. + /// The integer representation of the Roman numeral. + public static int RomanToInt(string roman) + { + if (string.IsNullOrEmpty(roman)) + throw new ArgumentException("Input cannot be null or empty.", nameof(roman)); + + var romanMap = new Dictionary + { + { 'I', 1 }, + { 'V', 5 }, + { 'X', 10 }, + { 'L', 50 }, + { 'C', 100 }, + { 'D', 500 }, + { 'M', 1000 } + }; + + int total = 0; + int prevValue = 0; + + foreach (char c in roman.ToUpper()) + { + if (!romanMap.ContainsKey(c)) + throw new ArgumentException($"Invalid Roman numeral character: {c}", nameof(roman)); + + int currentValue = romanMap[c]; + + // If the current value is greater than the previous value, subtract twice the previous value + // (to account for the addition in the previous iteration). + if (currentValue > prevValue) + { + total += currentValue - 2 * prevValue; + } + else + { + total += currentValue; + } + + prevValue = currentValue; + } + + return total; + } + } + + public class Numbers + { + private static readonly Dictionary NumberWords = new Dictionary + { + { 0, "Zero" }, + { 1, "One" }, + { 2, "Two" }, + { 3, "Three" }, + { 4, "Four" }, + { 5, "Five" }, + { 6, "Six" }, + { 7, "Seven" }, + { 8, "Eight" }, + { 9, "Nine" }, + { 10, "Ten" }, + { 11, "Eleven" }, + { 12, "Twelve" }, + { 13, "Thirteen" }, + { 14, "Fourteen" }, + { 15, "Fifteen" }, + { 16, "Sixteen" }, + { 17, "Seventeen" }, + { 18, "Eighteen" }, + { 19, "Nineteen" }, + { 20, "Twenty" }, + { 30, "Thirty" }, + { 40, "Forty" }, + { 50, "Fifty" }, + { 60, "Sixty" }, + { 70, "Seventy" }, + { 80, "Eighty" }, + { 90, "Ninety" }, + { 100, "Hundred" }, + { 1000, "Thousand" }, + { 1000000, "Million" }, + { 1000000000, "Billion" } + }; + + private static readonly Dictionary WordsToNumber = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + { "Zero", 0 }, + { "One", 1 }, + { "Two", 2 }, + { "Three", 3 }, + { "Four", 4 }, + { "Five", 5 }, + { "Six", 6 }, + { "Seven", 7 }, + { "Eight", 8 }, + { "Nine", 9 }, + { "Ten", 10 }, + { "Eleven", 11 }, + { "Twelve", 12 }, + { "Thirteen", 13 }, + { "Fourteen", 14 }, + { "Fifteen", 15 }, + { "Sixteen", 16 }, + { "Seventeen", 17 }, + { "Eighteen", 18 }, + { "Nineteen", 19 }, + { "Twenty", 20 }, + { "Thirty", 30 }, + { "Forty", 40 }, + { "Fifty", 50 }, + { "Sixty", 60 }, + { "Seventy", 70 }, + { "Eighty", 80 }, + { "Ninety", 90 }, + { "Hundred", 100 }, + { "Thousand", 1000 }, + { "Million", 1000000 }, + { "Billion", 1000000000 } + }; + + /// + /// Converts a number to its English word representation. + /// + /// The number to convert (0 to 999,999,999). + /// The English word representation of the number. + public static string NumberToWords(int number) + { + if (number < 0 || number > 999999999) + throw new ArgumentOutOfRangeException(nameof(number), "Value must be in the range 0-999,999,999."); + + if (number == 0) + return "Zero"; + + if (NumberWords.TryGetValue(number, out var word)) + return word; + + List parts = new List(); + + // Billions + int billions = number / 1000000000; + if (billions > 0) + { + parts.Add(NumberToWords(billions) + " Billion"); + number %= 1000000000; + } + + // Millions + int millions = number / 1000000; + if (millions > 0) + { + parts.Add(NumberToWords(millions) + " Million"); + number %= 1000000; + } + + // Thousands + int thousands = number / 1000; + if (thousands > 0) + { + parts.Add(NumberToWords(thousands) + " Thousand"); + number %= 1000; + } + + // Hundreds + int hundreds = number / 100; + if (hundreds > 0) + { + parts.Add(NumberWords[hundreds] + " Hundred"); + number %= 100; + } + + // Ones and Tens + if (number > 0) + { + if (number < 20) + { + parts.Add(NumberWords[number]); + } + else + { + int tens = number / 10; + int ones = number % 10; + string tensWord = NumberWords[tens * 10]; + if (ones > 0) + { + parts.Add(tensWord + " " + NumberWords[ones]); + } + else + { + parts.Add(tensWord); + } + } + } + + return string.Join(" ", parts); + } + + /// + /// Converts English number words to an integer. + /// Handles written forms like "Twenty One", "One Hundred Thirty Four", etc. + /// + /// The English words representing a number. + /// The integer representation, or null if conversion fails. + public static int? WordsToNumbers(string words) + { + if (string.IsNullOrWhiteSpace(words)) + return null; + + // Normalize spacing and remove extra whitespace + words = Regex.Replace(words.Trim(), @"\s+", " "); + string[] tokens = words.Split(' ', StringSplitOptions.RemoveEmptyEntries); + + int result = 0; + int current = 0; + + foreach (string token in tokens) + { + if (!WordsToNumber.TryGetValue(token, out int value)) + return null; // Invalid token + + if (value >= 1000) + { + current += result; + result = current * value; + current = 0; + } + else if (value == 100) + { + current *= value; + } + else + { + current += value; + } + } + + result += current; + return result >= 0 ? result : null; + } + } } /// diff --git a/gaseous-lib/Classes/Database/DatabaseMigration.cs b/gaseous-lib/Classes/Database/DatabaseMigration.cs index 46995ed..ed6f8ac 100644 --- a/gaseous-lib/Classes/Database/DatabaseMigration.cs +++ b/gaseous-lib/Classes/Database/DatabaseMigration.cs @@ -519,11 +519,17 @@ namespace gaseous_server.Classes md5hash = (string)row["MD5"], sha1hash = (string)row["SHA1"] }; - Signatures_Games signature = await fileSignature.GetFileSignatureAsync( + + FileSignature.FileHash fileHash = new FileSignature.FileHash() + { + Library = library, + Hash = hash, + FileName = (string)row["RelativePath"] + }; + + var (_, signature) = await fileSignature.GetFileSignatureAsync( library, - hash, - new FileInfo((string)row["Path"]), - (string)row["Path"] + fileHash ); gaseous_server.Classes.Plugins.MetadataProviders.MetadataTypes.Platform platform = await Platforms.GetPlatform((long)row["PlatformId"]); diff --git a/gaseous-lib/Classes/Filters.cs b/gaseous-lib/Classes/Filters.cs index b551c87..d77d863 100644 --- a/gaseous-lib/Classes/Filters.cs +++ b/gaseous-lib/Classes/Filters.cs @@ -69,7 +69,7 @@ namespace gaseous_server.Classes ORDER BY p.Name"; DataTable dbResponse = await db.ExecuteCMDAsync(sql, new DatabaseMemoryCacheOptions(CacheEnabled: true, ExpirationSeconds: 300)); - + foreach (DataRow dr in dbResponse.Rows) { FilterItem item = new FilterItem(dr); @@ -107,7 +107,7 @@ namespace gaseous_server.Classes GROUP BY AgeGroupId ORDER BY AgeGroupId DESC"; dbResponse = await db.ExecuteCMDAsync(sql, new DatabaseMemoryCacheOptions(CacheEnabled: true, ExpirationSeconds: 300)); - + foreach (DataRow dr in dbResponse.Rows) { FilterItem filterAgeGrouping = new FilterItem(); @@ -203,7 +203,7 @@ namespace gaseous_server.Classes await db.ExecuteCMDAsync(baseFilteredGamesQuery, new DatabaseMemoryCacheOptions(CacheEnabled: false)); // Now run lightweight queries against the temp table - + // platforms List platforms = new List(); string sql = @" @@ -219,7 +219,7 @@ namespace gaseous_server.Classes ORDER BY p.Name"; DataTable dbResponse = await db.ExecuteCMDAsync(sql, new DatabaseMemoryCacheOptions(CacheEnabled: false)); - + foreach (DataRow dr in dbResponse.Rows) { FilterItem platformItem = new FilterItem(dr); @@ -253,7 +253,7 @@ namespace gaseous_server.Classes GROUP BY AgeGroupId ORDER BY AgeGroupId DESC"; dbResponse = await db.ExecuteCMDAsync(sql, new DatabaseMemoryCacheOptions(CacheEnabled: false)); - + foreach (DataRow dr in dbResponse.Rows) { FilterItem filterAgeGrouping = new FilterItem(); @@ -282,46 +282,37 @@ namespace gaseous_server.Classes private static async Task> GenerateFilterSetFromTemp(Database db, string Name) { - List filter = new List(); DataTable dbResponse = await GetGenericFilterItemFromTemp(db, Name); + Dictionary filterDict = new Dictionary(StringComparer.Ordinal); foreach (DataRow dr in dbResponse.Rows) { FilterItem filterItem = new FilterItem(dr); - if (filterItem != null) + if (filterItem?.Name != null) { - bool nameExists = false; - foreach (var filterObject in filter) + if (filterDict.TryGetValue(filterItem.Name, out FilterItem? existingItem)) { - if (filterObject.Name == filterItem.Name) + // Merge with existing item + if (existingItem?.Ids != null && filterItem.Ids != null) { - // add the ids to the existing genre - if (filterObject.Ids == null) - { - filterObject.Ids = new Dictionary(); - } - foreach (var id in filterItem.Ids) { - if (filterObject.Ids.ContainsKey(id.Key) == false) + if (!existingItem.Ids.ContainsKey(id.Key)) { - filterObject.Ids.Add(id.Key, id.Value); - filterObject.GameCount += filterItem.GameCount; + existingItem.Ids[id.Key] = id.Value; + existingItem.GameCount += filterItem.GameCount; } } - - nameExists = true; } } - - if (nameExists == false) + else { - filter.Add(filterItem); + filterDict[filterItem.Name] = filterItem; } } } - return filter; + return new List(filterDict.Values); } private static async Task GetGenericFilterItemFromTemp(Database db, string Name) @@ -343,52 +334,43 @@ namespace gaseous_server.Classes ORDER BY item.Name"; sql = sql.Replace("", Name); DataTable dbResponse = await db.ExecuteCMDAsync(sql, new DatabaseMemoryCacheOptions(CacheEnabled: false)); - + return dbResponse; } private static async Task> GenerateFilterSet(Database db, string Name, string AgeRestriction) { - List filter = new List(); DataTable dbResponse = await GetGenericFilterItem(db, Name, AgeRestriction); + Dictionary filterDict = new Dictionary(StringComparer.Ordinal); foreach (DataRow dr in dbResponse.Rows) { FilterItem filterItem = new FilterItem(dr); - if (filterItem != null) + if (filterItem?.Name != null) { - bool nameExists = false; - foreach (var filterObject in filter) + if (filterDict.TryGetValue(filterItem.Name, out FilterItem? existingItem)) { - if (filterObject.Name == filterItem.Name) + // Merge with existing item + if (existingItem?.Ids != null && filterItem.Ids != null) { - // add the ids to the existing genre - if (filterObject.Ids == null) - { - filterObject.Ids = new Dictionary(); - } - foreach (var id in filterItem.Ids) { - if (filterObject.Ids.ContainsKey(id.Key) == false) + if (!existingItem.Ids.ContainsKey(id.Key)) { - filterObject.Ids.Add(id.Key, id.Value); - filterObject.GameCount += filterItem.GameCount; + existingItem.Ids[id.Key] = id.Value; + existingItem.GameCount += filterItem.GameCount; } } - - nameExists = true; } } - - if (nameExists == false) + else { - filter.Add(filterItem); + filterDict[filterItem.Name] = filterItem; } } } - return filter; + return new List(filterDict.Values); } private static async Task GetGenericFilterItem(Database db, string Name, string AgeRestriction) @@ -428,7 +410,7 @@ namespace gaseous_server.Classes ORDER BY item.Name"; sql = sql.Replace("", Name); DataTable dbResponse = await db.ExecuteCMDAsync(sql, new DatabaseMemoryCacheOptions(CacheEnabled: true, ExpirationSeconds: 300)); - + return dbResponse; } @@ -436,23 +418,29 @@ namespace gaseous_server.Classes { public FilterItem() { - + this.Name = string.Empty; } public FilterItem(DataRow dr) { + this.Name = string.Empty; + if (dr.Table.Columns.Contains("GameIdType")) { - HasheousClient.Models.MetadataSources SourceId = (HasheousClient.Models.MetadataSources)Enum.Parse(typeof(HasheousClient.Models.MetadataSources), dr["GameIdType"].ToString()); + int gameIdTypeIndex = dr.Table.Columns.IndexOf("GameIdType"); + int idIndex = dr.Table.Columns.IndexOf("Id"); - if (this.Ids == null) + object? gameIdTypeValue = dr[gameIdTypeIndex]; + if (gameIdTypeValue != null && gameIdTypeValue != DBNull.Value) { - this.Ids = new Dictionary(); - } - - if (this.Ids.ContainsKey(SourceId) == false) - { - this.Ids.Add(SourceId, (long)dr["Id"]); + if (int.TryParse(gameIdTypeValue.ToString(), out int sourceIdValue)) + { + HasheousClient.Models.MetadataSources SourceId = (HasheousClient.Models.MetadataSources)sourceIdValue; + this.Ids = new Dictionary(1) + { + { SourceId, (long)dr[idIndex] } + }; + } } } else @@ -460,7 +448,8 @@ namespace gaseous_server.Classes this.Id = (long)dr["Id"]; } - this.Name = (string)dr["Name"]; + object? nameValue = dr["Name"]; + this.Name = nameValue?.ToString() ?? string.Empty; this.GameCount = (int)(long)dr["GameCount"]; } diff --git a/gaseous-lib/Classes/GameLibrary.cs b/gaseous-lib/Classes/GameLibrary.cs index d58627a..511c358 100644 --- a/gaseous-lib/Classes/GameLibrary.cs +++ b/gaseous-lib/Classes/GameLibrary.cs @@ -4,7 +4,6 @@ using System.Threading.Tasks; using gaseous_server.Classes; using gaseous_server.Classes.Metadata; using gaseous_server.Models; -using Microsoft.CodeAnalysis.FlowAnalysis.DataFlow; namespace gaseous_server { diff --git a/gaseous-lib/Classes/HTTPComms.cs b/gaseous-lib/Classes/HTTPComms.cs index 2f12fa5..84b0c16 100644 --- a/gaseous-lib/Classes/HTTPComms.cs +++ b/gaseous-lib/Classes/HTTPComms.cs @@ -216,26 +216,11 @@ namespace gaseous_server.Classes // Use provided options or fall back to default var options = jsonSerializerOptions ?? _jsonOptions; - // Clear all previous headers from the HttpClient - _httpClient.DefaultRequestHeaders.Clear(); - - // Set User-Agent header - _httpClient.DefaultRequestHeaders.Add("User-Agent", _userAgent); - // Build a per-request timeout using a linked cancellation token (avoid mutating HttpClient.Timeout) using var timeoutCts = new System.Threading.CancellationTokenSource(timeout ?? _defaultTimeout); using var linkedCts = System.Threading.CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCts.Token); var effectiveToken = linkedCts.Token; - // Add headers if provided - if (headers != null) - { - foreach (var header in headers) - { - _httpClient.DefaultRequestHeaders.Add(header.Key, header.Value); - } - } - // Create empty response object var response = new HttpResponse(); @@ -312,6 +297,20 @@ namespace gaseous_server.Classes requestMessage.Content = new StringContent(bodyContent, System.Text.Encoding.UTF8, effectiveContentType); } + // Set default and request-specific headers on the per-request message. + // Avoid mutating HttpClient.DefaultRequestHeaders, which is unsafe under concurrent requests. + requestMessage.Headers.TryAddWithoutValidation("User-Agent", _userAgent); + if (headers != null) + { + foreach (var header in headers) + { + if (!requestMessage.Headers.TryAddWithoutValidation(header.Key, header.Value) && requestMessage.Content != null) + { + requestMessage.Content.Headers.TryAddWithoutValidation(header.Key, header.Value); + } + } + } + // If resuming a large binary download, add Range header if (typeof(T) == typeof(byte[]) && enableResume && resumeFromBytes > 0 && (method == HttpMethod.GET)) { @@ -352,6 +351,24 @@ namespace gaseous_server.Classes var mediaType = httpResponseMessage.Content.Headers.ContentType?.MediaType?.ToLowerInvariant(); long? contentLength = httpResponseMessage.Content.Headers.ContentLength; bool isBinary = mediaType != null && (mediaType.Contains("octet-stream") || mediaType.StartsWith("image/") || mediaType.StartsWith("video/") || mediaType.StartsWith("audio/")); + bool isCloudFlareError = false; + + static bool IsLikelyJson(string value) + { + var trimmed = value.TrimStart(); + if (trimmed.Length == 0) + { + return false; + } + + char first = trimmed[0]; + return first == '{' || first == '[' || first == '"' || first == 't' || first == 'f' || first == 'n' || first == '-' || char.IsDigit(first); + } + + static bool ContainsCloudFlare1015(string value) + { + return value.IndexOf("error code: 1015", StringComparison.OrdinalIgnoreCase) >= 0; + } if (typeof(T) == typeof(byte[])) { @@ -391,75 +408,87 @@ namespace gaseous_server.Classes var bytes = await httpResponseMessage.Content.ReadAsByteArrayAsync(effectiveToken); response.Body = (T)(object)bytes; } - } - else if (typeof(T) == typeof(string)) - { - string responseBody = await httpResponseMessage.Content.ReadAsStringAsync(effectiveToken); - response.Body = (T)(object)responseBody; - } - else if (mediaType != null && (mediaType.Contains("json") || mediaType.StartsWith("text"))) - { - string responseBody = await httpResponseMessage.Content.ReadAsStringAsync(effectiveToken); - if (!string.IsNullOrWhiteSpace(responseBody)) + + if (response.Body != null) { - response.Body = System.Text.Json.JsonSerializer.Deserialize(responseBody, options); + try + { + string bodyText = System.Text.Encoding.UTF8.GetString((byte[])(object)response.Body!).ToLowerInvariant(); + if (bodyText.Contains("error code: 1015")) + { + isCloudFlareError = true; + } + } + catch + { + // Ignore decode issues and continue standard handling. + } } } - else if (mediaType != null && mediaType.Contains("xml")) - { - string responseBody = await httpResponseMessage.Content.ReadAsStringAsync(effectiveToken); - if (!string.IsNullOrWhiteSpace(responseBody)) - { - var serializer = new System.Xml.Serialization.XmlSerializer(typeof(T)); - using var reader = new StringReader(responseBody); - response.Body = (T?)serializer.Deserialize(reader); - } - } - else if (isBinary) - { - var bytes = await httpResponseMessage.Content.ReadAsByteArrayAsync(effectiveToken); - // If T is not byte[], attempt to deserialize is risky; return default - if (typeof(T) == typeof(byte[])) - response.Body = (T)(object)bytes; - } else { - // Fallback: try JSON string responseBody = await httpResponseMessage.Content.ReadAsStringAsync(effectiveToken); - if (!string.IsNullOrWhiteSpace(responseBody)) - { - response.Body = System.Text.Json.JsonSerializer.Deserialize(responseBody, options); - } - } - // Check for CloudFlare rate limiting first (error code: 1015 can come with 200 status) - bool isCloudFlareError = false; - if (response.Body != null && typeof(T) == typeof(byte[])) - { - try + // CloudFlare can return HTML/text with 200 and "error code: 1015"; detect this before any JSON parsing. + if (!string.IsNullOrWhiteSpace(responseBody) && ContainsCloudFlare1015(responseBody)) { - string bodyText = System.Text.Encoding.UTF8.GetString((byte[])(object)response.Body!).ToLower(); - if (bodyText.Contains("error code: 1015")) + isCloudFlareError = true; + } + + if (!isCloudFlareError) + { + if (typeof(T) == typeof(string)) { - isCloudFlareError = true; - attempts++; - if (attempts < maxAttempts) + response.Body = (T)(object)responseBody; + } + else if (mediaType != null && mediaType.Contains("xml")) + { + if (!string.IsNullOrWhiteSpace(responseBody)) { - int waitTime = _rateLimit429WaitTimeSeconds; - await Task.Delay(waitTime * 1000, effectiveToken); - continue; + var serializer = new System.Xml.Serialization.XmlSerializer(typeof(T)); + using var reader = new StringReader(responseBody); + response.Body = (T?)serializer.Deserialize(reader); + } + } + else if (mediaType != null && (mediaType.Contains("json") || mediaType.StartsWith("text"))) + { + if (!string.IsNullOrWhiteSpace(responseBody) && IsLikelyJson(responseBody)) + { + response.Body = System.Text.Json.JsonSerializer.Deserialize(responseBody, options); + } + } + else if (isBinary) + { + // Not expected for non-byte[] callers; leave body unset. + } + else + { + // Fallback: only attempt JSON parse if payload looks like JSON. + if (!string.IsNullOrWhiteSpace(responseBody) && IsLikelyJson(responseBody)) + { + response.Body = System.Text.Json.JsonSerializer.Deserialize(responseBody, options); } } } - catch + } + + if (isCloudFlareError) + { + attempts++; + if (attempts < maxAttempts) { - // If unable to parse response body, continue with normal flow + int waitTime = _rateLimit429WaitTimeSeconds; + await Task.Delay(waitTime * 1000, effectiveToken); + continue; } } // If request was successful and not a CloudFlare error, exit loop if (httpResponseMessage.IsSuccessStatusCode && !isCloudFlareError) { + response.ErrorMessage = null; + response.ErrorType = null; + response.ErrorStackTrace = null; break; } diff --git a/gaseous-lib/Classes/HashObject.cs b/gaseous-lib/Classes/HashObject.cs index c6e4beb..5350e98 100644 --- a/gaseous-lib/Classes/HashObject.cs +++ b/gaseous-lib/Classes/HashObject.cs @@ -4,15 +4,55 @@ using System.Security.Cryptography; namespace gaseous_server.Classes { + /// + /// Represents a collection of hash values for a file, including MD5, SHA1, SHA256, and CRC32. + /// public class HashObject { + /// + /// The MD5 hash of the file, represented as a lowercase hexadecimal string. + /// public string md5hash { get; set; } = string.Empty; + + /// + /// The SHA1 hash of the file, represented as a lowercase hexadecimal string. + /// public string sha1hash { get; set; } = string.Empty; + + /// + /// The SHA256 hash of the file, represented as a lowercase hexadecimal string. + /// public string sha256hash { get; set; } = string.Empty; + + /// + /// The CRC32 hash of the file, represented as a lowercase hexadecimal string. + /// public string crc32hash { get; set; } = string.Empty; + /// + /// Initializes a new instance of the HashObject class with empty hash values. + /// public HashObject() { } + /// + /// Initializes a new instance of the HashObject class with specified hash values. + /// + /// The MD5 hash value. + /// The SHA1 hash value. + /// The SHA256 hash value. + /// The CRC32 hash value. + public HashObject(string md5, string sha1, string sha256, string crc32) + { + md5hash = md5; + sha1hash = sha1; + sha256hash = sha256; + crc32hash = crc32; + } + + /// + /// Initializes a new instance of the HashObject class by computing the hash values for the specified file. + /// + /// The path to the file for which to compute hash values. public HashObject(string fileName) { using var fileStream = File.OpenRead(fileName); diff --git a/gaseous-lib/Classes/ImportGames.cs b/gaseous-lib/Classes/ImportGames.cs index 1fadc71..3df5f16 100644 --- a/gaseous-lib/Classes/ImportGames.cs +++ b/gaseous-lib/Classes/ImportGames.cs @@ -301,9 +301,9 @@ namespace gaseous_server.Classes { Logging.LogKey(Logging.LogType.Information, "process.import_game", "importgame.file_not_in_database_processing", null, new string[] { FilePath }); - FileInfo fi = new FileInfo(FilePath); FileSignature fileSignature = new FileSignature(); - gaseous_server.Models.Signatures_Games discoveredSignature = fileSignature.GetFileSignatureAsync(GameLibrary.GetDefaultLibrary, Hash, fi, FilePath).Result; + FileHash fileHash = GetFileHashesAsync(GameLibrary.GetDefaultLibrary, FilePath).Result; + var (updatedFileHash, discoveredSignature) = fileSignature.GetFileSignatureAsync(GameLibrary.GetDefaultLibrary, fileHash).Result; if (discoveredSignature.Flags.GameId == 0) { try @@ -475,12 +475,7 @@ namespace gaseous_server.Classes dbDict.Add("romtypemedia", Common.ReturnValueIfNull(signature.Rom.RomTypeMedia, "")); dbDict.Add("medialabel", Common.ReturnValueIfNull(signature.Rom.MediaLabel, "")); - string libraryRootPath = library.Path; - if (libraryRootPath.EndsWith(Path.DirectorySeparatorChar.ToString()) == false) - { - libraryRootPath += Path.DirectorySeparatorChar; - } - dbDict.Add("path", filePath.Replace(libraryRootPath, "")); + dbDict.Add("path", Path.GetRelativePath(library.Path, filePath)); DataTable romInsert = await db.ExecuteCMDAsync(sql, dbDict); if (romId == 0) @@ -617,33 +612,212 @@ namespace gaseous_server.Classes return searchResults; } - private static List GetSearchCandidates(string GameName) + public static List GetSearchCandidates(string GameName) { - // remove version numbers from name - GameName = Common.StripVersionsFromFileName(GameName); - - // assumption: no games have () in their titles so we'll remove them - int idx = GameName.IndexOf('('); - if (idx >= 0) + if (string.IsNullOrWhiteSpace(GameName)) { - GameName = GameName.Substring(0, idx); + return new List(); } - List SearchCandidates = new List(); - SearchCandidates.Add(GameName.Trim()); - if (GameName.Contains(" - ")) + List searchCandidates = new List(); + + string NormalizeWhitespace(string value) { - SearchCandidates.Add(GameName.Replace(" - ", ": ").Trim()); - SearchCandidates.Add(GameName.Substring(0, GameName.IndexOf(" - ")).Trim()); - } - if (GameName.Contains(": ")) - { - SearchCandidates.Add(GameName.Substring(0, GameName.IndexOf(": ")).Trim()); + return Regex.Replace(value, @"\s+", " ").Trim(); } - Logging.LogKey(Logging.LogType.Information, "process.import_game", "importgame.search_candidates", null, new string[] { String.Join(", ", SearchCandidates) }); + string NormalizeDashes(string value) + { + return Regex.Replace(value, @"[\u2012\u2013\u2014\u2015\u2212]", "-"); + } - return SearchCandidates; + void AddCandidate(string value) + { + string normalized = NormalizeWhitespace(value); + if (!string.IsNullOrWhiteSpace(normalized)) + { + searchCandidates.Add(normalized); + } + } + + string baseName = NormalizeWhitespace(GameName); + AddCandidate(baseName); + + string dashNormalized = NormalizeDashes(baseName); + if (!string.Equals(dashNormalized, baseName, StringComparison.Ordinal)) + { + AddCandidate(dashNormalized); + } + + // remove common trailing version/revision markers while keeping the original + string versionStripped = Regex.Replace(dashNormalized, @"\s*(?:v|ver\.?|version)\s*(\d+(?:\.\d+)*)\s*$", "", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); + if (!string.Equals(versionStripped, dashNormalized, StringComparison.Ordinal)) + { + AddCandidate(versionStripped); + } + + string revisionStripped = Regex.Replace(dashNormalized, @"\s*(?:rev(?:ision)?\.?)(?:\s*[A-Za-z0-9]+)?\s*$", "", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); + if (!string.Equals(revisionStripped, dashNormalized, StringComparison.Ordinal)) + { + AddCandidate(revisionStripped); + } + + // remove trailing bracketed metadata while keeping the original + string bracketStripped = Regex.Replace(dashNormalized, @"\s*[\(\[][^\)\]]+[\)\]]\s*$", "", RegexOptions.IgnoreCase, TimeSpan.FromSeconds(1)); + if (!string.Equals(bracketStripped, dashNormalized, StringComparison.Ordinal)) + { + AddCandidate(bracketStripped); + } + + void AddDelimiterVariants(string value) + { + if (value.Contains(" - ", StringComparison.Ordinal)) + { + AddCandidate(value.Replace(" - ", ": ")); + AddCandidate(value.Replace(" - ", " ")); + } + + if (value.Contains(": ", StringComparison.Ordinal)) + { + AddCandidate(value.Replace(": ", " ")); + } + } + + void AddArticleVariants(string value) + { + if (value.StartsWith("The ", StringComparison.OrdinalIgnoreCase)) + { + string without = value.Substring(4).Trim(); + AddCandidate(without); + AddCandidate($"{without}, The"); + } + + if (value.StartsWith("A ", StringComparison.OrdinalIgnoreCase)) + { + string without = value.Substring(2).Trim(); + AddCandidate(without); + AddCandidate($"{without}, A"); + } + + if (value.StartsWith("An ", StringComparison.OrdinalIgnoreCase)) + { + string without = value.Substring(3).Trim(); + AddCandidate(without); + AddCandidate($"{without}, An"); + } + + if (value.EndsWith(", The", StringComparison.OrdinalIgnoreCase)) + { + string without = value.Substring(0, value.Length - 5).Trim(); + AddCandidate(without); + AddCandidate($"The {without}"); + } + + if (value.EndsWith(", A", StringComparison.OrdinalIgnoreCase)) + { + string without = value.Substring(0, value.Length - 3).Trim(); + AddCandidate(without); + AddCandidate($"A {without}"); + } + + if (value.EndsWith(", An", StringComparison.OrdinalIgnoreCase)) + { + string without = value.Substring(0, value.Length - 4).Trim(); + AddCandidate(without); + AddCandidate($"An {without}"); + } + } + + // expand with delimiter and article variants + foreach (string candidate in searchCandidates.ToList()) + { + AddDelimiterVariants(candidate); + AddArticleVariants(candidate); + } + + // convert roman numerals to numbers (token-based) + foreach (string candidate in searchCandidates.ToList()) + { + string romanConverted = Regex.Replace(candidate, @"\b[IVXLCDM]+\b", match => + { + return Common.RomanNumerals.RomanToInt(match.Value).ToString(); + }, RegexOptions.IgnoreCase); + + if (!string.Equals(romanConverted, candidate, StringComparison.Ordinal)) + { + AddCandidate(romanConverted); + } + } + + // convert numbers to roman numerals (token-based) + foreach (string candidate in searchCandidates.ToList()) + { + string numberToRoman = Regex.Replace(candidate, @"\b(\d+)\b", match => + { + if (int.TryParse(match.Groups[1].Value, out int num) && num >= 1 && num <= 3999) + { + return Common.RomanNumerals.IntToRoman(num); + } + return match.Value; + }); + + if (!string.Equals(numberToRoman, candidate, StringComparison.Ordinal)) + { + AddCandidate(numberToRoman); + } + } + + // convert numbers to English words and vice versa (token-based) + foreach (string candidate in searchCandidates.ToList()) + { + // Convert numbers to words + string numberToWords = Regex.Replace(candidate, @"\b(\d+)\b", match => + { + if (int.TryParse(match.Groups[1].Value, out int num) && num >= 0 && num <= 999999999) + { + return Common.Numbers.NumberToWords(num); + } + return match.Value; + }); + + if (!string.Equals(numberToWords, candidate, StringComparison.Ordinal)) + { + AddCandidate(numberToWords); + } + + // Convert English number words to numbers (look for sequences of number words) + string wordsToNumber = Regex.Replace(candidate, @"\b(?:Zero|One|Two|Three|Four|Five|Six|Seven|Eight|Nine|Ten|Eleven|Twelve|Thirteen|Fourteen|Fifteen|Sixteen|Seventeen|Eighteen|Nineteen|Twenty|Thirty|Forty|Fifty|Sixty|Seventy|Eighty|Ninety|Hundred|Thousand|Million|Billion)(?:\s+(?:Zero|One|Two|Three|Four|Five|Six|Seven|Eight|Nine|Ten|Eleven|Twelve|Thirteen|Fourteen|Fifteen|Sixteen|Seventeen|Eighteen|Nineteen|Twenty|Thirty|Forty|Fifty|Sixty|Seventy|Eighty|Ninety|Hundred|Thousand|Million|Billion))*\b", match => + { + var result = Common.Numbers.WordsToNumbers(match.Value); + return result.HasValue ? result.Value.ToString() : match.Value; + }, RegexOptions.IgnoreCase); + + if (!string.Equals(wordsToNumber, candidate, StringComparison.Ordinal)) + { + AddCandidate(wordsToNumber); + } + } + + // remove duplicates while preserving order + List distinctCandidates = new List(); + HashSet seen = new HashSet(StringComparer.OrdinalIgnoreCase); + foreach (string candidate in searchCandidates) + { + string normalized = NormalizeWhitespace(candidate); + if (string.IsNullOrWhiteSpace(normalized)) + { + continue; + } + + if (seen.Add(normalized)) + { + distinctCandidates.Add(normalized); + } + } + + Logging.LogKey(Logging.LogType.Information, "process.import_game", "importgame.search_candidates", null, new string[] { String.Join(", ", distinctCandidates) }); + + return distinctCandidates; } public static async Task ComputeROMPath(long RomId) @@ -750,13 +924,7 @@ namespace gaseous_server.Classes string sql = "UPDATE Games_Roms SET RelativePath=@path WHERE Id=@id"; Dictionary dbDict = new Dictionary(); dbDict.Add("id", RomId); - - string libraryRootPath = rom.Library.Path; - if (libraryRootPath.EndsWith(Path.DirectorySeparatorChar.ToString()) == false) - { - libraryRootPath += Path.DirectorySeparatorChar; - } - dbDict.Add("path", DestinationPath.Replace(libraryRootPath, "")); + dbDict.Add("path", Path.GetRelativePath(rom.Library.Path, DestinationPath)); await db.ExecuteCMDAsync(sql, dbDict); return true; @@ -884,11 +1052,9 @@ namespace gaseous_server.Classes // file is not in database - process it Logging.LogKey(Logging.LogType.Information, "process.library_scan", "libraryscan.orphaned_file_found_in_library", null, new string[] { LibraryFile }); - HashObject hash = new HashObject(LibraryFile); - FileInfo fi = new FileInfo(LibraryFile); - FileSignature fileSignature = new FileSignature(); - gaseous_server.Models.Signatures_Games sig = await fileSignature.GetFileSignatureAsync(library, hash, fi, LibraryFile); + FileHash fileHash = await GetFileHashesAsync(library, LibraryFile); + (_, gaseous_server.Models.Signatures_Games sig) = await fileSignature.GetFileSignatureAsync(library, fileHash); try { @@ -910,7 +1076,7 @@ namespace gaseous_server.Classes Game determinedGame = await SearchForGame(sig, PlatformId, true); - await StoreGame(library, hash, sig, determinedPlatform, LibraryFile, 0, false); + await StoreGame(library, fileHash.Hash, sig, determinedPlatform, LibraryFile, 0, false); } catch (Exception ex) { diff --git a/gaseous-lib/Classes/Metadata/Games.cs b/gaseous-lib/Classes/Metadata/Games.cs index 6e4cd1e..0142bbd 100644 --- a/gaseous-lib/Classes/Metadata/Games.cs +++ b/gaseous-lib/Classes/Metadata/Games.cs @@ -478,84 +478,107 @@ ORDER BY Platform.`Name`, view_Games_Roms.MetadataGameName;"; this.Screenshots = gameObject.Screenshots; this.FirstReleaseDate = gameObject.FirstReleaseDate; - // compile genres - this.Genres = new List(); - if (gameObject.Genres != null) + if (gameObject != null) { - foreach (long genreId in gameObject.Genres) + // compile genres + this.Genres = new List(); + if (gameObject.Genres != null) { - HasheousClient.Models.Metadata.IGDB.Genre? genre = Classes.Metadata.Genres.GetGenres(gameObject.MetadataSource, genreId).Result; - if (genre != null) + foreach (long genreId in gameObject.Genres) { - if (!this.Genres.Contains(genre.Name)) + if (gameObject.MetadataSource.Equals(FileSignature.MetadataSources.None)) { - this.Genres.Add(genre.Name); + continue; + } + HasheousClient.Models.Metadata.IGDB.Genre? genre = Classes.Metadata.Genres.GetGenres(gameObject.MetadataSource, genreId).Result; + if (genre != null) + { + if (!this.Genres.Contains(genre.Name)) + { + this.Genres.Add(genre.Name); + } } } } - } - // compile themes - this.Themes = new List(); - if (gameObject.Themes != null) - { - foreach (long themeId in gameObject.Themes) + // compile themes + this.Themes = new List(); + if (gameObject.Themes != null) { - HasheousClient.Models.Metadata.IGDB.Theme? theme = Classes.Metadata.Themes.GetGame_ThemesAsync(gameObject.MetadataSource, themeId).Result; - if (theme != null) + foreach (long themeId in gameObject.Themes) { - if (!this.Themes.Contains(theme.Name)) + if (gameObject.MetadataSource.Equals(FileSignature.MetadataSources.None)) { - this.Themes.Add(theme.Name); + continue; + } + HasheousClient.Models.Metadata.IGDB.Theme? theme = Classes.Metadata.Themes.GetGame_ThemesAsync(gameObject.MetadataSource, themeId).Result; + if (theme != null) + { + if (!this.Themes.Contains(theme.Name)) + { + this.Themes.Add(theme.Name); + } } } } - } - // compile players - this.Players = new List(); - if (gameObject.GameModes != null) - { - foreach (long playerId in gameObject.GameModes) + // compile players + this.Players = new List(); + if (gameObject.GameModes != null) { - HasheousClient.Models.Metadata.IGDB.GameMode? player = Classes.Metadata.GameModes.GetGame_Modes(gameObject.MetadataSource, playerId).Result; - if (player != null) + foreach (long playerId in gameObject.GameModes) { - if (!this.Players.Contains(player.Name)) + if (gameObject.MetadataSource.Equals(FileSignature.MetadataSources.None)) { - this.Players.Add(player.Name); + continue; + } + HasheousClient.Models.Metadata.IGDB.GameMode? player = Classes.Metadata.GameModes.GetGame_Modes(gameObject.MetadataSource, playerId).Result; + if (player != null) + { + if (!this.Players.Contains(player.Name)) + { + this.Players.Add(player.Name); + } } } } - } - // compile perspectives - this.Perspectives = new List(); - if (gameObject.PlayerPerspectives != null) - { - foreach (long perspectiveId in gameObject.PlayerPerspectives) + // compile perspectives + this.Perspectives = new List(); + if (gameObject.PlayerPerspectives != null) { - PlayerPerspective? perspective = Classes.Metadata.PlayerPerspectives.GetGame_PlayerPerspectives(gameObject.MetadataSource, perspectiveId).Result; - if (perspective != null) + foreach (long perspectiveId in gameObject.PlayerPerspectives) { - if (!this.Perspectives.Contains(perspective.Name)) + if (gameObject.MetadataSource.Equals(FileSignature.MetadataSources.None)) { - this.Perspectives.Add(perspective.Name); + continue; + } + PlayerPerspective? perspective = Classes.Metadata.PlayerPerspectives.GetGame_PlayerPerspectives(gameObject.MetadataSource, perspectiveId).Result; + if (perspective != null) + { + if (!this.Perspectives.Contains(perspective.Name)) + { + this.Perspectives.Add(perspective.Name); + } } } } - } - // compile age ratings - this.AgeRatings = new List(); - if (gameObject.AgeRatings != null) - { - foreach (long ageRatingId in gameObject.AgeRatings) + // compile age ratings + this.AgeRatings = new List(); + if (gameObject.AgeRatings != null) { - AgeRating? rating = Classes.Metadata.AgeRatings.GetAgeRating(gameObject.MetadataSource, ageRatingId).Result; - if (rating != null) + foreach (long ageRatingId in gameObject.AgeRatings) { - this.AgeRatings.Add(rating); + if (gameObject.MetadataSource.Equals(FileSignature.MetadataSources.None)) + { + continue; + } + AgeRating? rating = Classes.Metadata.AgeRatings.GetAgeRating(gameObject.MetadataSource, ageRatingId).Result; + if (rating != null) + { + this.AgeRatings.Add(rating); + } } } } diff --git a/gaseous-lib/Classes/Metadata/Images.cs b/gaseous-lib/Classes/Metadata/Images.cs index 6989fa9..c6a140e 100644 --- a/gaseous-lib/Classes/Metadata/Images.cs +++ b/gaseous-lib/Classes/Metadata/Images.cs @@ -26,9 +26,13 @@ namespace gaseous_server.Classes.Metadata // search for the first metadata map item that has a clear logo List metadataMapItemIds = await MetadataManagement.GetAssociatedMetadataMapIds(MetadataMapId); - foreach (long metadataMapItemId in metadataMapItemIds) + // Batch fetch all metadata maps to avoid N+1 queries + var metadataMapTasks = metadataMapItemIds.Select(id => MetadataManagement.GetMetadataMap(id)).ToList(); + var metadataMaps = await Task.WhenAll(metadataMapTasks); + + foreach (var metadataMapResult in metadataMaps) { - metadataMap = (await MetadataManagement.GetMetadataMap(metadataMapItemId)).MetadataMapItems.FirstOrDefault(x => x.SourceType == MetadataSource); + metadataMap = metadataMapResult.MetadataMapItems.FirstOrDefault(x => x.SourceType == MetadataSource); if (metadataMap != null) { game = await Games.GetGame(metadataMap.SourceType, metadataMap.SourceId); @@ -141,9 +145,10 @@ namespace gaseous_server.Classes.Metadata { return null; } - if (!Directory.Exists(Path.GetDirectoryName(imagePath))) + string? imageDir = Path.GetDirectoryName(imagePath); + if (imageDir != null && !Directory.Exists(imageDir)) { - Directory.CreateDirectory(Path.GetDirectoryName(imagePath)); + Directory.CreateDirectory(imageDir); } await System.IO.File.WriteAllBytesAsync(imagePath, imageBytes); } diff --git a/gaseous-lib/Classes/MetadataManagement.cs b/gaseous-lib/Classes/MetadataManagement.cs index 831d677..79b4ff0 100644 --- a/gaseous-lib/Classes/MetadataManagement.cs +++ b/gaseous-lib/Classes/MetadataManagement.cs @@ -7,6 +7,7 @@ using System.Linq; using HasheousClient.Models; using gaseous_server.Classes.Plugins.MetadataProviders.MetadataTypes; using NuGet.Protocol.Plugins; +using static gaseous_server.Classes.FileSignature; namespace gaseous_server.Classes { @@ -700,13 +701,30 @@ namespace gaseous_server.Classes hash = new HashObject(dr["Path"].ToString()); } + // get the attributes for the ROM from the datarow + // it is a JSON string in the database, so deserialize it into a Dictionary + // The entry we're interested in is "ZipContents", the value of which is a List object, which contains the file names and hashes of the contents of the ZIP file if the ROM is a ZIP archive + Dictionary? attributes = dr["Attributes"] == DBNull.Value ? null : Newtonsoft.Json.JsonConvert.DeserializeObject>(dr["Attributes"].ToString() ?? "{}"); + List? zipContents = null; + if (attributes != null && attributes.ContainsKey("ZipContents")) + { + zipContents = Newtonsoft.Json.JsonConvert.DeserializeObject>(attributes["ZipContents"].ToString() ?? "[]"); + } + // get the library for the ROM GameLibrary.LibraryItem library = await GameLibrary.GetLibrary((int)dr["LibraryId"]); // get the signature for the ROM FileInfo fi = new FileInfo(dr["Path"].ToString()); FileSignature fileSignature = new FileSignature(); - gaseous_server.Models.Signatures_Games signature = await fileSignature.GetFileSignatureAsync(library, hash, fi, fi.FullName); + FileHash fileHash = new FileHash() + { + Hash = hash, + Library = library, + FileName = dr["RelativePath"].ToString() ?? fi.Name, + ArchiveContents = zipContents != null ? zipContents : new List() + }; + var (updatedFileHash, signature) = await fileSignature.GetFileSignatureAsync(library, fileHash); // validate the signature - if it is invalid, skip the rest of the loop // validation rules: 1) signature must not be null, 2) signature must have a platform ID diff --git a/gaseous-lib/Classes/Plugins/FileSignatures/FileSignature.cs b/gaseous-lib/Classes/Plugins/FileSignatures/FileSignature.cs index 380501c..da55a79 100644 --- a/gaseous-lib/Classes/Plugins/FileSignatures/FileSignature.cs +++ b/gaseous-lib/Classes/Plugins/FileSignatures/FileSignature.cs @@ -69,28 +69,31 @@ namespace gaseous_server.Classes } /// - /// Gets the file signature for a game file, including analysis of compressed archives. + /// Gets the file hashes for a given file path, including analysis of compressed archives to extract hashes of contained files if applicable. /// - /// The library item containing the game file. - /// The hash object containing file checksums. - /// The FileInfo object for the game file. - /// The path to the game file being imported. - /// A task that represents the asynchronous operation. The task result contains the discovered file signature. - public async Task GetFileSignatureAsync(GameLibrary.LibraryItem library, HashObject hash, FileInfo fi, string GameFileImportPath) + /// The library item containing the file for which to compute hashes. + /// The path to the file for which to compute hashes. + /// A task that represents the asynchronous operation. The task result contains the file hash information. + public async static Task GetFileHashesAsync(GameLibrary.LibraryItem library, string filePath) { - Logging.LogKey(Logging.LogType.Information, "process.get_signature", "getsignature.getting_signature_for_file", null, new string[] { GameFileImportPath }); - gaseous_server.Models.Signatures_Games discoveredSignature = new gaseous_server.Models.Signatures_Games(); - discoveredSignature = await _GetFileSignatureAsync(hash, fi.Name, fi.Extension, fi.Length, GameFileImportPath, false); + string ImportedFileExtension = Path.GetExtension(filePath); - string ImportedFileExtension = Path.GetExtension(GameFileImportPath); + FileInfo fi = new FileInfo(filePath); + + FileHash fileHash = new FileHash + { + Library = library, + FileName = Path.GetRelativePath(library.Path, filePath), + Hash = new HashObject(filePath) + }; if (SupportedCompressionExtensions.Contains(ImportedFileExtension) && (fi.Length < 1073741824)) { // file is a zip and less than 1 GiB // extract the zip file and search the contents - string ExtractPath = Path.Combine(Config.LibraryConfiguration.LibraryTempDirectory, library.Id.ToString(), Path.GetRandomFileName()); - Logging.LogKey(Logging.LogType.Information, "process.get_signature", "getsignature.decompressing_file_to_path", null, new string[] { GameFileImportPath, ExtractPath }); + + Logging.LogKey(Logging.LogType.Information, "process.get_signature", "getsignature.decompressing_file_to_path", null, new string[] { filePath, ExtractPath }); if (!Directory.Exists(ExtractPath)) { Directory.CreateDirectory(ExtractPath); } try { @@ -99,88 +102,117 @@ namespace gaseous_server.Classes if (matchingPlugin != null) { - await matchingPlugin.DecompressFile(GameFileImportPath, ExtractPath); + await matchingPlugin.DecompressFile(filePath, ExtractPath); } Logging.LogKey(Logging.LogType.Information, "process.get_signature", "getsignature.processing_decompressed_files_for_signature_matches"); - // loop through contents until we find the first signature match + + // extract hashes of all files in the archive and add to fileHash.ArchiveContents List archiveFiles = new List(); - bool signatureFound = false; - bool signatureSelectorAlreadyApplied = false; - foreach (string file in Directory.GetFiles(ExtractPath, "*.*", SearchOption.AllDirectories)) + foreach (string file in Directory.GetFiles(ExtractPath, "*.*", SearchOption.AllDirectories).Where(File.Exists)) { - bool signatureSelector = false; - if (File.Exists(file)) + FileInfo zfi = new FileInfo(file); + HashObject zhash = new HashObject(file); + + ArchiveData archiveData = new ArchiveData { - FileInfo zfi = new FileInfo(file); - HashObject zhash = new HashObject(file); - - Logging.LogKey(Logging.LogType.Information, "process.get_signature", "getsignature.checking_signature_of_decompressed_file", null, new string[] { file }); - - if (zfi != null) - { - if (signatureFound == false) - { - gaseous_server.Models.Signatures_Games zDiscoveredSignature = await _GetFileSignatureAsync(zhash, zfi.Name, zfi.Extension, zfi.Length, file, true); - zDiscoveredSignature.Rom.Name = Path.ChangeExtension(zDiscoveredSignature.Rom.Name, ImportedFileExtension); - - if (zDiscoveredSignature.Score > discoveredSignature.Score) - { - if ( - zDiscoveredSignature.Rom.SignatureSource == gaseous_server.Models.Signatures_Games.RomItem.SignatureSourceType.MAMEArcade || - zDiscoveredSignature.Rom.SignatureSource == gaseous_server.Models.Signatures_Games.RomItem.SignatureSourceType.MAMEMess - ) - { - zDiscoveredSignature.Rom.Name = zDiscoveredSignature.Game.Description + ImportedFileExtension; - } - zDiscoveredSignature.Rom.Crc = discoveredSignature.Rom.Crc; - zDiscoveredSignature.Rom.Md5 = discoveredSignature.Rom.Md5; - zDiscoveredSignature.Rom.Sha1 = discoveredSignature.Rom.Sha1; - zDiscoveredSignature.Rom.Sha256 = discoveredSignature.Rom.Sha256; - zDiscoveredSignature.Rom.Size = discoveredSignature.Rom.Size; - discoveredSignature = zDiscoveredSignature; - - signatureFound = true; - - if (signatureSelectorAlreadyApplied == false) - { - signatureSelector = true; - signatureSelectorAlreadyApplied = true; - } - } - } - - ArchiveData archiveData = new ArchiveData - { - FileName = Path.GetFileName(file), - FilePath = zfi.Directory.FullName.Replace(ExtractPath, ""), - Size = zfi.Length, - MD5 = zhash.md5hash, - SHA1 = zhash.sha1hash, - SHA256 = zhash.sha256hash, - CRC = zhash.crc32hash, - isSignatureSelector = signatureSelector - }; - archiveFiles.Add(archiveData); - } - } + FileName = Path.GetFileName(file), + FilePath = (zfi.DirectoryName ?? string.Empty).Replace(ExtractPath, ""), + Size = zfi.Length, + MD5 = zhash.md5hash, + SHA1 = zhash.sha1hash, + SHA256 = zhash.sha256hash, + CRC = zhash.crc32hash, + isSignatureSelector = false + }; + archiveFiles.Add(archiveData); } - if (discoveredSignature.Rom.Attributes == null) - { - discoveredSignature.Rom.Attributes = new Dictionary(); - } - - discoveredSignature.Rom.Attributes.Add( - "ZipContents", Newtonsoft.Json.JsonConvert.SerializeObject(archiveFiles) - ); + fileHash.ArchiveContents = archiveFiles; } catch (Exception ex) { - Logging.LogKey(Logging.LogType.Critical, "process.get_signature", "getsignature.error_processing_compressed_file", null, new string[] { GameFileImportPath }, ex); + Logging.LogKey(Logging.LogType.Critical, "process.get_signature", "getsignature.error_processing_compressed_file", null, new string[] { filePath }, ex); } } + return fileHash; + } + + /// + /// Gets the file signature for a game file, including analysis of compressed archives. + /// + /// The library item containing the game file. + /// The file hash object containing file checksums and archive contents. + /// A task that represents the asynchronous operation. The task result contains the discovered file signature. + public async Task<(FileHash, Signatures_Games)> GetFileSignatureAsync(GameLibrary.LibraryItem library, FileHash fileHash) + { + Logging.LogKey(Logging.LogType.Information, "process.get_signature", "getsignature.getting_signature_for_file", null, new string[] { fileHash.FullFilePath }); + gaseous_server.Models.Signatures_Games discoveredSignature = new gaseous_server.Models.Signatures_Games(); + + FileInfo fi = new FileInfo(fileHash.FullFilePath); + + string ImportedFileExtension = Path.GetExtension(fileHash.FullFilePath); + + // get signature for the file itself first - if it's a zip, we'll then use this as the baseline signature to compare against signatures of the contained files to find the best match and determine if we need to apply a signature selector flag to any of the contained files in order to identify them as the primary signature match for the overall archive + discoveredSignature = await _GetFileSignatureAsync(fileHash.Hash, fi.Name, fi.Extension, fi.Length, fileHash.FileName, false); + + Logging.LogKey(Logging.LogType.Information, "process.get_signature", "getsignature.processing_decompressed_files_for_signature_matches"); + // loop through archive contents until we find the first signature match + bool signatureFound = false; + bool signatureSelectorAlreadyApplied = false; + if (fileHash.ArchiveContents != null) + { + foreach (var file in fileHash.ArchiveContents) + { + file.isSignatureSelector = false; + HashObject zhash = new HashObject(file.MD5, file.SHA1, file.SHA256, file.CRC); + + Logging.LogKey(Logging.LogType.Information, "process.get_signature", "getsignature.checking_signature_of_decompressed_file", null, new string[] { file.FilePath }); + + if (!signatureFound) + { + gaseous_server.Models.Signatures_Games zDiscoveredSignature = await _GetFileSignatureAsync(zhash, file.FileName, Path.GetExtension(file.FileName), file.Size, file.FilePath, true); + zDiscoveredSignature.Rom.Name = Path.ChangeExtension(zDiscoveredSignature.Rom.Name, ImportedFileExtension); + + if (zDiscoveredSignature.Score > discoveredSignature.Score) + { + if ( + zDiscoveredSignature.Rom.SignatureSource == gaseous_server.Models.Signatures_Games.RomItem.SignatureSourceType.MAMEArcade || + zDiscoveredSignature.Rom.SignatureSource == gaseous_server.Models.Signatures_Games.RomItem.SignatureSourceType.MAMEMess + ) + { + zDiscoveredSignature.Rom.Name = zDiscoveredSignature.Game.Description + ImportedFileExtension; + } + zDiscoveredSignature.Rom.Crc = discoveredSignature.Rom.Crc; + zDiscoveredSignature.Rom.Md5 = discoveredSignature.Rom.Md5; + zDiscoveredSignature.Rom.Sha1 = discoveredSignature.Rom.Sha1; + zDiscoveredSignature.Rom.Sha256 = discoveredSignature.Rom.Sha256; + zDiscoveredSignature.Rom.Size = discoveredSignature.Rom.Size; + discoveredSignature = zDiscoveredSignature; + + signatureFound = true; + + if (!signatureSelectorAlreadyApplied) + { + file.isSignatureSelector = true; + signatureSelectorAlreadyApplied = true; + } + } + } + } + } + + if (discoveredSignature.Rom.Attributes == null) + { + discoveredSignature.Rom.Attributes = new Dictionary(); + } + + discoveredSignature.Rom.Attributes.Add( + "ZipContents", Newtonsoft.Json.JsonConvert.SerializeObject(fileHash.ArchiveContents) + ); + + // get discovered platform Platform? determinedPlatform = null; if (library.DefaultPlatformId == null || library.DefaultPlatformId == 0) @@ -207,7 +239,7 @@ namespace gaseous_server.Classes discoveredSignature.MetadataSources.AddGame(0, gameName, FileSignature.MetadataSources.None); } - return discoveredSignature; + return (fileHash, discoveredSignature); } private async Task _GetFileSignatureAsync(HashObject hash, string ImageName, string ImageExtension, long ImageSize, string GameFileImportPath, bool IsInZip) @@ -259,6 +291,43 @@ namespace gaseous_server.Classes return discoveredSignature; } + /// + /// Represents the hash information for a file, including any relevant metadata about archive contents if the file is a compressed archive. + /// + public class FileHash + { + /// + /// Gets or sets the library item associated with the file for which the hash information is being stored. + /// + public required GameLibrary.LibraryItem Library { get; set; } + + /// + /// Gets or sets the name of the file and path for which the hash information is being stored. Should be relative to the library path and include teh file name and extension (e.g. "Super Mario World.smc" or "Archive.zip"). If the file is a compressed archive, this should be the name of the archive file itself, not the individual files contained within the archive. + /// + public required string FileName { get; set; } + + /// + /// Gets the full file path by combining the library path and the file name. This is a convenience property to easily access the full path of the file for which the hash information is being stored. + /// + public string FullFilePath + { + get + { + return Path.Combine(Library.Path, FileName); + } + } + + /// + /// Gets or sets the hash information of the file. + /// + public required HashObject Hash { get; set; } + + /// + /// Gets or sets the list of files contained within the archive, if the file is a compressed archive. + /// + public List ArchiveContents { get; set; } = new List(); + } + /// /// Represents data about a file contained within an archive. /// @@ -267,37 +336,37 @@ namespace gaseous_server.Classes /// /// Gets or sets the name of the file. /// - public string FileName { get; set; } + public required string FileName { get; set; } /// /// Gets or sets the path of the file within the archive. /// - public string FilePath { get; set; } + public required string FilePath { get; set; } /// /// Gets or sets the size of the file in bytes. /// - public long Size { get; set; } + public long Size { get; set; } = 0; /// /// Gets or sets the MD5 hash of the file. /// - public string MD5 { get; set; } + public required string MD5 { get; set; } /// /// Gets or sets the SHA1 hash of the file. /// - public string SHA1 { get; set; } + public required string SHA1 { get; set; } /// /// Gets or sets the SHA256 hash of the file. /// - public string SHA256 { get; set; } + public required string SHA256 { get; set; } /// /// Gets or sets the CRC32 hash of the file. /// - public string CRC { get; set; } + public required string CRC { get; set; } /// /// Gets or sets a value indicating whether this file is used as a signature selector. diff --git a/gaseous-lib/Classes/Plugins/FileSignatures/FileSignaturePlugins/Hasheous.cs b/gaseous-lib/Classes/Plugins/FileSignatures/FileSignaturePlugins/Hasheous.cs index 878c341..b4395f2 100644 --- a/gaseous-lib/Classes/Plugins/FileSignatures/FileSignaturePlugins/Hasheous.cs +++ b/gaseous-lib/Classes/Plugins/FileSignatures/FileSignaturePlugins/Hasheous.cs @@ -11,6 +11,8 @@ namespace gaseous_server.Classes.Plugins.FileSignatures /// public class Hasheous : IFileSignaturePlugin { + private static readonly JsonSerializerSettings HasheousJsonSerializerSettings = CreateHasheousJsonSerializerSettings(); + /// public string Name { get; } = "Hasheous"; @@ -20,6 +22,23 @@ namespace gaseous_server.Classes.Plugins.FileSignatures /// public bool UsesInternet { get; } = true; + private static JsonSerializerSettings CreateHasheousJsonSerializerSettings() + { + JsonSerializerSettings serializerSettings = new JsonSerializerSettings(); + serializerSettings.Converters.Add(new UnknownEnumFallbackConverter()); + return serializerSettings; + } + + private static HasheousClient.Models.LookupItemModel? DeserializeLookupItemModel(string? json) + { + if (string.IsNullOrWhiteSpace(json)) + { + return null; + } + + return JsonConvert.DeserializeObject(json, HasheousJsonSerializerSettings); + } + /// public async Task GetSignature(HashObject hash, string ImageName, string ImageExtension, long ImageSize, string GameFileImportPath) { @@ -54,7 +73,7 @@ namespace gaseous_server.Classes.Plugins.FileSignatures if (cacheFile.LastWriteTimeUtc > DateTime.UtcNow.AddDays(-30)) { Logging.LogKey(Logging.LogType.Information, "process.get_signature", "getsignature.using_cached_signature_from_hasheous"); - HasheousResult = Newtonsoft.Json.JsonConvert.DeserializeObject(await File.ReadAllTextAsync(cacheFilePath)); + HasheousResult = DeserializeLookupItemModel(await File.ReadAllTextAsync(cacheFilePath)); } } @@ -86,9 +105,9 @@ namespace gaseous_server.Classes.Plugins.FileSignatures var response = await comms.SendRequestAsync(HTTPComms.HttpMethod.POST, new Uri("https://hasheous.org/api/v1/Lookup/ByHash" + sourceList), headers, body, contentType: "application/json", returnRawResponse: true); if (response != null && response.StatusCode == 200) { - if (response.Body != "The provided hash was not found in the signature database.") + if (!string.IsNullOrWhiteSpace(response.Body) && response.Body != "The provided hash was not found in the signature database.") { - HasheousResult = Newtonsoft.Json.JsonConvert.DeserializeObject(response.Body); + HasheousResult = DeserializeLookupItemModel(response.Body); if (HasheousResult != null) { @@ -116,7 +135,7 @@ namespace gaseous_server.Classes.Plugins.FileSignatures if (File.Exists(cacheFilePath)) { Logging.LogKey(Logging.LogType.Warning, "process.get_signature", "getsignature.error_retrieving_signature_from_hasheous_using_cached_signature", null, null, ex); - HasheousResult = Newtonsoft.Json.JsonConvert.DeserializeObject(await File.ReadAllTextAsync(cacheFilePath)); + HasheousResult = DeserializeLookupItemModel(await File.ReadAllTextAsync(cacheFilePath)); } else { diff --git a/gaseous-lib/Classes/ProcessQueue/ProcessQueue.cs b/gaseous-lib/Classes/ProcessQueue/ProcessQueue.cs index 480c442..1bfd539 100644 --- a/gaseous-lib/Classes/ProcessQueue/ProcessQueue.cs +++ b/gaseous-lib/Classes/ProcessQueue/ProcessQueue.cs @@ -632,7 +632,15 @@ namespace gaseous_server.ProcessQueue { if (_ItemState != QueueItemState.Disabled) { - if ((DateTime.UtcNow > NextRunTime || _ForceExecute == true) && _ItemState != QueueItemState.Running) + string? OutProcessData = CallContext.GetData("OutProcess")?.ToString(); + bool isOutProcess = false; + // isOutProcess is only true if we're launched from the process host with the intention to run out of process, if the value is not set, or is not a valid boolean, then we should default to running in process + if (bool.TryParse(OutProcessData, out bool outProcessValue)) + { + isOutProcess = outProcessValue; + } + + if ((DateTime.UtcNow > NextRunTime || _ForceExecute == true || isOutProcess == true) && _ItemState != QueueItemState.Running) { // we can run - do some setup before we start processing _LastRunTime = DateTime.UtcNow; @@ -653,14 +661,6 @@ namespace gaseous_server.ProcessQueue // log the start Logging.LogKey(Logging.LogType.Debug, "process.timered_event", "timered_event.executing_item_with_correlation_id", null, new[] { _ItemType.ToString(), _CorrelationId }); - string? OutProcessData = CallContext.GetData("OutProcess")?.ToString(); - bool isOutProcess = false; - // isOutProcess is only true if we're launched from the process host with the intention to run out of process, if the value is not set, or is not a valid boolean, then we should default to running in process - if (bool.TryParse(OutProcessData, out bool outProcessValue)) - { - isOutProcess = outProcessValue; - } - DateTime startTime = DateTime.UtcNow; if (isOutProcess == false && this.RunInProcess == false) @@ -673,6 +673,11 @@ namespace gaseous_server.ProcessQueue "--correlationid", _CorrelationId, "--reportingserver", $"http://localhost:{Config.LocalCommsPort}" }; + // if we're forcing the execution, add the force flag + if (_ForceExecute) + { + args = args.Append("--force").ToArray(); + } using var process = new System.Diagnostics.Process { StartInfo = new System.Diagnostics.ProcessStartInfo diff --git a/gaseous-lib/Classes/ProcessQueue/QueueItemStatus.cs b/gaseous-lib/Classes/ProcessQueue/QueueItemStatus.cs index f08f75f..b10bef3 100644 --- a/gaseous-lib/Classes/ProcessQueue/QueueItemStatus.cs +++ b/gaseous-lib/Classes/ProcessQueue/QueueItemStatus.cs @@ -54,7 +54,14 @@ namespace gaseous_server.Classes break; } - SendStatusToReportingServer(CallingQueueItem.GetType(), state, progress); + try + { + SendStatusToReportingServer(CallingQueueItem.GetType(), state, progress); + } + catch (Exception ex) + { + // swallow the error + } } } diff --git a/gaseous-lib/Classes/UnknownEnumFallbackConverter.cs b/gaseous-lib/Classes/UnknownEnumFallbackConverter.cs new file mode 100644 index 0000000..e7e9213 --- /dev/null +++ b/gaseous-lib/Classes/UnknownEnumFallbackConverter.cs @@ -0,0 +1,118 @@ +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace gaseous_server.Classes +{ + /// + /// A Newtonsoft.Json converter that gracefully handles unrecognised enum values during + /// deserialization. When an unknown string or numeric value is encountered the converter + /// looks for a member named Unknown on the target enum (case-insensitive) and + /// returns that. If no such member exists it falls back to the first declared member, + /// or the default value for the type. + /// + /// This converter is safe to use with any enum type or nullable enum type. + /// + public sealed class UnknownEnumFallbackConverter : JsonConverter + { + /// + public override bool CanConvert(Type objectType) + { + Type enumType = Nullable.GetUnderlyingType(objectType) ?? objectType; + return enumType.IsEnum; + } + + /// + public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) + { + Type enumType = Nullable.GetUnderlyingType(objectType) ?? objectType; + bool isNullable = Nullable.GetUnderlyingType(objectType) != null; + + if (reader.TokenType == JsonToken.Null) + { + return isNullable ? null : GetUnknownValue(enumType); + } + + JToken token = JToken.Load(reader); + object? parsedValue = TryParseEnumValue(token, enumType); + if (parsedValue != null) + { + return parsedValue; + } + + object unknownValue = GetUnknownValue(enumType); + Logging.LogKey(Logging.LogType.Warning, "UnknownEnumFallbackConverter", + $"Unknown enum value \"{token}\" for enum \"{enumType.Name}\". Using \"{unknownValue}\"."); + return unknownValue; + } + + /// + public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer) + { + if (value == null) + { + writer.WriteNull(); + return; + } + + writer.WriteValue(value.ToString()); + } + + private static object? TryParseEnumValue(JToken token, Type enumType) + { + if (token.Type == JTokenType.String) + { + string? stringValue = token.ToObject(); + if (string.IsNullOrWhiteSpace(stringValue)) + { + return null; + } + + string? matchingName = Enum.GetNames(enumType) + .FirstOrDefault(name => string.Equals(name, stringValue, StringComparison.OrdinalIgnoreCase)); + if (matchingName != null) + { + return Enum.Parse(enumType, matchingName); + } + + if (long.TryParse(stringValue, System.Globalization.NumberStyles.Integer, + System.Globalization.CultureInfo.InvariantCulture, out long numericStringValue)) + { + return TryParseNumericEnumValue(enumType, numericStringValue); + } + + return null; + } + + if (token.Type == JTokenType.Integer) + { + long numericValue = token.ToObject(); + return TryParseNumericEnumValue(enumType, numericValue); + } + + return null; + } + + private static object? TryParseNumericEnumValue(Type enumType, long numericValue) + { + object candidateValue = Enum.ToObject(enumType, numericValue); + return Enum.IsDefined(enumType, candidateValue) ? candidateValue : null; + } + + /// + /// Returns the Unknown member of if one exists, + /// otherwise the first declared member, or a default-constructed value as a last resort. + /// + private static object GetUnknownValue(Type enumType) + { + string? unknownName = Enum.GetNames(enumType) + .FirstOrDefault(name => string.Equals(name, "Unknown", StringComparison.OrdinalIgnoreCase)); + if (unknownName != null) + { + return Enum.Parse(enumType, unknownName); + } + + Array values = Enum.GetValues(enumType); + return values.Length > 0 ? values.GetValue(0)! : Activator.CreateInstance(enumType)!; + } + } +} diff --git a/gaseous-lib/gaseous-lib.csproj b/gaseous-lib/gaseous-lib.csproj index acbb726..3df4228 100644 --- a/gaseous-lib/gaseous-lib.csproj +++ b/gaseous-lib/gaseous-lib.csproj @@ -15,27 +15,27 @@ bin\Release\net10.0\gaseous-lib.xml - + - - + + - - - + + + - - - - - - + + + + + + - - + + diff --git a/gaseous-processhost/Program.cs b/gaseous-processhost/Program.cs index d8dc590..37e2b74 100644 --- a/gaseous-processhost/Program.cs +++ b/gaseous-processhost/Program.cs @@ -25,6 +25,7 @@ if (cmdArgs.Contains("--version")) string serviceName = null; string reportingServerUrl = null; string correlationId = null; +bool forceExecute = false; for (int i = 0; i < cmdArgs.Length; i++) { @@ -40,6 +41,10 @@ for (int i = 0; i < cmdArgs.Length; i++) { correlationId = cmdArgs[i + 1]; } + else if (cmdArgs[i] == "--force") + { + forceExecute = true; + } } // If no service name is provided, display help @@ -83,7 +88,10 @@ gaseous_server.ProcessQueue.QueueProcessor.QueueItem Task = new QueueProcessor.Q { CorrelationId = correlationId }; -Task.ForceExecute(); +if (forceExecute) +{ + Task.ForceExecute(); +} // start the task try diff --git a/gaseous-server.Tests/HTTPCommsTests.cs b/gaseous-server.Tests/HTTPCommsTests.cs index f14470a..c492b12 100644 --- a/gaseous-server.Tests/HTTPCommsTests.cs +++ b/gaseous-server.Tests/HTTPCommsTests.cs @@ -1,4 +1,7 @@ using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; using System.Net; using System.Net.Http; using System.Threading; @@ -21,6 +24,18 @@ namespace gaseous_server.Tests return comms; } + private static void SetPrivateStaticIntField(string fieldName, int value) + { + var field = typeof(HTTPComms).GetField(fieldName, System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); + field?.SetValue(null, value); + } + + private static int GetPrivateStaticIntField(string fieldName) + { + var field = typeof(HTTPComms).GetField(fieldName, System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Static); + return field != null ? (int)field.GetValue(null)! : 0; + } + [Fact] public async Task SendRequestAsync_DeserializesJson() { @@ -33,7 +48,7 @@ namespace gaseous_server.Tests var result = await comms.SendRequestAsync(HTTPComms.HttpMethod.GET, new Uri("https://example.com/api")); Assert.Equal(200, result.StatusCode); Assert.NotNull(result.Body); - Assert.Equal(0, result.Body!.Value); + Assert.Equal(42, result.Body!.Value); } [Fact] @@ -93,6 +108,131 @@ namespace gaseous_server.Tests var ex = await Assert.ThrowsAsync(() => comms.SendRequestAsync(HTTPComms.HttpMethod.GET, new Uri("https://example.com/cancel"), cancellationToken: cts.Token)); } + [Fact] + public async Task SendRequestAsync_ConcurrentCalls_PreservePerRequestHeaders() + { + var seenRequestIds = new ConcurrentBag(); + + var comms = CreateComms(async (req, ct) => + { + // Encourage overlap in in-flight requests to exercise thread-safety. + await Task.Delay(25, ct); + + if (req.Headers.TryGetValues("X-Request-Id", out var values)) + { + var id = values.SingleOrDefault(); + if (!string.IsNullOrWhiteSpace(id)) + { + seenRequestIds.Add(id); + } + } + + return new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"value\":1}", System.Text.Encoding.UTF8, "application/json") + }; + }); + + var expectedIds = Enumerable.Range(1, 20).Select(i => i.ToString()).ToArray(); + var tasks = new List>>(); + + foreach (var id in expectedIds) + { + var headers = new Dictionary + { + ["X-Request-Id"] = id + }; + + tasks.Add(comms.SendRequestAsync(HTTPComms.HttpMethod.GET, new Uri("https://example.com/concurrent"), headers)); + } + + var results = await Task.WhenAll(tasks); + + Assert.All(results, r => Assert.Equal(200, r.StatusCode)); + Assert.Equal(expectedIds.Length, seenRequestIds.Count); + Assert.Equal(expectedIds.OrderBy(x => x), seenRequestIds.OrderBy(x => x)); + } + + [Fact] + public async Task SendRequestAsync_Cloudflare1015Text_RetriesWithoutJsonExceptionState() + { + int call = 0; + var comms = CreateComms((req, ct) => + { + call++; + if (call == 1) + { + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("error code: 1015", System.Text.Encoding.UTF8, "text/plain") + }); + } + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"value\":7}", System.Text.Encoding.UTF8, "application/json") + }); + }); + + // Avoid long waits in the retry branch. + int originalWait = GetPrivateStaticIntField("_rateLimit429WaitTimeSeconds"); + SetPrivateStaticIntField("_rateLimit429WaitTimeSeconds", 0); + try + { + var result = await comms.SendRequestAsync(HTTPComms.HttpMethod.GET, new Uri("https://example.com/cf1015"), retryCount: 2); + + Assert.Equal(2, call); + Assert.Equal(200, result.StatusCode); + Assert.NotNull(result.Body); + Assert.Equal(7, result.Body!.Value); + Assert.Null(result.ErrorType); + Assert.Null(result.ErrorMessage); + } + finally + { + SetPrivateStaticIntField("_rateLimit429WaitTimeSeconds", originalWait); + } + } + + [Fact] + public async Task SendRequestAsync_Status420_WaitsAndRetries() + { + int call = 0; + var comms = CreateComms((req, ct) => + { + call++; + if (call == 1) + { + return Task.FromResult(new HttpResponseMessage((HttpStatusCode)420) + { + Content = new StringContent("rate limited") + }); + } + + return Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new StringContent("{\"value\":9}", System.Text.Encoding.UTF8, "application/json") + }); + }); + + // Keep test fast while still exercising 420 retry branch. + int originalWait = GetPrivateStaticIntField("_rateLimit420WaitTimeSeconds"); + SetPrivateStaticIntField("_rateLimit420WaitTimeSeconds", 0); + try + { + var result = await comms.SendRequestAsync(HTTPComms.HttpMethod.GET, new Uri("https://example.com/420"), retryCount: 2); + + Assert.Equal(2, call); + Assert.Equal(200, result.StatusCode); + Assert.NotNull(result.Body); + Assert.Equal(9, result.Body!.Value); + } + finally + { + SetPrivateStaticIntField("_rateLimit420WaitTimeSeconds", originalWait); + } + } + private class TestDto { public int Value { get; set; } diff --git a/gaseous-server.Tests/gaseous-server.Tests.csproj b/gaseous-server.Tests/gaseous-server.Tests.csproj index 554c312..a99e294 100644 --- a/gaseous-server.Tests/gaseous-server.Tests.csproj +++ b/gaseous-server.Tests/gaseous-server.Tests.csproj @@ -10,7 +10,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive all - + diff --git a/gaseous-server/Controllers/V1.0/BiosController.cs b/gaseous-server/Controllers/V1.0/BiosController.cs index 0734cd4..bed307d 100644 --- a/gaseous-server/Controllers/V1.0/BiosController.cs +++ b/gaseous-server/Controllers/V1.0/BiosController.cs @@ -139,7 +139,6 @@ namespace gaseous_server.Controllers { string filename = Path.GetFileName(biosItem.biosPath); string filepath = biosItem.biosPath; - byte[] filedata = System.IO.File.ReadAllBytes(filepath); string contentType = "application/octet-stream"; var cd = new System.Net.Mime.ContentDisposition @@ -151,7 +150,7 @@ namespace gaseous_server.Controllers Response.Headers.Add("Content-Disposition", cd.ToString()); Response.Headers.Add("Cache-Control", "public, max-age=604800"); - return File(filedata, contentType); + return PhysicalFile(filepath, contentType, enableRangeProcessing: true); } else { diff --git a/gaseous-server/Controllers/V1.0/GamesController.cs b/gaseous-server/Controllers/V1.0/GamesController.cs index 9c7c12b..7f02c9a 100644 --- a/gaseous-server/Controllers/V1.0/GamesController.cs +++ b/gaseous-server/Controllers/V1.0/GamesController.cs @@ -342,6 +342,7 @@ namespace gaseous_server.Controllers [Route("{MetadataMapId}/{MetadataSource}/{ImageType}/{ImageId}/image/{size}/{imagename}")] [ProducesResponseType(typeof(FileStreamResult), StatusCodes.Status200OK)] [ProducesResponseType(StatusCodes.Status404NotFound)] + [ResponseCache(CacheProfileName = "7Days")] public async Task GameImage(long MetadataMapId, FileSignature.MetadataSources MetadataSource, ImageType imageType, long ImageId, Classes.Plugins.PluginManagement.ImageResize.ImageSize size, string imagename = "") { try @@ -355,7 +356,17 @@ namespace gaseous_server.Controllers { string filename = imgData["imageName"]; string filepath = imgData["imagePath"]; - string contentType = "image/jpg"; + + // Determine content type based on file extension + string extension = Path.GetExtension(filepath).ToLowerInvariant(); + string contentType = extension switch + { + ".png" => "image/png", + ".gif" => "image/gif", + ".webp" => "image/webp", + ".svg" => "image/svg+xml", + _ => "image/jpeg" + }; var cd = new System.Net.Mime.ContentDisposition { @@ -364,24 +375,20 @@ namespace gaseous_server.Controllers }; Response.Headers.Add("Content-Disposition", cd.ToString()); - Response.Headers.Add("Cache-Control", "public, max-age=604800"); - byte[] filedata = null; - using (FileStream fs = System.IO.File.OpenRead(filepath)) - { - using (BinaryReader binaryReader = new BinaryReader(fs)) - { - filedata = binaryReader.ReadBytes((int)fs.Length); - } - } + // Add ETag for efficient caching + var fileInfo = new System.IO.FileInfo(filepath); + string eTag = $"\"{fileInfo.LastWriteTimeUtc.Ticks:X}-{fileInfo.Length:X}\""; + Response.Headers.Add("ETag", eTag); - return File(filedata, contentType); + return PhysicalFile(filepath, contentType, enableRangeProcessing: true); } return NotFound(); } - catch + catch (Exception ex) { + Logging.LogKey(Logging.LogType.Warning, "game.image", "gameimage.error_retrieving_image", exceptionValue: ex); return NotFound(); } } @@ -734,6 +741,7 @@ namespace gaseous_server.Controllers [Route("{MetadataMapId}/{MetadataSource}/companies/{CompanyId}/image")] [ProducesResponseType(typeof(FileStreamResult), StatusCodes.Status200OK)] [ProducesResponseType(StatusCodes.Status404NotFound)] + [ResponseCache(CacheProfileName = "7Days")] public async Task GameCompanyImage(long MetadataMapId, FileSignature.MetadataSources MetadataSource, long CompanyId) { try @@ -752,7 +760,6 @@ namespace gaseous_server.Controllers { string filename = "Logo.png"; string filepath = coverFilePath; - byte[] filedata = await System.IO.File.ReadAllBytesAsync(filepath); string contentType = "image/png"; var cd = new System.Net.Mime.ContentDisposition @@ -762,9 +769,8 @@ namespace gaseous_server.Controllers }; Response.Headers.Add("Content-Disposition", cd.ToString()); - Response.Headers.Add("Cache-Control", "public, max-age=604800"); - return File(filedata, contentType); + return PhysicalFile(filepath, contentType, enableRangeProcessing: true); } else { @@ -1474,9 +1480,9 @@ namespace gaseous_server.Controllers if (RomId > 0) { Classes.Roms.GameRomItem romItem = await Classes.Roms.GetRom(RomId); - HashObject hash = new HashObject(romItem.Path); FileSignature fileSignature = new FileSignature(); - gaseous_server.Models.Signatures_Games romSig = await fileSignature.GetFileSignatureAsync(romItem.Library, hash, new FileInfo(romItem.Path), romItem.Path); + Classes.FileSignature.FileHash fileHash = await Classes.FileSignature.GetFileHashesAsync(romItem.Library, romItem.Path); + (_, gaseous_server.Models.Signatures_Games romSig) = await fileSignature.GetFileSignatureAsync(romItem.Library, fileHash); List searchResults = await Classes.ImportGame.SearchForGame_GetAll(romSig.Game.Name, romSig.Flags.PlatformId); return Ok(searchResults); diff --git a/gaseous-server/Controllers/V1.0/PlatformsController.cs b/gaseous-server/Controllers/V1.0/PlatformsController.cs index bae0426..28724c8 100644 --- a/gaseous-server/Controllers/V1.0/PlatformsController.cs +++ b/gaseous-server/Controllers/V1.0/PlatformsController.cs @@ -164,16 +164,12 @@ namespace gaseous_server.Controllers Response.Headers.Append("Content-Disposition", svgcd.ToString()); Response.Headers.Append("Cache-Control", "public, max-age=604800"); - byte[] svgfiledata = null; - using (FileStream svgfs = System.IO.File.OpenRead(imagePath)) - { - using (BinaryReader binaryReader = new BinaryReader(svgfs)) - { - svgfiledata = binaryReader.ReadBytes((int)svgfs.Length); - } - } + // Add ETag for efficient caching + var svgFileInfo = new System.IO.FileInfo(imagePath); + string svgETag = $"\"{svgFileInfo.LastWriteTimeUtc.Ticks:X}-{svgFileInfo.Length:X}\""; + Response.Headers.Append("ETag", svgETag); - return File(svgfiledata, "image/svg+xml"); + return PhysicalFile(imagePath, "image/svg+xml", enableRangeProcessing: true); } } } @@ -256,16 +252,7 @@ namespace gaseous_server.Controllers Response.Headers.Append("Cache-Control", "public, max-age=604800"); // TODO: Resize image according to size parameter - byte[] filedata = null; - using (FileStream fs = System.IO.File.OpenRead(filepath)) - { - using (BinaryReader binaryReader = new BinaryReader(fs)) - { - filedata = binaryReader.ReadBytes((int)fs.Length); - } - } - - return File(filedata, contentType); + return PhysicalFile(filepath, contentType, enableRangeProcessing: true); } else { diff --git a/gaseous-server/Controllers/V1.1/GamesController.cs b/gaseous-server/Controllers/V1.1/GamesController.cs index 74f94bd..9b1fa30 100644 --- a/gaseous-server/Controllers/V1.1/GamesController.cs +++ b/gaseous-server/Controllers/V1.1/GamesController.cs @@ -211,17 +211,14 @@ namespace gaseous_server.Controllers.v1_1 { string whereClause = ""; string havingClause = ""; - Dictionary whereParams = new Dictionary(); + Dictionary whereParams = new Dictionary(20); whereParams.Add("userid", user.Id); - List joinClauses = new List(); + List joinClauses = new List(5); string joinClauseTemplate = "LEFT JOIN `Relation_Game_s` ON `Game`.`Id` = `Relation_Game_s`.`GameId` AND `Relation_Game_s`.`GameSourceId` = `Game`.`SourceId` LEFT JOIN `Metadata_` AS `` ON `Relation_Game_s`.`sId` = ``.`Id` AND `Relation_Game_s`.`GameSourceId` = ``.`SourceId`"; - List whereClauses = new List(); - List havingClauses = new List(); + List whereClauses = new List(10); + List havingClauses = new List(10); - string tempVal = ""; - - string nameWhereClause = ""; if (model.Name.Length > 0) { whereClauses.Add("(MATCH(`Game`.`Name`) AGAINST (@GameName IN BOOLEAN MODE) OR MATCH(`AlternativeName`.`Name`) AGAINST (@GameName IN BOOLEAN MODE))"); @@ -270,7 +267,7 @@ namespace gaseous_server.Controllers.v1_1 if (model.GameRating != null) { - List ratingClauses = new List(); + List ratingClauses = new List(4); if (model.GameRating.MinimumRating != -1) { string ratingTempMinVal = "totalRating >= @totalMinRating"; @@ -300,18 +297,7 @@ namespace gaseous_server.Controllers.v1_1 } // generate rating sub clause - string ratingClauseValue = ""; - if (ratingClauses.Count > 0) - { - foreach (string ratingClause in ratingClauses) - { - if (ratingClauseValue.Length > 0) - { - ratingClauseValue += " AND "; - } - ratingClauseValue += ratingClause; - } - } + string ratingClauseValue = string.Join(" AND ", ratingClauses); string unratedClause = ""; if (model.GameRating.IncludeUnrated == true) @@ -336,19 +322,19 @@ namespace gaseous_server.Controllers.v1_1 { if (model.Platform.Count > 0) { - tempVal = "`MetadataMap`.`PlatformId` IN ("; + var sb = new System.Text.StringBuilder("`MetadataMap`.`PlatformId` IN (", model.Platform.Count * 15); for (int i = 0; i < model.Platform.Count; i++) { if (i > 0) { - tempVal += ", "; + sb.Append(", "); } string platformLabel = "@Platform" + i; - tempVal += platformLabel; + sb.Append(platformLabel); whereParams.Add(platformLabel, model.Platform[i]); } - tempVal += ")"; - whereClauses.Add(tempVal); + sb.Append(')'); + whereClauses.Add(sb.ToString()); } } @@ -356,19 +342,19 @@ namespace gaseous_server.Controllers.v1_1 { if (model.Genre.Count > 0) { - tempVal = "Genre.`Name` IN ("; + var sb = new System.Text.StringBuilder("Genre.`Name` IN (", model.Genre.Count * 15); for (int i = 0; i < model.Genre.Count; i++) { if (i > 0) { - tempVal += ", "; + sb.Append(", "); } string genreLabel = "@Genre" + i; - tempVal += genreLabel; + sb.Append(genreLabel); whereParams.Add(genreLabel, model.Genre[i]); } - tempVal += ")"; - whereClauses.Add(tempVal); + sb.Append(')'); + whereClauses.Add(sb.ToString()); joinClauses.Add(joinClauseTemplate.Replace("", "Genre")); } @@ -378,19 +364,19 @@ namespace gaseous_server.Controllers.v1_1 { if (model.GameMode.Count > 0) { - tempVal = "GameMode.`Name` IN ("; + var sb = new System.Text.StringBuilder("GameMode.`Name` IN (", model.GameMode.Count * 15); for (int i = 0; i < model.GameMode.Count; i++) { if (i > 0) { - tempVal += ", "; + sb.Append(", "); } string gameModeLabel = "@GameMode" + i; - tempVal += gameModeLabel; + sb.Append(gameModeLabel); whereParams.Add(gameModeLabel, model.GameMode[i]); } - tempVal += ")"; - whereClauses.Add(tempVal); + sb.Append(')'); + whereClauses.Add(sb.ToString()); joinClauses.Add(joinClauseTemplate.Replace("", "GameMode")); } @@ -400,19 +386,19 @@ namespace gaseous_server.Controllers.v1_1 { if (model.PlayerPerspective.Count > 0) { - tempVal = "PlayerPerspective.`Name` IN ("; + var sb = new System.Text.StringBuilder("PlayerPerspective.`Name` IN (", model.PlayerPerspective.Count * 15); for (int i = 0; i < model.PlayerPerspective.Count; i++) { if (i > 0) { - tempVal += ", "; + sb.Append(", "); } string playerPerspectiveLabel = "@PlayerPerspective" + i; - tempVal += playerPerspectiveLabel; + sb.Append(playerPerspectiveLabel); whereParams.Add(playerPerspectiveLabel, model.PlayerPerspective[i]); } - tempVal += ")"; - whereClauses.Add(tempVal); + sb.Append(')'); + whereClauses.Add(sb.ToString()); joinClauses.Add(joinClauseTemplate.Replace("", "PlayerPerspective")); } @@ -422,53 +408,53 @@ namespace gaseous_server.Controllers.v1_1 { if (model.Theme.Count > 0) { - tempVal = "Theme.`Name` IN ("; + var sb = new System.Text.StringBuilder("Theme.`Name` IN (", model.Theme.Count * 15); for (int i = 0; i < model.Theme.Count; i++) { if (i > 0) { - tempVal += ", "; + sb.Append(", "); } string themeLabel = "@Theme" + i; - tempVal += themeLabel; + sb.Append(themeLabel); whereParams.Add(themeLabel, model.Theme[i]); } - tempVal += ")"; - whereClauses.Add(tempVal); + sb.Append(')'); + whereClauses.Add(sb.ToString()); joinClauses.Add(joinClauseTemplate.Replace("", "Theme")); } } - string gameAgeRatingString = "("; if (model.GameAgeRating != null) { + var sb = new System.Text.StringBuilder("("); if (model.GameAgeRating.AgeGroupings.Count > 0) { - tempVal = "AgeGroup.AgeGroupId IN ("; + sb.Append("AgeGroup.AgeGroupId IN ("); for (int i = 0; i < model.GameAgeRating.AgeGroupings.Count; i++) { if (i > 0) { - tempVal += ", "; + sb.Append(", "); } string themeLabel = "@Rating" + i; - tempVal += themeLabel; + sb.Append(themeLabel); whereParams.Add(themeLabel, model.GameAgeRating.AgeGroupings[i]); } - tempVal += ")"; + sb.Append(')'); } if (model.GameAgeRating.IncludeUnrated == true) { - if (tempVal.Length > 0) + if (model.GameAgeRating.AgeGroupings.Count > 0) { - tempVal += " OR "; + sb.Append(" OR "); } - tempVal += "AgeGroup.AgeGroupId IS NULL"; + sb.Append("AgeGroup.AgeGroupId IS NULL"); } - gameAgeRatingString += tempVal + ")"; - whereClauses.Add(gameAgeRatingString); + sb.Append(')'); + whereClauses.Add(sb.ToString()); } // build where clause @@ -689,23 +675,43 @@ FROM DataTable dbResponse; DataTable fullDataset = null; int? RecordCount = null; - string limiter = ""; - if (returnGames == true) + // Optimize query execution: if we need both summary and games, execute once and slice in memory + if (returnSummary == true && returnGames == true) { - limiter += " LIMIT @pageOffset, @pageSize"; - whereParams.Add("pageOffset", pageSize * (pageNumber - 1)); - whereParams.Add("pageSize", pageSize); - } - - dbResponse = await db.ExecuteCMDAsync(sql + limiter, whereParams, new DatabaseMemoryCacheOptions(CacheEnabled: true, ExpirationSeconds: 60)); - - // get full count for summary if needed - if (returnSummary == true) - { - // Execute query without limit to get total record count and full dataset for alpha list + // Execute full query once fullDataset = await db.ExecuteCMDAsync(sql, whereParams, new DatabaseMemoryCacheOptions(CacheEnabled: true, ExpirationSeconds: 60)); RecordCount = fullDataset.Rows.Count; + + // Create a view for the paginated results + dbResponse = fullDataset.Clone(); + int startIndex = pageSize * (pageNumber - 1); + int endIndex = Math.Min(startIndex + pageSize, fullDataset.Rows.Count); + + for (int i = startIndex; i < endIndex; i++) + { + dbResponse.ImportRow(fullDataset.Rows[i]); + } + } + else if (returnGames == true) + { + // Only need paginated results + string limiter = " LIMIT @pageOffset, @pageSize"; + whereParams.Add("pageOffset", pageSize * (pageNumber - 1)); + whereParams.Add("pageSize", pageSize); + dbResponse = await db.ExecuteCMDAsync(sql + limiter, whereParams, new DatabaseMemoryCacheOptions(CacheEnabled: true, ExpirationSeconds: 60)); + } + else if (returnSummary == true) + { + // Only need summary + fullDataset = await db.ExecuteCMDAsync(sql, whereParams, new DatabaseMemoryCacheOptions(CacheEnabled: true, ExpirationSeconds: 60)); + RecordCount = fullDataset.Rows.Count; + dbResponse = fullDataset.Clone(); // Empty table + } + else + { + // Neither requested (edge case) + dbResponse = new DataTable(); } int indexInPage = 0; @@ -718,46 +724,25 @@ FROM List? RetVal = null; if (returnGames == true) { - RetVal = new List(); - foreach (int i in Enumerable.Range(0, dbResponse.Rows.Count)) + RetVal = new List(dbResponse.Rows.Count); + for (int i = 0; i < dbResponse.Rows.Count; i++) { - Game retGame = Storage.BuildCacheObject(new Game(), dbResponse.Rows[i]); - retGame.MetadataMapId = (long)dbResponse.Rows[i]["MetadataMapId"]; - retGame.SourceType = (FileSignature.MetadataSources)dbResponse.Rows[i]["GameIdType"]; + DataRow row = dbResponse.Rows[i]; + Game retGame = Storage.BuildCacheObject(new Game(), row); + retGame.MetadataMapId = (long)row["MetadataMapId"]; + retGame.SourceType = (FileSignature.MetadataSources)row["GameIdType"]; Games.MinimalGameItem retMinGame = new Games.MinimalGameItem(retGame); - retMinGame.Index = indexInPage; - indexInPage += 1; - if ( - dbResponse.Rows[i]["RomSavedStates"] != DBNull.Value || - dbResponse.Rows[i]["RomGroupSavedStates"] != DBNull.Value || - dbResponse.Rows[i]["RomSavedFiles"] != DBNull.Value || - dbResponse.Rows[i]["RomGroupSavedFiles"] != DBNull.Value - ) - { - if ( - (long)dbResponse.Rows[i]["RomSavedStates"] >= 1 || - (long)dbResponse.Rows[i]["RomGroupSavedStates"] >= 1 || - (long)dbResponse.Rows[i]["RomSavedFiles"] >= 1 || - (long)dbResponse.Rows[i]["RomGroupSavedFiles"] >= 1 - ) - { - retMinGame.HasSavedGame = true; - } - else - { - retMinGame.HasSavedGame = false; - } - } + retMinGame.Index = indexInPage++; - if ((int)dbResponse.Rows[i]["Favourite"] == 0) - { - retMinGame.IsFavourite = false; - } - else - { - retMinGame.IsFavourite = true; - } + // Check for saved games more efficiently + retMinGame.HasSavedGame = + (row["RomSavedStates"] != DBNull.Value && (long)row["RomSavedStates"] >= 1) || + (row["RomGroupSavedStates"] != DBNull.Value && (long)row["RomGroupSavedStates"] >= 1) || + (row["RomSavedFiles"] != DBNull.Value && (long)row["RomSavedFiles"] >= 1) || + (row["RomGroupSavedFiles"] != DBNull.Value && (long)row["RomGroupSavedFiles"] >= 1); + + retMinGame.IsFavourite = (int)row["Favourite"] != 0; RetVal.Add(retMinGame); } @@ -766,7 +751,7 @@ FROM Dictionary? AlphaList = null; if (returnSummary == true) { - AlphaList = new Dictionary(); + AlphaList = new Dictionary(27); // build alpha list if (orderByField == "NameThe" || orderByField == "Name") @@ -775,7 +760,7 @@ FROM int nextPageIndex = pageSize > 0 ? pageSize : int.MaxValue; string alphaSearchField = orderByField == "NameThe" ? "NameThe" : "Name"; - HashSet seenKeys = new HashSet(); + HashSet seenKeys = new HashSet(27); for (int i = 0; i < fullDataset.Rows.Count; i++) { @@ -797,14 +782,13 @@ FROM key = (firstChar >= 'A' && firstChar <= 'Z') ? firstChar.ToString() : "#"; } - if (!seenKeys.Contains(key)) + if (seenKeys.Add(key)) { AlphaList[key] = new GameReturnPackage.AlphaListItem { Index = i, Page = currentPage }; - seenKeys.Add(key); } } }