Merge pull request #963 from 13xforever/vnext

Maintenance

Clients/CirrusCiClient/CirrusCiClient.csproj

@@ -8,9 +8,9 @@
     <ProjectReference Include="..\CompatApiClient\CompatApiClient.csproj" />
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.0" />
-    <PackageReference Include="StrawberryShake.Server" Version="13.9.12" />
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
+    <PackageReference Include="Microsoft.Extensions.Http" Version="8.0.1" />
+    <PackageReference Include="StrawberryShake.Server" Version="13.9.14" />
   </ItemGroup>
 </Project>

Clients/CompatApiClient/CompatApiClient.csproj

@@ -15,9 +15,9 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.AspNet.WebApi.Client" Version="6.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
     <PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.1" />
-    <PackageReference Include="NLog" Version="5.3.3" />
+    <PackageReference Include="NLog" Version="5.3.4" />
   </ItemGroup>
 
 </Project>

Clients/GithubClient/GithubClient.csproj

@@ -5,7 +5,7 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
     <PackageReference Include="Octokit" Version="13.0.1" />
   </ItemGroup>
   <ItemGroup>

Clients/IrdLibraryClient/IrdClient.cs

@@ -11,59 +11,65 @@
 using CompatApiClient.Compression;
 using IrdLibraryClient.IrdFormat;
 using IrdLibraryClient.POCOs;
+using Microsoft.Extensions.Caching.Memory;
 
 namespace IrdLibraryClient
 {
     public class IrdClient
     {
-        public static readonly string JsonUrl = "https://flexby420.github.io/playstation_3_ird_database/all.json";
-        private readonly HttpClient client;
-        private readonly JsonSerializerOptions jsonOptions;
-        private static readonly string BaseDownloadUri = "https://github.com/FlexBy420/playstation_3_ird_database/raw/main/";
-
-        public IrdClient()
+        private static readonly Uri BaseDownloadUri = new("https://github.com/FlexBy420/playstation_3_ird_database/raw/main/");
+        private static readonly HttpClient Client = HttpClientFactory.Create(new CompressionMessageHandler());
+        private static readonly JsonSerializerOptions JsonOptions= new()
         {
-            client = HttpClientFactory.Create(new CompressionMessageHandler());
-            jsonOptions = new JsonSerializerOptions
-            {
-                PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
-                DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
-                IncludeFields = true,
-            };
-        }
+            PropertyNamingPolicy = JsonNamingPolicy.KebabCaseLower,
+            DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+            IncludeFields = true,
+        };
+        private static readonly MemoryCache JsonCache = new(new MemoryCacheOptions{ ExpirationScanFrequency = TimeSpan.FromHours(1) });
+
+        public static readonly Uri JsonUrl = new("https://flexby420.github.io/playstation_3_ird_database/all.json");
 
         public async Task<List<IrdInfo>> SearchAsync(string query, CancellationToken cancellationToken)
-        {
-            query = query.ToUpper();
-            try
+        {            
+            List<IrdInfo> result = [];
+            if (!JsonCache.TryGetValue("json", out Dictionary<string, List<IrdInfo>>? irdData)
+                || irdData is not { Count: > 0 })
             {
-                using var response = await client.GetAsync(JsonUrl, cancellationToken).ConfigureAwait(false);
-                if (!response.IsSuccessStatusCode)
+                try
                 {
-                    ApiConfig.Log.Error($"Failed to fetch IRD data: {response.StatusCode}");
-                    return new List<IrdInfo>();
-                }
+                    using var response = await Client.GetAsync(JsonUrl, cancellationToken).ConfigureAwait(false);
+                    if (!response.IsSuccessStatusCode)
+                    {
+                        ApiConfig.Log.Error($"Failed to fetch IRD data: {response.StatusCode}");
+                        return result;
+                    }
 
-                var jsonResult = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
-                var irdData = JsonSerializer.Deserialize<Dictionary<string, List<IrdInfo>>>(jsonResult, jsonOptions);
-                if (irdData == null)
-                {
-                    ApiConfig.Log.Error("Failed to deserialize IRD JSON data.");
-                    return new List<IrdInfo>();
+                    var jsonResult = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
+                    irdData = JsonSerializer.Deserialize<Dictionary<string, List<IrdInfo>>>(jsonResult, JsonOptions);
+                    JsonCache.Set("json", irdData, TimeSpan.FromHours(4));
                 }
-
-                if (irdData.TryGetValue(query, out var items))
+                catch (Exception e)
                 {
-                    return items;
+                    ApiConfig.Log.Error(e, "Failed to fetch IRD data.");
+                    return result;
                 }
-
-                return new List<IrdInfo>();
             }
-            catch (Exception e)
+
+            if (irdData is null)
             {
-                ApiConfig.Log.Error(e);
-                return new List<IrdInfo>();
+                ApiConfig.Log.Error("Failed to deserialize IRD JSON data.");
+                return result;
             }
+
+            if (irdData.TryGetValue(query.ToUpperInvariant(), out var items))
+                result.AddRange(items);
+            result.AddRange(
+                from lst in irdData.Values
+                from irdInfo in lst
+                where irdInfo.Title?.Contains(query, StringComparison.OrdinalIgnoreCase) ?? false
+                select irdInfo 
+            );
+            return result;
         }
 
         public async Task<List<Ird>> DownloadAsync(string productCode, string localCachePath, CancellationToken cancellationToken)
@@ -72,7 +78,7 @@ public async Task<List<Ird>> DownloadAsync(string productCode, string localCache
             try
             {
                 var searchResults = await SearchAsync(productCode, cancellationToken).ConfigureAwait(false);
-                if (searchResults == null || !searchResults.Any())
+                if (searchResults is not {Count: >0})
                 {
                     ApiConfig.Log.Debug($"No IRD files found for {productCode}");
                     return result;
@@ -81,22 +87,34 @@ public async Task<List<Ird>> DownloadAsync(string productCode, string localCache
                 foreach (var item in searchResults)
                 {
                     var localFilePath = Path.Combine(localCachePath, $"{productCode}-{item.Link.Split('/').Last()}.ird");
-                    if (!File.Exists(localFilePath))
+                    if (File.Exists(localFilePath))
+                    {
+                        var irdData = await File.ReadAllBytesAsync(localFilePath, cancellationToken).ConfigureAwait(false);
+                        try
+                        {
+                            result.Add(IrdParser.Parse(irdData));
+                        }
+                        catch (Exception e)
+                        {
+                            ApiConfig.Log.Warn(e, $"Failed to parse locally cached IRD file {localFilePath}");
+                            try { File.Delete(localFilePath); } catch {}
+                        }
+                    }
+                    else
                     {
                         try
                         {
                             var downloadLink = GetDownloadLink(item.Link);
-                            var fileBytes = await client.GetByteArrayAsync(downloadLink, cancellationToken).ConfigureAwait(false);
+                            var fileBytes = await Client.GetByteArrayAsync(downloadLink, cancellationToken).ConfigureAwait(false);
                             await File.WriteAllBytesAsync(localFilePath, fileBytes, cancellationToken).ConfigureAwait(false);
                             result.Add(IrdParser.Parse(fileBytes));
                         }
-                        catch (Exception ex)
+                        catch (Exception e)
                         {
-                            ApiConfig.Log.Warn(ex, $"Failed to download {item.Link}: {ex.Message}");
+                            ApiConfig.Log.Warn(e, $"Failed to download {item.Link}: {e.Message}");
                         }
                     }
                 }
-
                 ApiConfig.Log.Debug($"Returning {result.Count} .ird files for {productCode}");
                 return result;
             }
@@ -106,24 +124,7 @@ public async Task<List<Ird>> DownloadAsync(string productCode, string localCache
                 return result;
             }
         }
-        public static string GetDownloadLink(string relativeLink)
-        {
-            var fullUrl = new Uri(new Uri(BaseDownloadUri), relativeLink);
-            return Uri.EscapeUriString(fullUrl.ToString());
-        }
-    }
-
-    public class IrdInfo
-    {
-    [JsonPropertyName("title")]
-    public string Title { get; set; } = null!;
-    [JsonPropertyName("fw-ver")]
-    public string? FwVer { get; set; }
-    [JsonPropertyName("game-ver")]
-    public string? GameVer { get; set; }
-    [JsonPropertyName("app-ver")]
-    public string? AppVer { get; set; }
-    [JsonPropertyName("link")]
-    public string Link { get; set; } = null!;
+
+        public static Uri GetDownloadLink(string relativeLink) => new(BaseDownloadUri, relativeLink);
     }
 }

Clients/IrdLibraryClient/IrdLibraryClient.csproj

@@ -6,7 +6,6 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="DiscUtils.OpticalDisk" Version="0.16.13" />
-    <PackageReference Include="HtmlAgilityPack" Version="1.11.65" />
     <PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.1" />
     <PackageReference Include="System.IO.Hashing" Version="8.0.0" />
   </ItemGroup>

Clients/IrdLibraryClient/POCOs/IrdInfo.cs

@@ -0,0 +1,10 @@
+namespace IrdLibraryClient.POCOs;
+
+public class IrdInfo
+{
+    public string Title { get; set; } = null!;
+    public string? FwVer { get; set; }
+    public string? GameVer { get; set; }
+    public string? AppVer { get; set; }
+    public string Link { get; set; } = null!;
+}

Clients/PsnClient/PsnClient.csproj

@@ -11,7 +11,7 @@
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="Microsoft.AspNet.WebApi.Client" Version="6.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="8.0.1" />
     <PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.1" />
   </ItemGroup>
   <ItemGroup>

CompatBot/CompatBot.csproj

@@ -43,32 +43,32 @@
     <PackageReference Include="DSharpPlus.CommandsNext" Version="4.5.0" />
     <PackageReference Include="DSharpPlus.Interactivity" Version="4.5.0" />
     <PackageReference Include="DSharpPlus.SlashCommands" Version="4.5.0" />
-    <PackageReference Include="Google.Apis.Drive.v3" Version="1.68.0.3508" />
+    <PackageReference Include="Google.Apis.Drive.v3" Version="1.68.0.3568" />
     <PackageReference Include="ksemenenko.ColorThief" Version="1.1.1.4" />
     <PackageReference Include="MathParser.org-mXparser" Version="6.0.0" />
     <PackageReference Include="MegaApiClient" Version="1.10.4" />
     <PackageReference Include="Microsoft.ApplicationInsights.DependencyCollector" Version="2.22.0" />
     <PackageReference Include="Microsoft.ApplicationInsights.PerfCounterCollector" Version="2.22.0" />
     <PackageReference Include="Microsoft.Azure.CognitiveServices.Vision.ComputerVision" Version="7.0.1" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.8" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.8">
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.10" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.10">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.10" />
     <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="8.0.2" />
     <PackageReference Include="Microsoft.Extensions.Configuration.EnvironmentVariables" Version="8.0.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.UserSecrets" Version="8.0.1" />
     <PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.1" />
     <PackageReference Include="Microsoft.TeamFoundationServer.Client" Version="19.225.1" />
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.0" />
     <PackageReference Include="Nerdbank.Streams" Version="2.11.79" />
-    <PackageReference Include="NLog" Version="5.3.3" />
-    <PackageReference Include="NLog.Extensions.Logging" Version="5.3.12" />
+    <PackageReference Include="NLog" Version="5.3.4" />
+    <PackageReference Include="NLog.Extensions.Logging" Version="5.3.14" />
     <PackageReference Include="NReco.Text.AhoCorasickDoubleArrayTrie" Version="1.1.1" />
     <PackageReference Include="SharpCompress" Version="0.38.0" />
     <PackageReference Include="SixLabors.ImageSharp.Drawing" Version="2.1.4" />
-    <PackageReference Include="System.Drawing.Common" Version="8.0.8" />
+    <PackageReference Include="System.Drawing.Common" Version="8.0.10" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Clients\CirrusCiClient\CirrusCiClient.csproj" />

CompatBot/Utils/Extensions/MemoryCacheExtensions.cs

@@ -8,29 +8,43 @@ namespace CompatBot.Utils;
 
 internal static class MemoryCacheExtensions
 {
+    private static readonly object throaway = new object();
+
     public static List<T> GetCacheKeys<T>(this MemoryCache memoryCache)
     {
+        // idk why it requires access before it populates the internal state
+        memoryCache.TryGetValue("str", out _);
+        memoryCache.TryGetValue(throaway, out _);
+
+        // get the internal state object
         var stateField = memoryCache.GetType()
             .GetFields(BindingFlags.NonPublic | BindingFlags.Instance)
             .FirstOrDefault(fi => fi.Name == "_coherentState");
         var coherentState = stateField?.GetValue(memoryCache);
         if (coherentState is null)
         {
             Config.Log.Error($"Looks like {nameof(MemoryCache)} internals have changed");
-            return new();
+            return [];
         }
 
-        var field = coherentState.GetType()
+        // get the actual underlying key-value object
+        var stringField = coherentState.GetType()
             .GetFields(BindingFlags.NonPublic | BindingFlags.Instance)
-            .FirstOrDefault(fi => fi.Name == "_entries");
-        if (field is null)
+            .FirstOrDefault(fi => fi.Name == "_stringEntries");
+        var nonStringField = coherentState.GetType()
+            .GetFields(BindingFlags.NonPublic | BindingFlags.Instance)
+            .FirstOrDefault(fi => fi.Name == "_nonStringEntries");
+        if (stringField is null || nonStringField is null)
         {
             Config.Log.Error($"Looks like {nameof(MemoryCache)} internals have changed");
-            return new();
+            return [];
         }
 
-        var value = (IDictionary?)field.GetValue(coherentState);
-        return value?.Keys.OfType<T>().ToList() ?? new List<T>();
+        // read the keys
+        var value = typeof(T) == typeof(string)
+            ? (IDictionary?)stringField.GetValue(coherentState)
+            : (IDictionary?)nonStringField.GetValue(coherentState);
+        return value?.Keys.OfType<T>().ToList() ?? [];
     }
 
     public static Dictionary<TKey, ICacheEntry?> GetCacheEntries<TKey>(this MemoryCache memoryCache)

CompatBot/Utils/ResultFormatters/IrdSearchResultFormatter.cs

@@ -1,9 +1,12 @@
+using System;
 using CompatApiClient.Utils;
 using DSharpPlus.Entities;
 using IrdLibraryClient;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
+using CompatApiClient;
+using IrdLibraryClient.POCOs;
 
 namespace CompatBot.Utils.ResultFormatters
 {
@@ -16,20 +19,26 @@ public static DiscordEmbedBuilder AsEmbed(this List<IrdInfo> irdInfos)
                 // Title = "IRD Library Search Result",
                 Color = Config.Colors.DownloadLinks,
             };
-            if (irdInfos == null || !irdInfos.Any())
+            if (irdInfos is not {Count: >0})
             {
                 result.Color = Config.Colors.LogResultFailed;
                 result.Description = "No matches were found";
                 return result;
             }
-            foreach (var item in irdInfos)
+
+            foreach (var item in irdInfos.Where(i => i.Link is {Length: >5}).Take(EmbedPager.MaxFields))
             {
-                if (string.IsNullOrEmpty(item.Link))
-                    continue;
-                result.AddField(
-                    $"{item.Title} [v{item.GameVer} FW {item.FwVer}]",
-                    $"[⏬ {Path.GetFileName(item.Link)}]({IrdClient.GetDownloadLink(item.Link)})"
-                );
+                try
+                {
+                    result.AddField(
+                        $"{item.Title.Sanitize().Trim(EmbedPager.MaxFieldTitleLength - 18)} [v{item.GameVer} FW {item.FwVer}]",
+                        $"[⏬ {Path.GetFileName(item.Link)}]({IrdClient.GetDownloadLink(item.Link)})"
+                    );
+                }
+                catch (Exception e)
+                {
+                    ApiConfig.Log.Warn(e, "Failed to format embed field for IRD search result");
+                }
             }
             return result;
         }