Merge pull request #961 from FlexBy420/patch-3

Update ird command

Clients/IrdLibraryClient/IrdClient.cs

@@ -1,215 +1,129 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Net.Http;
 using System.Text.Json;
 using System.Text.Json.Serialization;
-using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
 using CompatApiClient;
 using CompatApiClient.Compression;
-using CompatApiClient.Formatters;
-using CompatApiClient.Utils;
-using HtmlAgilityPack;
 using IrdLibraryClient.IrdFormat;
 using IrdLibraryClient.POCOs;
 
-namespace IrdLibraryClient;
-
-public class IrdClient
+namespace IrdLibraryClient
 {
-    public static readonly string BaseUrl = "https://ps3.aldostools.org";
-
-    private readonly HttpClient client;
-    private readonly JsonSerializerOptions jsonOptions;
-
-    public IrdClient()
+    public class IrdClient
     {
-        client = HttpClientFactory.Create(new CompressionMessageHandler());
-        jsonOptions = new()
-        {
-            PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase,
-            DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
-            IncludeFields = true,
-        };
-    }
-
-    public static string GetDownloadLink(string irdFilename) => $"{BaseUrl}/ird/{irdFilename}";
+        public static readonly string JsonUrl = "https://flexby420.github.io/playstation_3_ird_database/all.json";
+        private readonly HttpClient client;
+        private readonly JsonSerializerOptions jsonOptions;
+        private static readonly string BaseDownloadUri = "https://github.com/FlexBy420/playstation_3_ird_database/raw/main/";
 
-    public async Task<SearchResult?> SearchAsync(string query, CancellationToken cancellationToken)
-    {
-        query = query.ToUpper();
-        try
+        public IrdClient()
         {
-            var requestUri = new Uri(BaseUrl + "/ird.html");
-            using var getMessage = new HttpRequestMessage(HttpMethod.Get, requestUri);
-            using var response = await client.SendAsync(getMessage, cancellationToken).ConfigureAwait(false);
-            try
+            client = HttpClientFactory.Create(new CompressionMessageHandler());
+            jsonOptions = new JsonSerializerOptions
             {
-                await response.Content.LoadIntoBufferAsync().ConfigureAwait(false);
-                var result = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
-                HtmlDocument doc = new();
-                doc.LoadHtml(result);
-                return new()
-                {
-                    Data = doc.DocumentNode.Descendants("tr")
-                        .Skip(1)
-                        .Select(tr => tr.Elements("td").ToList())
-                        .Where(tds => tds.Count > 1 && tds[0].InnerText == query)
-                        .Select(tds =>
-                        {
-                            var i = tds.Select(td => td.InnerText.Trim()).ToArray();
-                            return new SearchResultItem
-                            {
-                                Id = i[0],
-                                Title = i[1],
-                                GameVersion = i[2],
-                                UpdateVersion = i[3],
-                                Size = i[4],
-                                FileCount = i[5],
-                                FolderCount = i[6],
-                                MD5 = i[7],
-                                IrdName = i[8],
-                                Filename = i[0] + "-" + i[8] + ".ird",
-                            };
-                        })
-                        .ToList(),
-                };
-            }
-            catch (Exception e)
-            {
-                ConsoleLogger.PrintError(e, response);
-                return null;
-            }
-        }
-        catch (Exception e)
-        {
-            ApiConfig.Log.Error(e);
-            return null;
+                PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
+                DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
+                IncludeFields = true,
+            };
         }
-    }
 
-    public async Task<List<Ird>> DownloadAsync(string productCode, string localCachePath, CancellationToken cancellationToken)
-    {
-        var result = new List<Ird>();
-        try
+        public async Task<List<IrdInfo>> SearchAsync(string query, CancellationToken cancellationToken)
         {
-            // first we search local cache and try to load whatever data we can
-            var localCacheItems = new List<string>();
+            query = query.ToUpper();
             try
             {
-                var tmpCacheItemList = Directory.GetFiles(localCachePath, productCode + "*.ird", SearchOption.TopDirectoryOnly)
-                    .Select(Path.GetFileName)
-                    .ToList();
-                foreach (var item in tmpCacheItemList)
+                using var response = await client.GetAsync(JsonUrl, cancellationToken).ConfigureAwait(false);
+                if (!response.IsSuccessStatusCode)
                 {
-                    if (string.IsNullOrEmpty(item))
-                        continue;
-
-                    try
-                    {
-                        result.Add(IrdParser.Parse(await File.ReadAllBytesAsync(Path.Combine(localCachePath, item), cancellationToken).ConfigureAwait(false)));
-                        localCacheItems.Add(item);
-                    }
-                    catch (Exception ex)
-                    {
-                        ApiConfig.Log.Warn(ex, "Error reading local IRD file: " + ex.Message);
-                    }
+                    ApiConfig.Log.Error($"Failed to fetch IRD data: {response.StatusCode}");
+                    return new List<IrdInfo>();
                 }
-            }
-            catch (Exception e)
-            {
-                ApiConfig.Log.Warn(e, "Error accessing local IRD cache: " + e.Message);
-            }
-            ApiConfig.Log.Debug($"Found {localCacheItems.Count} cached items for {productCode}");
-            SearchResult? searchResult = null;
 
-            // then try to do IRD Library search
-            try
-            {
-                searchResult = await SearchAsync(productCode, cancellationToken).ConfigureAwait(false);
+                var jsonResult = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
+                var irdData = JsonSerializer.Deserialize<Dictionary<string, List<IrdInfo>>>(jsonResult, jsonOptions);
+                if (irdData == null)
+                {
+                    ApiConfig.Log.Error("Failed to deserialize IRD JSON data.");
+                    return new List<IrdInfo>();
+                }
+
+                if (irdData.TryGetValue(query, out var items))
+                {
+                    return items;
+                }
+
+                return new List<IrdInfo>();
             }
             catch (Exception e)
             {
                 ApiConfig.Log.Error(e);
+                return new List<IrdInfo>();
             }
-            var tmpFilesToGet = searchResult?.Data?
-                .Select(i => i.Filename)
-                .Except(localCacheItems, StringComparer.InvariantCultureIgnoreCase)
-                .ToList();
-            if (tmpFilesToGet is null or {Count: 0})
-                return result;
+        }
 
-            // as IRD Library could return more data than we found, try to check for all the items locally
-            var filesToDownload = new List<string>();
-            foreach (var item in tmpFilesToGet)
+        public async Task<List<Ird>> DownloadAsync(string productCode, string localCachePath, CancellationToken cancellationToken)
+        {
+            var result = new List<Ird>();
+            try
             {
-                if (string.IsNullOrEmpty(item))
-                    continue;
-
-                try
+                var searchResults = await SearchAsync(productCode, cancellationToken).ConfigureAwait(false);
+                if (searchResults == null || !searchResults.Any())
                 {
-                    var localItemPath = Path.Combine(localCachePath, item);
-                    if (File.Exists(localItemPath))
-                    {
-                        result.Add(IrdParser.Parse(await File.ReadAllBytesAsync(localItemPath, cancellationToken).ConfigureAwait(false)));
-                        localCacheItems.Add(item);
-                    }
-                    else
-                        filesToDownload.Add(item);
-                }
-                catch (Exception ex)
-                {
-                    ApiConfig.Log.Warn(ex, "Error reading local IRD file: " + ex.Message);
-                    filesToDownload.Add(item);
+                    ApiConfig.Log.Debug($"No IRD files found for {productCode}");
+                    return result;
                 }
-            }
-            ApiConfig.Log.Debug($"Found {tmpFilesToGet.Count} total matches for {productCode}, {result.Count} already cached");
-            if (filesToDownload.Count == 0)
-                return result;
 
-            // download the remaining .ird files
-            foreach (var item in filesToDownload)
-            {
-                try
+                foreach (var item in searchResults)
                 {
-                    var resultBytes = await client.GetByteArrayAsync(GetDownloadLink(item), cancellationToken).ConfigureAwait(false);
-                    result.Add(IrdParser.Parse(resultBytes));
-                    try
+                    var localFilePath = Path.Combine(localCachePath, $"{productCode}-{item.Link.Split('/').Last()}.ird");
+                    if (!File.Exists(localFilePath))
                     {
-                        await File.WriteAllBytesAsync(Path.Combine(localCachePath, item), resultBytes, cancellationToken).ConfigureAwait(false);
-                    }
-                    catch (Exception ex)
-                    {
-                        ApiConfig.Log.Warn(ex, $"Failed to write {item} to local cache: {ex.Message}");
+                        try
+                        {
+                            var downloadLink = GetDownloadLink(item.Link);
+                            var fileBytes = await client.GetByteArrayAsync(downloadLink, cancellationToken).ConfigureAwait(false);
+                            await File.WriteAllBytesAsync(localFilePath, fileBytes, cancellationToken).ConfigureAwait(false);
+                            result.Add(IrdParser.Parse(fileBytes));
+                        }
+                        catch (Exception ex)
+                        {
+                            ApiConfig.Log.Warn(ex, $"Failed to download {item.Link}: {ex.Message}");
+                        }
                     }
                 }
-                catch (Exception e)
-                {
-                    ApiConfig.Log.Warn(e, $"Failed to download {item}: {e.Message}");
-                }
+
+                ApiConfig.Log.Debug($"Returning {result.Count} .ird files for {productCode}");
+                return result;
+            }
+            catch (Exception e)
+            {
+                ApiConfig.Log.Error(e);
+                return result;
             }
-            ApiConfig.Log.Debug($"Returning {result.Count} .ird files for {productCode}");
-            return result;
         }
-        catch (Exception e)
+        public static string GetDownloadLink(string relativeLink)
         {
-            ApiConfig.Log.Error(e);
-            return result;
+            var fullUrl = new Uri(new Uri(BaseDownloadUri), relativeLink);
+            return Uri.EscapeUriString(fullUrl.ToString());
         }
     }
 
-    private static string? GetTitle(string? html)
+    public class IrdInfo
     {
-        if (string.IsNullOrEmpty(html))
-            return null;
-
-        var idx = html.LastIndexOf("</span>", StringComparison.Ordinal);
-        var result = html[(idx + 7)..].Trim();
-        if (result is {Length: >0})
-            return result;
-        return null;
+    [JsonPropertyName("title")]
+    public string Title { get; set; } = null!;
+    [JsonPropertyName("fw-ver")]
+    public string? FwVer { get; set; }
+    [JsonPropertyName("game-ver")]
+    public string? GameVer { get; set; }
+    [JsonPropertyName("app-ver")]
+    public string? AppVer { get; set; }
+    [JsonPropertyName("link")]
+    public string Link { get; set; } = null!;
     }
-}
+}

CompatBot/Utils/ResultFormatters/IrdSearchResultFormatter.cs

@@ -1,39 +1,37 @@
-using CompatApiClient.Utils;
+using CompatApiClient.Utils;
 using DSharpPlus.Entities;
 using IrdLibraryClient;
-using IrdLibraryClient.POCOs;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
 
-namespace CompatBot.Utils.ResultFormatters;
-
-public static class IrdSearchResultFormatter
+namespace CompatBot.Utils.ResultFormatters
 {
-    public static DiscordEmbedBuilder AsEmbed(this SearchResult? searchResult)
+    public static class IrdSearchResultFormatter
     {
-        var result = new DiscordEmbedBuilder
-        {
-            //Title = "IRD Library Search Result",
-            Color = Config.Colors.DownloadLinks,
-        };
-        if (searchResult?.Data is null or {Count: 0})
+        public static DiscordEmbedBuilder AsEmbed(this List<IrdInfo> irdInfos)
         {
-            result.Color = Config.Colors.LogResultFailed;
-            result.Description = "No matches were found";
+            var result = new DiscordEmbedBuilder
+            {
+                // Title = "IRD Library Search Result",
+                Color = Config.Colors.DownloadLinks,
+            };
+            if (irdInfos == null || !irdInfos.Any())
+            {
+                result.Color = Config.Colors.LogResultFailed;
+                result.Description = "No matches were found";
+                return result;
+            }
+            foreach (var item in irdInfos)
+            {
+                if (string.IsNullOrEmpty(item.Link))
+                    continue;
+                result.AddField(
+                    $"{item.Title} [v{item.GameVer} FW {item.FwVer}]",
+                    $"[⏬ {Path.GetFileName(item.Link)}]({IrdClient.GetDownloadLink(item.Link)})"
+                );
+            }
             return result;
         }
-
-        foreach (var item in searchResult.Data)
-        {
-            if (string.IsNullOrEmpty(item.Filename))
-                continue;
-
-            string[] parts = item.Filename.Split('-');
-            if (parts.Length == 1)
-                parts = ["", item.Filename];
-            result.AddField(
-                $"[{parts[0]} v{item.GameVersion}] {item.Title?.Sanitize().Trim(EmbedPager.MaxFieldTitleLength)}",
-                $"[⏬ `{parts[1].Sanitize().Trim(200)}`]({IrdClient.GetDownloadLink(item.Filename)})"
-            );
-        }
-        return result;
     }
-}
+}