From c1150a17e3083be49028e53d3f14913b49fa082f Mon Sep 17 00:00:00 2001 From: FlexBy420 <68403300+FlexBy420@users.noreply.github.com> Date: Sun, 29 Sep 2024 01:01:54 +0200 Subject: [PATCH 1/3] Update IrdClient.cs --- Clients/IrdLibraryClient/IrdClient.cs | 248 +++++++++----------------- 1 file changed, 81 insertions(+), 167 deletions(-) diff --git a/Clients/IrdLibraryClient/IrdClient.cs b/Clients/IrdLibraryClient/IrdClient.cs index 9c2e8788..b127bcf7 100644 --- a/Clients/IrdLibraryClient/IrdClient.cs +++ b/Clients/IrdLibraryClient/IrdClient.cs @@ -1,215 +1,129 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net.Http; using System.Text.Json; using System.Text.Json.Serialization; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using CompatApiClient; using CompatApiClient.Compression; -using CompatApiClient.Formatters; -using CompatApiClient.Utils; -using HtmlAgilityPack; using IrdLibraryClient.IrdFormat; using IrdLibraryClient.POCOs; -namespace IrdLibraryClient; - -public class IrdClient +namespace IrdLibraryClient { - public static readonly string BaseUrl = "https://ps3.aldostools.org"; - - private readonly HttpClient client; - private readonly JsonSerializerOptions jsonOptions; - - public IrdClient() + public class IrdClient { - client = HttpClientFactory.Create(new CompressionMessageHandler()); - jsonOptions = new() - { - PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase, - DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, - IncludeFields = true, - }; - } - - public static string GetDownloadLink(string irdFilename) => $"{BaseUrl}/ird/{irdFilename}"; + public static readonly string JsonUrl = "https://flexby420.github.io/playstation_3_ird_database/all.json"; + private readonly HttpClient client; + private readonly JsonSerializerOptions jsonOptions; + private static readonly string BaseDownloadUri = "https://github.com/FlexBy420/playstation_3_ird_database/raw/main/"; - public async Task SearchAsync(string query, CancellationToken cancellationToken) - { - query = query.ToUpper(); - try + public IrdClient() { - var requestUri = new Uri(BaseUrl + "/ird.html"); - using var getMessage = new HttpRequestMessage(HttpMethod.Get, requestUri); - using var response = await client.SendAsync(getMessage, cancellationToken).ConfigureAwait(false); - try + client = HttpClientFactory.Create(new CompressionMessageHandler()); + jsonOptions = new JsonSerializerOptions { - await response.Content.LoadIntoBufferAsync().ConfigureAwait(false); - var result = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); - HtmlDocument doc = new(); - doc.LoadHtml(result); - return new() - { - Data = doc.DocumentNode.Descendants("tr") - .Skip(1) - .Select(tr => tr.Elements("td").ToList()) - .Where(tds => tds.Count > 1 && tds[0].InnerText == query) - .Select(tds => - { - var i = tds.Select(td => td.InnerText.Trim()).ToArray(); - return new SearchResultItem - { - Id = i[0], - Title = i[1], - GameVersion = i[2], - UpdateVersion = i[3], - Size = i[4], - FileCount = i[5], - FolderCount = i[6], - MD5 = i[7], - IrdName = i[8], - Filename = i[0] + "-" + i[8] + ".ird", - }; - }) - .ToList(), - }; - } - catch (Exception e) - { - ConsoleLogger.PrintError(e, response); - return null; - } - } - catch (Exception e) - { - ApiConfig.Log.Error(e); - return null; + PropertyNamingPolicy = JsonNamingPolicy.CamelCase, + DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull, + IncludeFields = true, + }; } - } - public async Task> DownloadAsync(string productCode, string localCachePath, CancellationToken cancellationToken) - { - var result = new List(); - try + public async Task> SearchAsync(string query, CancellationToken cancellationToken) { - // first we search local cache and try to load whatever data we can - var localCacheItems = new List(); + query = query.ToUpper(); try { - var tmpCacheItemList = Directory.GetFiles(localCachePath, productCode + "*.ird", SearchOption.TopDirectoryOnly) - .Select(Path.GetFileName) - .ToList(); - foreach (var item in tmpCacheItemList) + using var response = await client.GetAsync(JsonUrl, cancellationToken).ConfigureAwait(false); + if (!response.IsSuccessStatusCode) { - if (string.IsNullOrEmpty(item)) - continue; - - try - { - result.Add(IrdParser.Parse(await File.ReadAllBytesAsync(Path.Combine(localCachePath, item), cancellationToken).ConfigureAwait(false))); - localCacheItems.Add(item); - } - catch (Exception ex) - { - ApiConfig.Log.Warn(ex, "Error reading local IRD file: " + ex.Message); - } + ApiConfig.Log.Error($"Failed to fetch IRD data: {response.StatusCode}"); + return new List(); } - } - catch (Exception e) - { - ApiConfig.Log.Warn(e, "Error accessing local IRD cache: " + e.Message); - } - ApiConfig.Log.Debug($"Found {localCacheItems.Count} cached items for {productCode}"); - SearchResult? searchResult = null; - // then try to do IRD Library search - try - { - searchResult = await SearchAsync(productCode, cancellationToken).ConfigureAwait(false); + var jsonResult = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false); + var irdData = JsonSerializer.Deserialize>>(jsonResult, jsonOptions); + if (irdData == null) + { + ApiConfig.Log.Error("Failed to deserialize IRD JSON data."); + return new List(); + } + + if (irdData.TryGetValue(query, out var items)) + { + return items; + } + + return new List(); } catch (Exception e) { ApiConfig.Log.Error(e); + return new List(); } - var tmpFilesToGet = searchResult?.Data? - .Select(i => i.Filename) - .Except(localCacheItems, StringComparer.InvariantCultureIgnoreCase) - .ToList(); - if (tmpFilesToGet is null or {Count: 0}) - return result; + } - // as IRD Library could return more data than we found, try to check for all the items locally - var filesToDownload = new List(); - foreach (var item in tmpFilesToGet) + public async Task> DownloadAsync(string productCode, string localCachePath, CancellationToken cancellationToken) + { + var result = new List(); + try { - if (string.IsNullOrEmpty(item)) - continue; - - try + var searchResults = await SearchAsync(productCode, cancellationToken).ConfigureAwait(false); + if (searchResults == null || !searchResults.Any()) { - var localItemPath = Path.Combine(localCachePath, item); - if (File.Exists(localItemPath)) - { - result.Add(IrdParser.Parse(await File.ReadAllBytesAsync(localItemPath, cancellationToken).ConfigureAwait(false))); - localCacheItems.Add(item); - } - else - filesToDownload.Add(item); - } - catch (Exception ex) - { - ApiConfig.Log.Warn(ex, "Error reading local IRD file: " + ex.Message); - filesToDownload.Add(item); + ApiConfig.Log.Debug($"No IRD files found for {productCode}"); + return result; } - } - ApiConfig.Log.Debug($"Found {tmpFilesToGet.Count} total matches for {productCode}, {result.Count} already cached"); - if (filesToDownload.Count == 0) - return result; - // download the remaining .ird files - foreach (var item in filesToDownload) - { - try + foreach (var item in searchResults) { - var resultBytes = await client.GetByteArrayAsync(GetDownloadLink(item), cancellationToken).ConfigureAwait(false); - result.Add(IrdParser.Parse(resultBytes)); - try + var localFilePath = Path.Combine(localCachePath, $"{productCode}-{item.Link.Split('/').Last()}.ird"); + if (!File.Exists(localFilePath)) { - await File.WriteAllBytesAsync(Path.Combine(localCachePath, item), resultBytes, cancellationToken).ConfigureAwait(false); - } - catch (Exception ex) - { - ApiConfig.Log.Warn(ex, $"Failed to write {item} to local cache: {ex.Message}"); + try + { + var downloadLink = GetDownloadLink(item.Link); + var fileBytes = await client.GetByteArrayAsync(downloadLink, cancellationToken).ConfigureAwait(false); + await File.WriteAllBytesAsync(localFilePath, fileBytes, cancellationToken).ConfigureAwait(false); + result.Add(IrdParser.Parse(fileBytes)); + } + catch (Exception ex) + { + ApiConfig.Log.Warn(ex, $"Failed to download {item.Link}: {ex.Message}"); + } } } - catch (Exception e) - { - ApiConfig.Log.Warn(e, $"Failed to download {item}: {e.Message}"); - } + + ApiConfig.Log.Debug($"Returning {result.Count} .ird files for {productCode}"); + return result; + } + catch (Exception e) + { + ApiConfig.Log.Error(e); + return result; } - ApiConfig.Log.Debug($"Returning {result.Count} .ird files for {productCode}"); - return result; } - catch (Exception e) + public static string GetDownloadLink(string relativeLink) { - ApiConfig.Log.Error(e); - return result; + var fullUrl = new Uri(new Uri(BaseDownloadUri), relativeLink); + return Uri.EscapeUriString(fullUrl.ToString()); } } - private static string? GetTitle(string? html) + public class IrdInfo { - if (string.IsNullOrEmpty(html)) - return null; - - var idx = html.LastIndexOf("", StringComparison.Ordinal); - var result = html[(idx + 7)..].Trim(); - if (result is {Length: >0}) - return result; - return null; + [JsonPropertyName("title")] + public string Title { get; set; } = null!; + [JsonPropertyName("fw-ver")] + public string? FwVer { get; set; } + [JsonPropertyName("game-ver")] + public string? GameVer { get; set; } + [JsonPropertyName("app-ver")] + public string? AppVer { get; set; } + [JsonPropertyName("link")] + public string Link { get; set; } = null!; } -} \ No newline at end of file +} From d6898162cd20f9b0d2c83073957d66c492861491 Mon Sep 17 00:00:00 2001 From: FlexBy420 <68403300+FlexBy420@users.noreply.github.com> Date: Sun, 29 Sep 2024 01:03:00 +0200 Subject: [PATCH 2/3] Update IrdSearchResultFormatter.cs --- .../IrdSearchResultFormatter.cs | 58 +++++++++---------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/CompatBot/Utils/ResultFormatters/IrdSearchResultFormatter.cs b/CompatBot/Utils/ResultFormatters/IrdSearchResultFormatter.cs index 303bac3c..765e6079 100644 --- a/CompatBot/Utils/ResultFormatters/IrdSearchResultFormatter.cs +++ b/CompatBot/Utils/ResultFormatters/IrdSearchResultFormatter.cs @@ -1,39 +1,37 @@ -using CompatApiClient.Utils; +using CompatApiClient.Utils; using DSharpPlus.Entities; using IrdLibraryClient; -using IrdLibraryClient.POCOs; +using System.Collections.Generic; +using System.IO; +using System.Linq; -namespace CompatBot.Utils.ResultFormatters; - -public static class IrdSearchResultFormatter +namespace CompatBot.Utils.ResultFormatters { - public static DiscordEmbedBuilder AsEmbed(this SearchResult? searchResult) + public static class IrdSearchResultFormatter { - var result = new DiscordEmbedBuilder - { - //Title = "IRD Library Search Result", - Color = Config.Colors.DownloadLinks, - }; - if (searchResult?.Data is null or {Count: 0}) + public static DiscordEmbedBuilder AsEmbed(this List irdInfos) { - result.Color = Config.Colors.LogResultFailed; - result.Description = "No matches were found"; + var result = new DiscordEmbedBuilder + { + // Title = "IRD Library Search Result", + Color = Config.Colors.DownloadLinks, + }; + if (irdInfos == null || !irdInfos.Any()) + { + result.Color = Config.Colors.LogResultFailed; + result.Description = "No matches were found"; + return result; + } + foreach (var item in irdInfos) + { + if (string.IsNullOrEmpty(item.Link)) + continue; + result.AddField( + $"[ {item.Title} v{item.GameVer} FW {item.FwVer} ]", + $"[⏬ {Path.GetFileName(item.Link)}]({IrdClient.GetDownloadLink(item.Link)})" + ); + } return result; } - - foreach (var item in searchResult.Data) - { - if (string.IsNullOrEmpty(item.Filename)) - continue; - - string[] parts = item.Filename.Split('-'); - if (parts.Length == 1) - parts = ["", item.Filename]; - result.AddField( - $"[{parts[0]} v{item.GameVersion}] {item.Title?.Sanitize().Trim(EmbedPager.MaxFieldTitleLength)}", - $"[⏬ `{parts[1].Sanitize().Trim(200)}`]({IrdClient.GetDownloadLink(item.Filename)})" - ); - } - return result; } -} \ No newline at end of file +} From 5016ad05add6aad29107d47257a5a83fec20c5d7 Mon Sep 17 00:00:00 2001 From: FlexBy420 <68403300+FlexBy420@users.noreply.github.com> Date: Sun, 29 Sep 2024 14:34:03 +0200 Subject: [PATCH 3/3] Update IrdSearchResultFormatter.cs --- CompatBot/Utils/ResultFormatters/IrdSearchResultFormatter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CompatBot/Utils/ResultFormatters/IrdSearchResultFormatter.cs b/CompatBot/Utils/ResultFormatters/IrdSearchResultFormatter.cs index 765e6079..3b12104b 100644 --- a/CompatBot/Utils/ResultFormatters/IrdSearchResultFormatter.cs +++ b/CompatBot/Utils/ResultFormatters/IrdSearchResultFormatter.cs @@ -27,7 +27,7 @@ public static DiscordEmbedBuilder AsEmbed(this List irdInfos) if (string.IsNullOrEmpty(item.Link)) continue; result.AddField( - $"[ {item.Title} v{item.GameVer} FW {item.FwVer} ]", + $"{item.Title} [v{item.GameVer} FW {item.FwVer}]", $"[⏬ {Path.GetFileName(item.Link)}]({IrdClient.GetDownloadLink(item.Link)})" ); }