Skip to content

Commit

Permalink
Update IrdClient.cs
Browse files Browse the repository at this point in the history
  • Loading branch information
@FlexBy420
FlexBy420 authored Sep 28, 2024
1 parent 86b5e5c commit c1150a1
Showing 1 changed file with 81 additions and 167 deletions.
248 changes: 81 additions & 167 deletions Clients/IrdLibraryClient/IrdClient.cs
View file
Original file line number Diff line number Diff line change
@@ -1,215 +1,129 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using CompatApiClient;
using CompatApiClient.Compression;
using CompatApiClient.Formatters;
using CompatApiClient.Utils;
using HtmlAgilityPack;
using IrdLibraryClient.IrdFormat;
using IrdLibraryClient.POCOs;

namespace IrdLibraryClient;

public class IrdClient
namespace IrdLibraryClient
{
public static readonly string BaseUrl = "https://ps3.aldostools.org";

private readonly HttpClient client;
private readonly JsonSerializerOptions jsonOptions;

public IrdClient()
public class IrdClient
{
client = HttpClientFactory.Create(new CompressionMessageHandler());
jsonOptions = new()
{
PropertyNamingPolicy = SpecialJsonNamingPolicy.SnakeCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
IncludeFields = true,
};
}

public static string GetDownloadLink(string irdFilename) => $"{BaseUrl}/ird/{irdFilename}";
public static readonly string JsonUrl = "https://flexby420.github.io/playstation_3_ird_database/all.json";
private readonly HttpClient client;
private readonly JsonSerializerOptions jsonOptions;
private static readonly string BaseDownloadUri = "https://github.com/FlexBy420/playstation_3_ird_database/raw/main/";

public async Task<SearchResult?> SearchAsync(string query, CancellationToken cancellationToken)
{
query = query.ToUpper();
try
public IrdClient()
{
var requestUri = new Uri(BaseUrl + "/ird.html");
using var getMessage = new HttpRequestMessage(HttpMethod.Get, requestUri);
using var response = await client.SendAsync(getMessage, cancellationToken).ConfigureAwait(false);
try
client = HttpClientFactory.Create(new CompressionMessageHandler());
jsonOptions = new JsonSerializerOptions
{
await response.Content.LoadIntoBufferAsync().ConfigureAwait(false);
var result = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
HtmlDocument doc = new();
doc.LoadHtml(result);
return new()
{
Data = doc.DocumentNode.Descendants("tr")
.Skip(1)
.Select(tr => tr.Elements("td").ToList())
.Where(tds => tds.Count > 1 && tds[0].InnerText == query)
.Select(tds =>
{
var i = tds.Select(td => td.InnerText.Trim()).ToArray();
return new SearchResultItem
{
Id = i[0],
Title = i[1],
GameVersion = i[2],
UpdateVersion = i[3],
Size = i[4],
FileCount = i[5],
FolderCount = i[6],
MD5 = i[7],
IrdName = i[8],
Filename = i[0] + "-" + i[8] + ".ird",
};
})
.ToList(),
};
}
catch (Exception e)
{
ConsoleLogger.PrintError(e, response);
return null;
}
}
catch (Exception e)
{
ApiConfig.Log.Error(e);
return null;
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
IncludeFields = true,
};
}
}

public async Task<List<Ird>> DownloadAsync(string productCode, string localCachePath, CancellationToken cancellationToken)
{
var result = new List<Ird>();
try
public async Task<List<IrdInfo>> SearchAsync(string query, CancellationToken cancellationToken)
{
// first we search local cache and try to load whatever data we can
var localCacheItems = new List<string>();
query = query.ToUpper();
try
{
var tmpCacheItemList = Directory.GetFiles(localCachePath, productCode + "*.ird", SearchOption.TopDirectoryOnly)
.Select(Path.GetFileName)
.ToList();
foreach (var item in tmpCacheItemList)
using var response = await client.GetAsync(JsonUrl, cancellationToken).ConfigureAwait(false);
if (!response.IsSuccessStatusCode)
{
if (string.IsNullOrEmpty(item))
continue;

try
{
result.Add(IrdParser.Parse(await File.ReadAllBytesAsync(Path.Combine(localCachePath, item), cancellationToken).ConfigureAwait(false)));
localCacheItems.Add(item);
}
catch (Exception ex)
{
ApiConfig.Log.Warn(ex, "Error reading local IRD file: " + ex.Message);
}
ApiConfig.Log.Error($"Failed to fetch IRD data: {response.StatusCode}");
return new List<IrdInfo>();
}
}
catch (Exception e)
{
ApiConfig.Log.Warn(e, "Error accessing local IRD cache: " + e.Message);
}
ApiConfig.Log.Debug($"Found {localCacheItems.Count} cached items for {productCode}");
SearchResult? searchResult = null;

// then try to do IRD Library search
try
{
searchResult = await SearchAsync(productCode, cancellationToken).ConfigureAwait(false);
var jsonResult = await response.Content.ReadAsStringAsync(cancellationToken).ConfigureAwait(false);
var irdData = JsonSerializer.Deserialize<Dictionary<string, List<IrdInfo>>>(jsonResult, jsonOptions);
if (irdData == null)
{
ApiConfig.Log.Error("Failed to deserialize IRD JSON data.");
return new List<IrdInfo>();
}

if (irdData.TryGetValue(query, out var items))
{
return items;
}

return new List<IrdInfo>();
}
catch (Exception e)
{
ApiConfig.Log.Error(e);
return new List<IrdInfo>();
}
var tmpFilesToGet = searchResult?.Data?
.Select(i => i.Filename)
.Except(localCacheItems, StringComparer.InvariantCultureIgnoreCase)
.ToList();
if (tmpFilesToGet is null or {Count: 0})
return result;
}

// as IRD Library could return more data than we found, try to check for all the items locally
var filesToDownload = new List<string>();
foreach (var item in tmpFilesToGet)
public async Task<List<Ird>> DownloadAsync(string productCode, string localCachePath, CancellationToken cancellationToken)
{
var result = new List<Ird>();
try
{
if (string.IsNullOrEmpty(item))
continue;

try
var searchResults = await SearchAsync(productCode, cancellationToken).ConfigureAwait(false);
if (searchResults == null || !searchResults.Any())
{
var localItemPath = Path.Combine(localCachePath, item);
if (File.Exists(localItemPath))
{
result.Add(IrdParser.Parse(await File.ReadAllBytesAsync(localItemPath, cancellationToken).ConfigureAwait(false)));
localCacheItems.Add(item);
}
else
filesToDownload.Add(item);
}
catch (Exception ex)
{
ApiConfig.Log.Warn(ex, "Error reading local IRD file: " + ex.Message);
filesToDownload.Add(item);
ApiConfig.Log.Debug($"No IRD files found for {productCode}");
return result;
}
}
ApiConfig.Log.Debug($"Found {tmpFilesToGet.Count} total matches for {productCode}, {result.Count} already cached");
if (filesToDownload.Count == 0)
return result;

// download the remaining .ird files
foreach (var item in filesToDownload)
{
try
foreach (var item in searchResults)
{
var resultBytes = await client.GetByteArrayAsync(GetDownloadLink(item), cancellationToken).ConfigureAwait(false);
result.Add(IrdParser.Parse(resultBytes));
try
var localFilePath = Path.Combine(localCachePath, $"{productCode}-{item.Link.Split('/').Last()}.ird");
if (!File.Exists(localFilePath))
{
await File.WriteAllBytesAsync(Path.Combine(localCachePath, item), resultBytes, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex)
{
ApiConfig.Log.Warn(ex, $"Failed to write {item} to local cache: {ex.Message}");
try
{
var downloadLink = GetDownloadLink(item.Link);
var fileBytes = await client.GetByteArrayAsync(downloadLink, cancellationToken).ConfigureAwait(false);
await File.WriteAllBytesAsync(localFilePath, fileBytes, cancellationToken).ConfigureAwait(false);
result.Add(IrdParser.Parse(fileBytes));
}
catch (Exception ex)
{
ApiConfig.Log.Warn(ex, $"Failed to download {item.Link}: {ex.Message}");
}
}
}
catch (Exception e)
{
ApiConfig.Log.Warn(e, $"Failed to download {item}: {e.Message}");
}

ApiConfig.Log.Debug($"Returning {result.Count} .ird files for {productCode}");
return result;
}
catch (Exception e)
{
ApiConfig.Log.Error(e);
return result;
}
ApiConfig.Log.Debug($"Returning {result.Count} .ird files for {productCode}");
return result;
}
catch (Exception e)
public static string GetDownloadLink(string relativeLink)
{
ApiConfig.Log.Error(e);
return result;
var fullUrl = new Uri(new Uri(BaseDownloadUri), relativeLink);
return Uri.EscapeUriString(fullUrl.ToString());
}
}

private static string? GetTitle(string? html)
public class IrdInfo
{
if (string.IsNullOrEmpty(html))
return null;

var idx = html.LastIndexOf("</span>", StringComparison.Ordinal);
var result = html[(idx + 7)..].Trim();
if (result is {Length: >0})
return result;
return null;
[JsonPropertyName("title")]
public string Title { get; set; } = null!;
[JsonPropertyName("fw-ver")]
public string? FwVer { get; set; }
[JsonPropertyName("game-ver")]
public string? GameVer { get; set; }
[JsonPropertyName("app-ver")]
public string? AppVer { get; set; }
[JsonPropertyName("link")]
public string Link { get; set; } = null!;
}
}
}

0 comments on commit c1150a1

Please sign in to comment.