ecenshu
1678a1e3b5
Watch for KeySet, initial install will not have a valid value TitleDatabase will use data folder
316 lines
12 KiB
C#
316 lines
12 KiB
C#
using System.Text.RegularExpressions;
|
||
using System.Text.Json;
|
||
using Microsoft.Extensions.Caching.Memory;
|
||
using Microsoft.Extensions.Options;
|
||
using TinfoilVibeServer.Models;
|
||
|
||
namespace TinfoilVibeServer.Services;
|
||
|
||
/// <summary>
|
||
/// * Loads the title‑database JSON that lives on GitHub.
|
||
/// * Caches the JSON file on disk (in a configurable “cache” folder).
|
||
/// * Builds a dictionary that maps a 16‑digit hex TitleId → the full
|
||
/// filesystem path of the NSP that contains it (for later look‑ups).
|
||
/// * Provides a convenient look‑up API (via <c>GetTitleByTitleId</c>).
|
||
/// </summary>
|
||
public sealed class TitleDatabaseService : IHostedService
|
||
{
|
||
#region Configuration keys
|
||
private const string CacheKey = "TitleDb";
|
||
#endregion
|
||
|
||
#region Fields
|
||
|
||
private readonly IOptionsMonitor<TitleDbOptions> _options;
|
||
private readonly ILogger<TitleDatabaseService> _logger;
|
||
private readonly IHttpClientFactory _httpFactory;
|
||
private readonly string _cacheFolder; // Where the JSON is cached.
|
||
|
||
private readonly IMemoryCache _cache;
|
||
private readonly ISnapshotService _snapshotService;
|
||
|
||
private readonly Dictionary<string,string> _titleIdToPath = new Dictionary<string, string>();
|
||
|
||
// Regex to find a 16‑digit hex TitleId in a filename
|
||
private static readonly Regex TitleIdRegex = new(
|
||
@"([0-9a-fA-F]{8})([0-9a-fA-F]{8})", RegexOptions.Compiled);
|
||
private readonly SemaphoreSlim _reloadLock = new(1, 1); // protects reload logic
|
||
private Task? _reloadTask; // the currently running or last finished reload
|
||
|
||
#endregion
|
||
|
||
#region ctor
|
||
|
||
/// <summary>
|
||
/// Register as a singleton IHostedService.
|
||
/// The constructor receives the values that are needed to build
|
||
/// the GitHub URL (CountryCode + Language) and the root
|
||
/// directories that contain the NSP files.
|
||
/// </summary>
|
||
public TitleDatabaseService(
|
||
IOptionsMonitor<TitleDbOptions> options,
|
||
ILogger<TitleDatabaseService> logger,
|
||
ISnapshotService snapshotService,
|
||
IHttpClientFactory httpFactory,
|
||
INSPExtractor nspExtractor,
|
||
IMemoryCache cache)
|
||
{
|
||
_options = options;
|
||
_logger = logger;
|
||
_snapshotService = snapshotService;
|
||
_httpFactory = httpFactory;
|
||
_cache = cache;
|
||
|
||
_cacheFolder = Path.Combine(AppContext.BaseDirectory, "data", "titledb-cache");
|
||
new List<string>
|
||
{
|
||
// You can extend this list – it is the set of directories that
|
||
// are scanned when the service starts up.
|
||
Path.Combine(AppContext.BaseDirectory, "Games")
|
||
};
|
||
// Reload cache immediately when a snapshot rebuild occurs
|
||
_snapshotService.SnapshotRebuilt += SnapshotServiceOnSnapshotRebuilt;
|
||
_options.OnChange(OptionsChanged);
|
||
}
|
||
|
||
private void OptionsChanged(TitleDbOptions arg1, string? arg2)
|
||
{
|
||
// todo: handle ttl changes
|
||
// todo: handle country code change
|
||
// todo: handle language change
|
||
}
|
||
|
||
#endregion
|
||
|
||
#region IHostedService
|
||
|
||
public async Task StartAsync(CancellationToken cancellationToken)
|
||
{
|
||
// 1️⃣ Load the JSON (download if not cached).
|
||
await ReloadCacheAsync();
|
||
|
||
_logger.LogInformation("Title database ready – {Count} entries loaded.",
|
||
GetAllAsync().Result.Count);
|
||
}
|
||
|
||
public Task StopAsync(CancellationToken cancellationToken)
|
||
=> Task.CompletedTask; // nothing special to do on shutdown
|
||
|
||
#endregion
|
||
|
||
/* ---------------------------------------------------------------- */
|
||
/* 1️⃣ Cache loading / reloading – sliding expiration
|
||
/* ---------------------------------------------------------------- */
|
||
private async Task ReloadCacheAsync()
|
||
{
|
||
var ttlSec = _options.CurrentValue.TtlSeconds;
|
||
_logger.LogInformation("Reloading title database cache (TTL={TTL}s).", ttlSec);
|
||
|
||
var entryOptions = new MemoryCacheEntryOptions()
|
||
.SetSlidingExpiration(TimeSpan.FromSeconds(ttlSec))
|
||
.RegisterPostEvictionCallback((key, value, reason, state) => { _logger.LogInformation("Cache eviction: {Key} ({Reason})", key, reason); });
|
||
|
||
var dict = await LoadFromDiskAsync() ?? new Dictionary<string, TitleInfoDto>();
|
||
_cache.Set(CacheKey, dict, entryOptions);
|
||
_logger.LogInformation("Title DB reloaded – {Count} items cached (TTL={TTL}s).",
|
||
dict.Count, ttlSec);
|
||
}
|
||
|
||
private async Task<Dictionary<string, TitleInfoDto>?> LoadFromDiskAsync()
|
||
{
|
||
var cacheFile = Path.Combine(_cacheFolder, $"{_options.CurrentValue.CountryCode}.{_options.CurrentValue.LanguageCode}.json");
|
||
if (!File.Exists(cacheFile))
|
||
{
|
||
_logger.LogInformation("Cache miss – downloading title DB from {Url}", cacheFile);
|
||
await LoadAndCacheTitleDb(CancellationToken.None);
|
||
}
|
||
|
||
return await ReadTitleDbAsync(cacheFile, CancellationToken.None);
|
||
}
|
||
|
||
/* ---------------------------------------------------------------- */
|
||
/* 2️⃣ Public API – every call slides the cache
|
||
/* ---------------------------------------------------------------- */
|
||
public async Task<Dictionary<string, TitleInfoDto>> GetAllAsync()
|
||
{
|
||
await EnsureCacheLoadedAsync().ConfigureAwait(false);
|
||
|
||
// The cache entry is guaranteed to exist now
|
||
_cache.TryGetValue(CacheKey, out Dictionary<string, TitleInfoDto>? dict);
|
||
return dict!;
|
||
}
|
||
|
||
public async Task<TitleInfoDto?> GetAsync(string titleId)
|
||
{
|
||
var all = await GetAllAsync(); // slides the entry
|
||
all.TryGetValue(titleId, out var dto);
|
||
return dto;
|
||
}
|
||
|
||
/* ---------------------------------------------------------------- */
|
||
/* 3️⃣ Persist to disk & notify snapshot service
|
||
/* ---------------------------------------------------------------- */
|
||
private async Task PersistAsync(Dictionary<string, TitleInfoDto> dict)
|
||
{
|
||
// Trigger a rebuild so SnapshotService (and any other listeners)
|
||
// can pick up the new snapshot.
|
||
//_snapshotService.RebuildSnapshot();
|
||
await Task.CompletedTask;
|
||
}
|
||
|
||
/* ---------------------------------------------------------------- */
|
||
/* 4️⃣ Dispose
|
||
/* ---------------------------------------------------------------- */
|
||
public void Dispose()
|
||
{
|
||
_snapshotService.SnapshotRebuilt -= SnapshotServiceOnSnapshotRebuilt;
|
||
_reloadLock.Dispose();
|
||
}
|
||
|
||
private async void SnapshotServiceOnSnapshotRebuilt(object? o, EventArgs eventArgs)
|
||
{
|
||
try
|
||
{
|
||
await EnsureCacheLoadedAsync();
|
||
}
|
||
catch (Exception e)
|
||
{
|
||
_logger.LogCritical(e, "Failed to reload title database cache");
|
||
}
|
||
}
|
||
|
||
#region Public API
|
||
|
||
/// <summary>
|
||
/// Return the <c>TitleInfoDto</c> for a known <c>TitleId</c>.
|
||
/// </summary>
|
||
public bool TryGetTitle(string titleId, out TitleInfoDto? title)
|
||
{
|
||
title = GetAsync(titleId).GetAwaiter().GetResult();
|
||
return title != null;
|
||
}
|
||
#endregion
|
||
|
||
#region Private helpers
|
||
/// <summary>
|
||
/// Makes sure the cache is loaded. If a reload is already in progress,
|
||
/// the caller simply awaits the same task. Otherwise it starts a new reload.
|
||
/// </summary>
|
||
private async Task EnsureCacheLoadedAsync()
|
||
{
|
||
// Fast path – a cache entry exists, no work needed
|
||
if (_cache.TryGetValue(CacheKey, out _))
|
||
return;
|
||
|
||
// Fast path – a reload is already underway
|
||
var existingTask = _reloadTask;
|
||
if (existingTask != null)
|
||
{
|
||
await existingTask.ConfigureAwait(false);
|
||
return;
|
||
}
|
||
|
||
// Slow path – we need to start the reload
|
||
await _reloadLock.WaitAsync().ConfigureAwait(false);
|
||
try
|
||
{
|
||
// Double‑check after acquiring the lock
|
||
if (_cache.TryGetValue(CacheKey, out _))
|
||
return;
|
||
|
||
// If another thread started the reload while we were waiting, use it
|
||
existingTask = _reloadTask;
|
||
if (existingTask != null)
|
||
{
|
||
await existingTask.ConfigureAwait(false);
|
||
return;
|
||
}
|
||
|
||
// Create the shared task
|
||
_reloadTask = ReloadCacheAsync();
|
||
await _reloadTask.ConfigureAwait(false);
|
||
}
|
||
finally
|
||
{
|
||
_reloadTask = null; // reset for the next miss
|
||
_reloadLock.Release();
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Downloads the JSON file from GitHub (raw) if it does not exist
|
||
/// or the cached copy is older than the remote one.
|
||
/// </summary>
|
||
private async Task LoadAndCacheTitleDb(CancellationToken ct)
|
||
{
|
||
// Build the raw URL
|
||
var rawUrl = $"https://raw.githubusercontent.com/blawar/titledb/refs/heads/master/{_options.CurrentValue.CountryCode}.{_options.CurrentValue.LanguageCode}.json";
|
||
|
||
// Ensure the cache directory exists.
|
||
Directory.CreateDirectory(_cacheFolder);
|
||
var cacheFile = Path.Combine(_cacheFolder, $"{_options.CurrentValue.CountryCode}.{_options.CurrentValue.LanguageCode}.json");
|
||
|
||
// If the file exists & is recent – no download needed.
|
||
if (File.Exists(cacheFile))
|
||
{
|
||
var fi = new FileInfo(cacheFile);
|
||
// If the file is newer than 24h – use it.
|
||
if (fi.LastWriteTimeUtc > DateTime.UtcNow.AddHours(-24))
|
||
{
|
||
_logger.LogInformation("Using cached title database {File}", cacheFile);
|
||
await LoadAndCacheTitleDb(ct);
|
||
return;
|
||
}
|
||
_logger.LogInformation("Cache miss – downloading title DB from {Url}", cacheFile);
|
||
}
|
||
|
||
_logger.LogInformation("Downloading title database from {Url}", rawUrl);
|
||
var client = _httpFactory.CreateClient();
|
||
using var response = await client.GetAsync(rawUrl, ct);
|
||
response.EnsureSuccessStatusCode();
|
||
await using var fs = new FileStream(cacheFile, FileMode.Create, FileAccess.Write, FileShare.None);
|
||
await response.Content.CopyToAsync(fs, ct);
|
||
|
||
_logger.LogInformation("Title database cached to {File}", cacheFile);
|
||
await ReadTitleDbAsync(cacheFile, ct);
|
||
}
|
||
|
||
/// <summary>
|
||
/// Read the JSON file and populate <c>_titleData</c>.
|
||
/// Also remap titleId for XCI files based on cnmts.json
|
||
/// </summary>
|
||
private async Task<Dictionary<string,TitleInfoDto>> ReadTitleDbAsync(string filePath, CancellationToken ct)
|
||
{
|
||
var json = await File.ReadAllTextAsync(filePath, ct);
|
||
|
||
var titleInfoDtos = JsonSerializer.Deserialize<Dictionary<string, TitleInfoDto>>(
|
||
json,
|
||
new JsonSerializerOptions
|
||
{
|
||
PropertyNameCaseInsensitive = true,
|
||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||
}) ?? new Dictionary<string, TitleInfoDto>(StringComparer.OrdinalIgnoreCase);
|
||
|
||
_logger.LogInformation("Loaded {Count} titles from the database.", titleInfoDtos.Count);
|
||
|
||
var titleData = new Dictionary<string, TitleInfoDto>(StringComparer.OrdinalIgnoreCase);
|
||
for (var i =0; i< titleInfoDtos.Values.Count; i++)
|
||
{
|
||
var entry = titleInfoDtos.Values.ElementAt(i);
|
||
var key = titleInfoDtos.Keys.ElementAt(i);
|
||
if (!string.IsNullOrWhiteSpace(key))
|
||
{
|
||
if (entry.Id != null)
|
||
{
|
||
if (entry.Id.Length == 16)
|
||
{
|
||
titleData[entry.Id] = entry;
|
||
continue;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
return titleData;
|
||
}
|
||
#endregion
|
||
} |