ecenshu/TinfoilVibeServer
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

TitleDb loading is handled for multithreading

Browse Source 
Fix some memory leaks
This commit is contained in:
Huy Nguyen
2025-11-07 14:31:59 +10:30
parent 209b766a1f
commit c2ed73e03f
6 changed files with 97 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
TinfoilVibeServer.sln.DotSettings.user
+1
View File
@@ -52,6 +52,7 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APartitionFileSystemMetaCore_00603_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F77078e9a1d254191bb508f54a277fc6e1c2e00_003F84_003F3a6a0a14_003FPartitionFileSystemMetaCore_00603_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APartitionFileSystem_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F77078e9a1d254191bb508f54a277fc6e1c2e00_003F94_003F00e38e6a_003FPartitionFileSystem_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APath_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fcf5011822fd54235e86fdc54ee3baa876b4876e5549223ffeadca5607e59f6af_003FPath_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003APath_002EWindows_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003F8bfc57955751d07e3fad13bb97d5f66af412d7a34486ec29ad916547ec8ce6b_003FPath_002EWindows_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARandomAccess_002EWindows_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003F5cb442cc26c0b982afc257bb62f18d1554630732b2b8245e97107fb51974d_003FRandomAccess_002EWindows_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARarArchiveEntry_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003Fc9e95c2dd9e75e585f636790bc74e1484a536fc029954c307fe585a3822129_003FRarArchiveEntry_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003ARarArchiveVolumeFactory_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003F416daa6b333a5ecff282ed559ac81071113ee993ebcdc662c5cf488d7072575b_003FRarArchiveVolumeFactory_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
TinfoilVibeServer/Services/ArchiveHandler.cs
+6 -1
View File
@@ -114,6 +114,7 @@ public sealed class ArchiveHandler : IArchiveHandler
var entryCount = 0;
try
{
// todo: handle and skip multipart archives
using var archive = RarArchive.Open(path);
entryCount = archive.Entries.Count;
foreach (var entry in archive.Entries)
@@ -122,10 +123,14 @@ public sealed class ArchiveHandler : IArchiveHandler
{
try
{
using var streamWrapper = new SeekableBufferedStream(entry.OpenEntryStream(), entry.Size, 64 * 1024 * 1024, false);
using var streamWrapper = new SeekableBufferedStream(entry.OpenEntryStream(), entry.Size, 64 * 1024 * 1024, true);
var title = _nspExtractor.ExtractFromStream(streamWrapper);
if (title != null) titles.Add((entry.Key, entry.Size, title));
}
catch (IncompleteArchiveException incompleteArchiveException)
{
_logger.LogWarning("Incomplete archive {Archive}: {Exception}", path, incompleteArchiveException.Message);
}
catch (Exception e)
{
if (e.Message.StartsWith("Failed to extract NSP"))
TinfoilVibeServer/Services/IndexBuilderService.cs
+5 -9
View File
@@ -1,13 +1,7 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Security.Cryptography;
using System.Security.Cryptography;
using System.Text.Json;
using System.Text.RegularExpressions;
using LibHac.Ncm;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
using TinfoilVibeServer.Models;
namespace TinfoilVibeServer.Services;
@@ -86,7 +80,7 @@ public sealed class IndexBuilderService: IHostedService
{
var titleId = title.TitleId;
var name = _titleDb.TryGetTitle(titleId, out var t)
? t.Name
? t?.Name
: "Unknown";
var versionNumberParsed = (title.ContentMetaType == ContentMetaType.Application) ? title.Version : title.Version * 0x10000;
@@ -96,7 +90,7 @@ public sealed class IndexBuilderService: IHostedService
{
// Patch should use the application title name
name = _titleDb.TryGetTitle(title.ApplicationTitle, out var appTitle)
? appTitle.Name
? appTitle?.Name
: "Unknown";
//_logger.LogInformation("Patch title {TitleId} uses application title {ApplicationTitle}", titleId, title.ApplicationTitle);
}
@@ -129,7 +123,9 @@ public sealed class IndexBuilderService: IHostedService
private IndexCache? LoadCache()
{
if (!File.Exists(_cachePath)) return null;
_lock.Wait();
var json = File.ReadAllText(_cachePath);
_lock.Release();
_logger.LogInformation("Loaded index cache from {Path}", _cachePath);
return JsonSerializer.Deserialize<IndexCache>(json);
}
TinfoilVibeServer/Services/NSPExtractor.cs
+5 -3
View File
@@ -186,13 +186,15 @@ namespace TinfoilVibeServer.Services
var storage = new StreamStorage(stream, true);
var partition = new PartitionFileSystem();
partition.Initialize(storage).ThrowIfFailure();
_logger.LogInformation("PFS0 found");
return true;
}
catch (Exception e)
catch
{
_logger.LogError("Failed to extract NSP: {Exception}", e.Message);
return false;
// ignored
}
return false;
}
private bool IsXciFileSystem(Stream stream)
{
TinfoilVibeServer/Services/SnapshotService.cs
+16 -6
View File
@@ -543,9 +543,16 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
public void RebuildSnapshot()
{
// Build a fresh snapshot and persist it.
BuildSnapshotAsync(); // private method inside the same class
PersistSnapshotAsync(); // private method inside the same class
// 1️⃣ Flush the old in‑memory snapshot
_cache.Clear();
_hashCache.Clear();
_archiveLookup.Clear();
_sizeLookup.Clear();
//_failedAttempts.Clear(); // if you keep per‑user counters
// 2️⃣ Re‑build from disk again
BuildSnapshotAsync().Wait(); // synchronous – we already own the lock
PersistSnapshotAsync().Wait(); // same
SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
}
#endregion
@@ -602,7 +609,6 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
var ext = Path.GetExtension(filePath).ToLowerInvariant();
if (ext is not ".nsp" and not ".xci" and not ".xcz")
{
// Open the NSP/XCI with LibHac and read the first stream.
// The first stream is the first entry returned by GetContentInfos().
try
@@ -612,12 +618,16 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
var first = reader.GetEntries().FirstOrDefault();
if (first == null) return ComputeFullHash(filePath);
return _nspExtractor.ExtractHashFromStream(first.Stream);
using var firstStream = first.Stream;
var hash = _nspExtractor.ExtractHashFromStream(firstStream);
return hash;
}
catch
{
// On error, fall back to the full file hash
return ComputeFullHash(filePath);
using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
var ncaMetadataWithHash = _nspExtractor.ExtractFromStream(fs);
return ncaMetadataWithHash?.Hash ?? string.Empty;
}
}
else
TinfoilVibeServer/Services/TitleDatabaseService.cs
+64 -21
View File
@@ -36,7 +36,9 @@ public sealed class TitleDatabaseService : IHostedService
// Regex to find a 16‑digit hex TitleId in a filename
private static readonly Regex TitleIdRegex = new(
@"([0-9a-fA-F]{8})([0-9a-fA-F]{8})", RegexOptions.Compiled);
private readonly SemaphoreSlim _reloadLock = new(1, 1); // protects reload logic
private Task? _reloadTask; // the currently running or last finished reload
#endregion
#region ctor
@@ -107,19 +109,12 @@ public sealed class TitleDatabaseService : IHostedService
{
var ttlSec = _options.CurrentValue.TtlSeconds;
_logger.LogInformation("Reloading title database cache (TTL={TTL}s).", ttlSec);
var entryOptions = new MemoryCacheEntryOptions()
.SetSlidingExpiration(TimeSpan.FromSeconds(ttlSec)).RegisterPostEvictionCallback((key, value, reason,
state) =>
{
_logger.LogInformation("Cache eviction: {Key} ({Reason})", key, reason);
}); // <‑‑ sliding!
.SetSlidingExpiration(TimeSpan.FromSeconds(ttlSec))
.RegisterPostEvictionCallback((key, value, reason, state) => { _logger.LogInformation("Cache eviction: {Key} ({Reason})", key, reason); });
var dict = await LoadFromDiskAsync() ?? new Dictionary<string, TitleInfoDto>();
// Set the new entry – the sliding expiration will now
// automatically move 30 s (or whatever you configured) forward
// every time the entry is accessed via Get/Set.
_cache.Set(CacheKey, dict, entryOptions);
_logger.LogInformation("Title DB reloaded – {Count} items cached (TTL={TTL}s).",
dict.Count, ttlSec);
@@ -142,21 +137,20 @@ public sealed class TitleDatabaseService : IHostedService
/* ---------------------------------------------------------------- */
public async Task<Dictionary<string, TitleInfoDto>> GetAllAsync()
{
if (!_cache.TryGetValue(CacheKey, out Dictionary<string, TitleInfoDto>? dict))
{
await ReloadCacheAsync(); // cache miss → load from disk
_cache.TryGetValue(CacheKey, out dict);
}
return dict!; // Get() has already slid the entry
await EnsureCacheLoadedAsync().ConfigureAwait(false);
// The cache entry is guaranteed to exist now
_cache.TryGetValue(CacheKey, out Dictionary<string, TitleInfoDto>? dict);
return dict!;
}
public async Task<TitleInfoDto?> GetAsync(string titleId)
{
var all = await GetAllAsync(); // slides the entry
var all = await GetAllAsync(); // slides the entry
all.TryGetValue(titleId, out var dto);
return dto;
}
/* ---------------------------------------------------------------- */
/* 3️⃣ Persist to disk & notify snapshot service
/* ---------------------------------------------------------------- */
@@ -165,18 +159,23 @@ public sealed class TitleDatabaseService : IHostedService
// Trigger a rebuild so SnapshotService (and any other listeners)
// can pick up the new snapshot.
//_snapshotService.RebuildSnapshot();
await Task.CompletedTask;
}
/* ---------------------------------------------------------------- */
/* 4️⃣ Dispose
/* ---------------------------------------------------------------- */
public void Dispose() => _snapshotService.SnapshotRebuilt -= SnapshotServiceOnSnapshotRebuilt;
public void Dispose()
{
_snapshotService.SnapshotRebuilt -= SnapshotServiceOnSnapshotRebuilt;
_reloadLock.Dispose();
}
private async void SnapshotServiceOnSnapshotRebuilt(object? o, EventArgs eventArgs)
{
try
{
await ReloadCacheAsync();
await EnsureCacheLoadedAsync();
}
catch (Exception e)
{
@@ -197,7 +196,51 @@ public sealed class TitleDatabaseService : IHostedService
#endregion
#region Private helpers
/// <summary>
/// Makes sure the cache is loaded. If a reload is already in progress,
/// the caller simply awaits the same task. Otherwise it starts a new reload.
/// </summary>
private async Task EnsureCacheLoadedAsync()
{
// Fast path – a cache entry exists, no work needed
if (_cache.TryGetValue(CacheKey, out _))
return;
// Fast path – a reload is already underway
var existingTask = _reloadTask;
if (existingTask != null)
{
await existingTask.ConfigureAwait(false);
return;
}
// Slow path – we need to start the reload
await _reloadLock.WaitAsync().ConfigureAwait(false);
try
{
// Double‑check after acquiring the lock
if (_cache.TryGetValue(CacheKey, out _))
return;
// If another thread started the reload while we were waiting, use it
existingTask = _reloadTask;
if (existingTask != null)
{
await existingTask.ConfigureAwait(false);
return;
}
// Create the shared task
_reloadTask = ReloadCacheAsync();
await _reloadTask.ConfigureAwait(false);
}
finally
{
_reloadTask = null; // reset for the next miss
_reloadLock.Release();
}
}
/// <summary>
/// Downloads the JSON file from GitHub (raw) if it does not exist
/// or the cached copy is older than the remote one.