4 Commits

Author SHA1 Message Date
ecenshu f501fd8671 Fix build warnings
ci / build_linux (push) Successful in 11m22s
ci / build_linux (pull_request) Successful in 5m26s
Snapshot now persisted in lastmodified date descending, hopefully aligns with snapshot simple check against first entry in a directory not existing in the snapshot during build
Building Snapshot should only be done synchrnonously and atomically
Blacklist watched for changes
2025-11-16 11:42:14 +10:30
ecenshu 8751a72176 feature/entry_from_filename (#4)
ci / build_linux (push) Failing after 13m35s
Build & Push Docker image / build-and-push (push) Failing after 13m40s
Reviewed-on: #4
Co-authored-by: Huy Nguyen <ecenshu@gmail.com>
Co-committed-by: Huy Nguyen <ecenshu@gmail.com>
2025-11-15 08:55:27 +00:00
ecenshu c260ebd566 If filename can extract to a NcaMetadata entry, don't use nspextractor to pull information (#3)
Build & Push Docker image / build-and-push (push) Successful in 5m39s
ci / build_linux (push) Successful in 4m36s
Scan directories sequentially to reduce memory footprint

Reviewed-on: #3
Co-authored-by: Huy Nguyen <ecenshu@gmail.com>
Co-committed-by: Huy Nguyen <ecenshu@gmail.com>
2025-11-15 06:59:25 +00:00
ecenshu 17d12fef6a Use a resource to initialise appsettings.json in config folder (#2)
Build & Push Docker image / build-and-push (push) Successful in 8m18s
ci / build_linux (push) Successful in 3m36s
Watch for KeySet, initial install will not have a valid value
TitleDatabase will use data folder

Reviewed-on: #2
Co-authored-by: Huy Nguyen <ecenshu@gmail.com>
Co-committed-by: Huy Nguyen <ecenshu@gmail.com>
2025-11-13 23:52:34 +00:00
15 changed files with 394 additions and 268 deletions
+2 -1
View File
@@ -1,4 +1,5 @@
<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation"> <wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=IP/@EntryIndexedValue">IP</s:String> <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=IP/@EntryIndexedValue">IP</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=NSP/@EntryIndexedValue">NSP</s:String> <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=NSP/@EntryIndexedValue">NSP</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=PFS/@EntryIndexedValue">PFS</s:String></wpf:ResourceDictionary> <s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=PFS/@EntryIndexedValue">PFS</s:String>
<s:String x:Key="/Default/CodeStyle/Naming/CSharpNaming/Abbreviations/=ROM/@EntryIndexedValue">ROM</s:String></wpf:ResourceDictionary>
+3
View File
@@ -87,7 +87,10 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUniqueRef_00601_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F77078e9a1d254191bb508f54a277fc6e1c2e00_003Fa0_003F631946a0_003FUniqueRef_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> <s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUniqueRef_00601_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F77078e9a1d254191bb508f54a277fc6e1c2e00_003Fa0_003F631946a0_003FUniqueRef_00601_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUnpack_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003F4c375bcb6a2d2378855cad1b1c7cfe7ca1448866f1e8af44226775b5f75df86_003FUnpack_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> <s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUnpack_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003F4c375bcb6a2d2378855cad1b1c7cfe7ca1448866f1e8af44226775b5f75df86_003FUnpack_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUnsafeHelpers_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F77078e9a1d254191bb508f54a277fc6e1c2e00_003Fd1_003Fc59f91c2_003FUnsafeHelpers_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> <s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUnsafeHelpers_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F77078e9a1d254191bb508f54a277fc6e1c2e00_003Fd1_003Fc59f91c2_003FUnsafeHelpers_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUriExt_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E3_003Fresharper_002Dhost_003FSourcesCache_003F25edf3e734d0c76e87b29a9954b8b4a7383648a69396554742e5529205e2dd7_003FUriExt_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUriSyntax_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E3_003Fresharper_002Dhost_003FSourcesCache_003F7ed82aa0f48a6bf284b4aba7c70aff35142349e44fb4f6caec3d71611f9929_003FUriSyntax_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUtility_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ea3ed6216d2412ac7c33016ad940618bcfbcafe1633dc26832be514633b4_003FUtility_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> <s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUtility_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ea3ed6216d2412ac7c33016ad940618bcfbcafe1633dc26832be514633b4_003FUtility_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AWaitHandle_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E3_003Fresharper_002Dhost_003FSourcesCache_003Fcab54c7c1f8d9b3da5f7878e39faba52467f16d0899a2c4b10086cb2ef73f2b_003FWaitHandle_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AXciPartition_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F77078e9a1d254191bb508f54a277fc6e1c2e00_003F04_003F4e8815da_003FXciPartition_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> <s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AXciPartition_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F77078e9a1d254191bb508f54a277fc6e1c2e00_003F04_003F4e8815da_003FXciPartition_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AZipArchive_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003F39ececcf144e1f9c884152723ed93931cd232485eaf2824bf5beb526f1f321b_003FZipArchive_002Ecs/@EntryIndexedValue">ForceIncluded</s:String> <s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AZipArchive_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003F39ececcf144e1f9c884152723ed93931cd232485eaf2824bf5beb526f1f321b_003FZipArchive_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
+35 -13
View File
@@ -34,10 +34,11 @@ public class AuthStore : IDisposable, IAuthStore
public readonly ConcurrentDictionary<string, Credential> Credentials = new(); public readonly ConcurrentDictionary<string, Credential> Credentials = new();
public readonly ConcurrentDictionary<string, List<string>> Fingerprints = new(); public readonly ConcurrentDictionary<string, List<string>> Fingerprints = new();
public readonly ConcurrentDictionary<string, int> FailedAttempts = new(); public readonly ConcurrentDictionary<string, int> FailedAttempts = new();
private readonly HashSet<string> BlacklistIPs = new(); private readonly HashSet<string> _blacklistIPs = new();
private readonly object _sync = new(); private readonly Lock _sync = new();
private readonly FileSystemWatcher _credentialsWatcher; private readonly FileSystemWatcher _credentialsWatcher;
private readonly FileSystemWatcher _blacklistWatcher;
public AuthStore(ILogger<AuthStore> logger, ConfigManager configManager, IHostEnvironment env) public AuthStore(ILogger<AuthStore> logger, ConfigManager configManager, IHostEnvironment env)
{ {
@@ -56,6 +57,16 @@ public class AuthStore : IDisposable, IAuthStore
}; };
_credentialsWatcher.Changed += (_, _) => OnCredentialsChanged(); _credentialsWatcher.Changed += (_, _) => OnCredentialsChanged();
_credentialsWatcher.EnableRaisingEvents = true; _credentialsWatcher.EnableRaisingEvents = true;
_blacklistWatcher = new FileSystemWatcher
{
Path = (!string.IsNullOrEmpty(directoryName)) ? directoryName : AppContext.BaseDirectory,
Filter = Path.GetFileName(_configManager.Settings?.BlacklistFile) ?? throw new InvalidOperationException(),
NotifyFilter = NotifyFilters.LastWrite | NotifyFilters.Size | NotifyFilters.Attributes
};
_blacklistWatcher.Changed += (_, _) => OnBlacklistChanged();
_blacklistWatcher.EnableRaisingEvents = true;
_logger.LogInformation("Started watching credentials file {File}", credentialsFilePath);
_logger.LogInformation("Started watching blacklist file {File}", Path.Combine(env.ContentRootPath, "data", Path.GetFileName(_configManager.Settings?.BlacklistFile) ?? throw new InvalidOperationException()));
} }
private static string DetermineCredentialsPath(string? settingsCredentialsFile, IHostEnvironment env) private static string DetermineCredentialsPath(string? settingsCredentialsFile, IHostEnvironment env)
@@ -66,7 +77,7 @@ public class AuthStore : IDisposable, IAuthStore
public void Dispose() public void Dispose()
{ {
_credentialsWatcher?.Dispose(); _credentialsWatcher.Dispose();
} }
#region Loading helpers #region Loading helpers
@@ -89,7 +100,7 @@ public class AuthStore : IDisposable, IAuthStore
} }
// fingerprints // fingerprints
if (File.Exists(_configManager.Settings.FingerprintsFile)) if (File.Exists(_configManager.Settings?.FingerprintsFile))
{ {
var txt = File.ReadAllText(_configManager.Settings.FingerprintsFile); var txt = File.ReadAllText(_configManager.Settings.FingerprintsFile);
var dict = JsonSerializer.Deserialize<Dictionary<string, List<string>>>(txt)!; var dict = JsonSerializer.Deserialize<Dictionary<string, List<string>>>(txt)!;
@@ -98,24 +109,26 @@ public class AuthStore : IDisposable, IAuthStore
} }
else else
{ {
if (_configManager.Settings?.FingerprintsFile != null)
FileSystemExtensions.EnsureDirectoryExists(Path.GetDirectoryName(Path.GetFullPath(_configManager.Settings.FingerprintsFile))); FileSystemExtensions.EnsureDirectoryExists(Path.GetDirectoryName(Path.GetFullPath(_configManager.Settings.FingerprintsFile)));
} }
// blacklist // blacklist
if (File.Exists(_configManager.Settings.BlacklistFile)) if (File.Exists(_configManager.Settings?.BlacklistFile))
{ {
var txt = File.ReadAllText(_configManager.Settings.BlacklistFile); var txt = File.ReadAllText(_configManager.Settings.BlacklistFile);
var arr = JsonSerializer.Deserialize<string[]>(txt)!; var arr = JsonSerializer.Deserialize<string[]>(txt)!;
foreach (var ip in arr) foreach (var ip in arr)
BlacklistIPs.Add(ip); _blacklistIPs.Add(ip);
} }
else else
{ {
if (_configManager.Settings?.BlacklistFile != null)
FileSystemExtensions.EnsureDirectoryExists(Path.GetDirectoryName(Path.GetFullPath(_configManager.Settings.BlacklistFile))); FileSystemExtensions.EnsureDirectoryExists(Path.GetDirectoryName(Path.GetFullPath(_configManager.Settings.BlacklistFile)));
} }
_logger.LogInformation("Loaded {UserCount} users, {FpCount} fingerprints, {IpCount} IPs", _logger.LogInformation("Loaded {UserCount} users, {FpCount} fingerprints, {IpCount} IPs",
Credentials.Count, Fingerprints.Count, BlacklistIPs.Count); Credentials.Count, Fingerprints.Count, _blacklistIPs.Count);
} }
#endregion #endregion
@@ -132,6 +145,16 @@ public class AuthStore : IDisposable, IAuthStore
}); });
} }
private void OnBlacklistChanged()
{
// Small debounce the file may still be locked by the editor.
Task.Run(async () =>
{
await Task.Delay(200);
LoadAll();
});
}
private void ReloadCredentials() private void ReloadCredentials()
{ {
var credentialsFilePath = DetermineCredentialsPath(_configManager.Settings?.CredentialsFile, _env); var credentialsFilePath = DetermineCredentialsPath(_configManager.Settings?.CredentialsFile, _env);
@@ -143,7 +166,6 @@ public class AuthStore : IDisposable, IAuthStore
try try
{ {
var txt = File.ReadAllText(credentialsFilePath); var txt = File.ReadAllText(credentialsFilePath);
var newDict = JsonSerializer.Deserialize<Dictionary<string, Credential>>(txt)!; var newDict = JsonSerializer.Deserialize<Dictionary<string, Credential>>(txt)!;
@@ -264,7 +286,7 @@ public class AuthStore : IDisposable, IAuthStore
if (newCount < _configManager.Settings.MaxFailedAttempts + 1) return newCount; if (newCount < _configManager.Settings.MaxFailedAttempts + 1) return newCount;
BlacklistIPs.Add(ip); _blacklistIPs.Add(ip);
PersistBlacklist(); PersistBlacklist();
lock (_sync) lock (_sync)
{ {
@@ -319,7 +341,7 @@ public class AuthStore : IDisposable, IAuthStore
private void PersistBlacklist() private void PersistBlacklist()
{ {
var json = JsonSerializer.Serialize(BlacklistIPs.ToArray(), new JsonSerializerOptions { WriteIndented = true }); var json = JsonSerializer.Serialize(_blacklistIPs.ToArray(), new JsonSerializerOptions { WriteIndented = true });
if (_configManager.Settings == null) if (_configManager.Settings == null)
{ {
_logger.LogCritical("Blacklist file not set, cannot persist"); _logger.LogCritical("Blacklist file not set, cannot persist");
@@ -336,17 +358,17 @@ public class AuthStore : IDisposable, IAuthStore
public bool IsIPBlacklisted(string ipAddress) public bool IsIPBlacklisted(string ipAddress)
{ {
return BlacklistIPs.Contains(ipAddress); return _blacklistIPs.Contains(ipAddress);
} }
public bool UnbanIp(string ipAddress) public bool UnbanIp(string ipAddress)
{ {
return BlacklistIPs.Remove(ipAddress); return _blacklistIPs.Remove(ipAddress);
} }
public bool BlacklistActive() public bool BlacklistActive()
{ {
return BlacklistIPs.Count > 0; return _blacklistIPs.Count > 0;
} }
#endregion #endregion
@@ -1,33 +1,15 @@
using System; using Microsoft.AspNetCore.Mvc;
using System.IO;
using System.Linq;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Configuration;
using SharpCompress.Readers; using SharpCompress.Readers;
using TinfoilVibeServer.Models; using TinfoilVibeServer.Models;
using TinfoilVibeServer.Services; using TinfoilVibeServer.Services;
using TinfoilVibeServer.Utilities;
namespace TinfoilVibeServer.Controllers; namespace TinfoilVibeServer.Controllers;
[ApiController] [ApiController]
[Route("/")] [Route("/")]
public sealed class IndexController : ControllerBase public sealed class IndexController(ISnapshotService snapshotService, IndexBuilderService indexBuilderService) : ControllerBase
{ {
private readonly ISnapshotService _snapshotService;
private readonly TitleDatabaseService _titleDb;
private readonly IConfiguration _configuration;
private readonly IndexBuilderService _indexBuilderService;
public IndexController(ISnapshotService snapshotService,
TitleDatabaseService titleDb,
IConfiguration configuration, IndexBuilderService indexBuilderService)
{
_snapshotService = snapshotService;
_titleDb = titleDb;
_configuration = configuration;
_indexBuilderService = indexBuilderService;
}
// ------------------------------------------------------------ // ------------------------------------------------------------
// GET / // GET /
// ------------------------------------------------------------ // ------------------------------------------------------------
@@ -40,10 +22,10 @@ public sealed class IndexController : ControllerBase
{ {
if (HttpContext.Request.Headers.CacheControl == "no-cache") if (HttpContext.Request.Headers.CacheControl == "no-cache")
{ {
_indexBuilderService.InvalidateIndex(this, EventArgs.Empty); indexBuilderService.InvalidateIndex(this, EventArgs.Empty);
} }
var index = _indexBuilderService.Build(HttpContext); var index = indexBuilderService.Build(HttpContext);
return Ok(index); return Ok(index);
} }
@@ -75,7 +57,7 @@ public sealed class IndexController : ControllerBase
var titleId = match.Groups["id"].Value.ToUpperInvariant(); var titleId = match.Groups["id"].Value.ToUpperInvariant();
// ---- 2️⃣ Find the file that contains this TitleId ------------ // ---- 2️⃣ Find the file that contains this TitleId ------------
var entry = _snapshotService.GetSnapshot().Files var entry = snapshotService.GetSnapshot().Files
.FirstOrDefault(e => { return e.Titles.FirstOrDefault(hash => hash.TitleId == titleId)?.TitleId == titleId; }); .FirstOrDefault(e => { return e.Titles.FirstOrDefault(hash => hash.TitleId == titleId)?.TitleId == titleId; });
if (entry == null) if (entry == null)
@@ -84,7 +66,7 @@ public sealed class IndexController : ControllerBase
// ---- 3️⃣ If the file is a normal NSP → send it ---------------- // ---- 3️⃣ If the file is a normal NSP → send it ----------------
if (Path.GetExtension(entry.Path).Equals(".nsp", StringComparison.OrdinalIgnoreCase) if (Path.GetExtension(entry.Path).Equals(".nsp", StringComparison.OrdinalIgnoreCase)
&& !entry.Path.Contains(_snapshotService.GetArchivePathSeparator())) && !entry.Path.Contains(snapshotService.GetArchivePathSeparator()))
{ {
if (System.IO.File.Exists(entry.Path)) if (System.IO.File.Exists(entry.Path))
{ {
@@ -116,7 +98,7 @@ public sealed class IndexController : ControllerBase
if (IsInsideArchive(entry.Path)) if (IsInsideArchive(entry.Path))
{ {
// Example: file is inside an archive use ArchiveHandler // Example: file is inside an archive use ArchiveHandler
var innerFileName = entry.Path.Split(_snapshotService.GetArchivePathSeparator()).Last(); var innerFileName = entry.Path.Split(snapshotService.GetArchivePathSeparator()).Last();
var stream = StreamFromArchive(entry, titleId, out var streamContainer); var stream = StreamFromArchive(entry, titleId, out var streamContainer);
if (stream == null) if (stream == null)
@@ -149,7 +131,7 @@ public sealed class IndexController : ControllerBase
// (e.g. "Games/MyGame.nsp" is a regular file; "archive.7z/mygame.nsp" // (e.g. "Games/MyGame.nsp" is a regular file; "archive.7z/mygame.nsp"
// would be inside an archive). For simplicity we only check // would be inside an archive). For simplicity we only check
// for common archive extensions. // for common archive extensions.
var filePath = path.Split(_snapshotService.GetArchivePathSeparator()).First(); var filePath = path.Split(snapshotService.GetArchivePathSeparator()).First();
return filePath.EndsWith(".zip", StringComparison.OrdinalIgnoreCase) || return filePath.EndsWith(".zip", StringComparison.OrdinalIgnoreCase) ||
filePath.EndsWith(".7z", StringComparison.OrdinalIgnoreCase) || filePath.EndsWith(".7z", StringComparison.OrdinalIgnoreCase) ||
filePath.EndsWith(".rar", StringComparison.OrdinalIgnoreCase); filePath.EndsWith(".rar", StringComparison.OrdinalIgnoreCase);
@@ -163,9 +145,9 @@ public sealed class IndexController : ControllerBase
private Stream? StreamFromArchive(FileEntry fileEntry, string titleId, out IDisposable? streamContainer) private Stream? StreamFromArchive(FileEntry fileEntry, string titleId, out IDisposable? streamContainer)
{ {
// Example: file is inside an archive use ArchiveHandler // Example: file is inside an archive use ArchiveHandler
var archivePath = fileEntry.Path.Split(_snapshotService.GetArchivePathSeparator()).First(); var archivePath = fileEntry.Path.Split(snapshotService.GetArchivePathSeparator()).First();
_snapshotService.GetArchiveName(titleId); snapshotService.GetArchiveName(titleId);
var innerFileName = Path.GetFileName(fileEntry.Path.Split(_snapshotService.GetArchivePathSeparator()).Last()); var innerFileName = Path.GetFileName(fileEntry.Path.Split(snapshotService.GetArchivePathSeparator()).Last());
// Use SharpCompress to open the archive and find the entry. // Use SharpCompress to open the archive and find the entry.
// Only the 3 archive types we support are handled. // Only the 3 archive types we support are handled.
@@ -212,44 +194,4 @@ public sealed class IndexController : ControllerBase
streamContainer = null; streamContainer = null;
return null; return null;
} }
public class DependentStream : Stream
{
private readonly Stream _innerStream;
private readonly IDisposable? _parentContainer;
public DependentStream(Stream innerStream, IDisposable? parentContainer)
{
_innerStream = innerStream;
_parentContainer = parentContainer;
}
public override void Flush() => _innerStream.Flush();
public override int Read(byte[] buffer, int offset, int count) => _innerStream.Read(buffer, offset, count);
public override long Seek(long offset, SeekOrigin origin) => _innerStream.Seek(offset, origin);
public override void SetLength(long value) => _innerStream.SetLength(value);
public override void Write(byte[] buffer, int offset, int count) => _innerStream.Write(buffer, offset, count);
public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken)
{
return _innerStream.CopyToAsync(destination, bufferSize, cancellationToken);
}
public override bool CanRead => _innerStream.CanRead;
public override bool CanSeek => _innerStream.CanSeek;
public override bool CanWrite => _innerStream.CanWrite;
public override long Length => _innerStream.Length;
public override long Position { get => _innerStream.Position; set => _innerStream.Position = value; }
protected override void Dispose(bool disposing)
{
_parentContainer?.Dispose();
base.Dispose(disposing);
}
}
} }
@@ -4,24 +4,17 @@ using TinfoilVibeServer.Authentication;
namespace TinfoilVibeServer.Middleware; namespace TinfoilVibeServer.Middleware;
/// <summary> /// <summary>
/// Minimal BasicAuth middleware that also checks UID, failure counters and a blacklist. /// Minimal BasicAuth middleware that also checks UID, failure counters, and a blacklist.
/// </summary> /// </summary>
public sealed class BasicAuthMiddleware public sealed class BasicAuthMiddleware(RequestDelegate next)
{ {
private readonly RequestDelegate _next;
public BasicAuthMiddleware(RequestDelegate next)
{
_next = next;
}
public async Task InvokeAsync(HttpContext context, IAuthStore store, ILogger<BasicAuthMiddleware> logger) public async Task InvokeAsync(HttpContext context, IAuthStore store, ILogger<BasicAuthMiddleware> logger)
{ {
// ------------- 1) Bypass auth for every path except “/” ---------------- // ------------- 1) Bypass auth for every path except “/” ----------------
// PathString is a struct compare its value directly. // PathString is a struct compare its value directly.
if (!context.Request.Path.Equals("/", StringComparison.Ordinal)) if (!context.Request.Path.Equals("/", StringComparison.Ordinal))
{ {
await _next(context); await next(context);
return; return;
} }
@@ -96,7 +89,7 @@ public sealed class BasicAuthMiddleware
// Authentication succeeded attach username for downstream handlers if needed // Authentication succeeded attach username for downstream handlers if needed
context.Items["User"] = username; context.Items["User"] = username;
logger.LogInformation("User {User} authenticated successfully (UID={UID})", username, uid); logger.LogInformation("User {User} authenticated successfully (UID={UID})", username, uid);
await _next(context); await next(context);
} }
private static void Challenge(HttpContext ctx) private static void Challenge(HttpContext ctx)
+1 -1
View File
@@ -8,6 +8,6 @@ namespace TinfoilVibeServer.Models;
public sealed record FileEntry( public sealed record FileEntry(
string Path, // nsp or archive path string Path, // nsp or archive path
long Size, // size of nsp or full archive long Size, // size of nsp or full archive
string Hash, // SHA256 hex of first NCA of first NCP in NSP or archive string? Hash, // SHA256 hex of first NCA of first NCP in NSP or archive
List<NcaMetadataWithHash> Titles // Details of all NSP Roms in the Path List<NcaMetadataWithHash> Titles // Details of all NSP Roms in the Path
); );
@@ -0,0 +1,26 @@
using LibHac.Ncm;
namespace TinfoilVibeServer.Models;
/// <summary>
/// DTO returned by the extractor contains all data the snapshot needs.
/// </summary>
public sealed class NcaMetadataWithHash
{
public string TitleId { get; }
public string ApplicationTitle { get; set; }
public int Version { get; }
public ContentMetaType ContentMetaType { get; set; }
public string? Hash { get; }
public NcaMetadataWithHash(string titleId, string applicationTitle, int version,
ContentMetaType contentMetaType, string? hash = null)
{
TitleId = titleId;
ApplicationTitle = applicationTitle;
Version = version;
ContentMetaType = contentMetaType;
Hash = hash;
}
}
@@ -119,9 +119,11 @@ public sealed class IndexBuilderService: IHostedService
var fileName = Uri.EscapeDataString($"{name}[{titleId}][v{versionNumberParsed}][{patchOrApp}].nsp"); var fileName = Uri.EscapeDataString($"{name}[{titleId}][v{versionNumberParsed}][{patchOrApp}].nsp");
var url = $"{baseUri.ToString().TrimEnd('/')}/{fileName}"; var url = $"{baseUri.ToString().TrimEnd('/')}/{fileName}";
if (Uri.IsWellFormedUriString(url, UriKind.Absolute)) var isWellFormed = Uri.TryCreate(url, UriKind.Absolute, out var parsedUri);
if (isWellFormed && parsedUri != null)
{ {
fileDtos.Add(new FileDto(url, e.Size)); fileDtos.Add(new FileDto(parsedUri.AbsoluteUri, e.Size));
} }
else else
{ {
+2 -25
View File
@@ -1,5 +1,4 @@
using System.ComponentModel.DataAnnotations; using System.Security.Cryptography;
using System.Security.Cryptography;
using LibHac.Common; using LibHac.Common;
using LibHac.Fs; using LibHac.Fs;
using LibHac.Fs.Fsa; using LibHac.Fs.Fsa;
@@ -11,6 +10,7 @@ using LibHac.Ncm;
using LibHac.Tools.Fs; using LibHac.Tools.Fs;
using LibHac.Tools.Ncm; using LibHac.Tools.Ncm;
using Microsoft.Extensions.Options; using Microsoft.Extensions.Options;
using TinfoilVibeServer.Models;
using Path = System.IO.Path; using Path = System.IO.Path;
namespace TinfoilVibeServer.Services namespace TinfoilVibeServer.Services
@@ -322,27 +322,4 @@ namespace TinfoilVibeServer.Services
{ {
public string? KeyFile { get; set; } public string? KeyFile { get; set; }
} }
/// <summary>
/// DTO returned by the extractor contains all data the snapshot needs.
/// </summary>
public sealed class NcaMetadataWithHash
{
public string TitleId { get; }
public string ApplicationTitle { get; set; }
public int Version { get; }
public ContentMetaType ContentMetaType { get; set; }
public string Hash { get; }
public NcaMetadataWithHash(string titleId, string applicationTitle, int version,
ContentMetaType contentMetaType, string hash)
{
TitleId = titleId;
ApplicationTitle = applicationTitle;
Version = version;
ContentMetaType = contentMetaType;
Hash = hash;
}
}
} }
+183 -84
View File
@@ -27,9 +27,18 @@ public interface ISnapshotService
public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedService public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedService
{ {
#region FileSystemWatcher #region FileSystemWatcher
/* ==============================================================
* 1️⃣ FileSystemWatcher
* ============================================================== */
private readonly List<FileSystemWatcher> _watchers = new(); private readonly List<FileSystemWatcher> _watchers = new();
#endregion #endregion
#region Snapshot options & helpers
/* ==============================================================
* 2️⃣ Snapshot options & helpers
* ============================================================== */
private readonly SnapshotOptions _options; private readonly SnapshotOptions _options;
private readonly INSPExtractor _nspExtractor; private readonly INSPExtractor _nspExtractor;
private readonly IArchiveHandler _archiveHandler; private readonly IArchiveHandler _archiveHandler;
@@ -38,9 +47,12 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
private readonly string _jsonPath; private readonly string _jsonPath;
private readonly string _snapshotPath; private readonly string _snapshotPath;
private readonly ConcurrentDictionary<string, SnapshotEntry> _cache = new(); private readonly ConcurrentDictionary<string, SnapshotEntry> _cache = new();
private readonly ConcurrentDictionary<string, string> _hashCache = new(); private readonly ConcurrentDictionary<string, string> _hashCache = new();
// Archive full path -> FileEntry.Path // Archive full path -> FileEntry.Path
private readonly ConcurrentDictionary<string, string> _archiveLookup = new(); private readonly ConcurrentDictionary<string, string> _archiveLookup = new();
// hash -> file size // hash -> file size
private readonly ConcurrentDictionary<string, long> _sizeLookup = new(); private readonly ConcurrentDictionary<string, long> _sizeLookup = new();
private readonly IMemoryCache _debouncerCache; private readonly IMemoryCache _debouncerCache;
@@ -49,8 +61,21 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
private readonly SemaphoreSlim _snapshotFileSemaphore = new(1, 1); private readonly SemaphoreSlim _snapshotFileSemaphore = new(1, 1);
private const char ArchivePathSeparator = '|'; private const char ArchivePathSeparator = '|';
public char GetArchivePathSeparator() => ArchivePathSeparator;
public char GetArchivePathSeparator() => ArchivePathSeparator;
#endregion
/* ==============================================================
* 3️⃣ Buildtime guard
* ============================================================== */
/// <summary>
/// Allows only one rebuild at a time.
/// </summary>
private readonly SemaphoreSlim _buildLock = new(1, 1);
/* ==============================================================
* 4️⃣ Constructor
* ============================================================== */
public SnapshotService( public SnapshotService(
IMemoryCache debouncerCache, IMemoryCache debouncerCache,
IOptionsMonitor<SnapshotOptions> options, IOptionsMonitor<SnapshotOptions> options,
@@ -75,25 +100,25 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
File.WriteAllText(_jsonPath, "[]"); File.WriteAllText(_jsonPath, "[]");
_snapshotFileSemaphore.Release(); _snapshotFileSemaphore.Release();
} }
_snapshotPath = Path.Combine(Path.DirectorySeparatorChar.ToString(), "app", "data", _options.SnapshotBackupFile); _snapshotPath = Path.Combine(Path.DirectorySeparatorChar.ToString(), "app", "data", _options.SnapshotBackupFile);
FileSystemExtensions.EnsureDirectoryExists(Path.GetFullPath(Path.GetDirectoryName(_snapshotPath) ?? throw new InvalidOperationException())); FileSystemExtensions.EnsureDirectoryExists(Path.GetFullPath(Path.GetDirectoryName(_snapshotPath) ?? throw new InvalidOperationException()));
// 1️⃣ Register for *property* changes // 1️⃣ Register for *property* changes
options.OnChange(snapshotOptions => options.OnChange((snapshotOptions, _) => { _options.RootDirectories = snapshotOptions.RootDirectories; });
{ _options.PropertyChanged += (_, e) => OnOptionsChanged(e.PropertyName);
_options.RootDirectories = snapshotOptions.RootDirectories;
});
_options.PropertyChanged += (s, e) => OnOptionsChanged(e.PropertyName);
if (_options.RootDirectories.Count == 0) if (_options.RootDirectories.Count == 0)
{ {
_logger.LogInformation("No directories set to watch for ROMS/Archives"); _logger.LogInformation("No directories set to watch for ROMS/Archives");
} }
foreach (var path in _options.RootDirectories) foreach (var path in _options.RootDirectories)
{ {
AddWatchDirectory(path); AddWatchDirectory(path);
} }
} }
// --------- Private helpers --------- // --------- Private helpers ---------
private void OnOptionsChanged(string? propertyName) private void OnOptionsChanged(string? propertyName)
{ {
@@ -122,6 +147,10 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
#region FileSystemWatcher #region FileSystemWatcher
/* ==============================================================
* 5️⃣ FileSystemWatcher helpers
* ============================================================== */
private void AddWatchDirectory(string path) private void AddWatchDirectory(string path)
{ {
if (!Directory.Exists(path)) return; if (!Directory.Exists(path)) return;
@@ -156,6 +185,15 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
private void ThrottleSnapshotUpdate(FileSystemEventArgs fileSystemEventArgs) private void ThrottleSnapshotUpdate(FileSystemEventArgs fileSystemEventArgs)
{ {
// If a rebuild is in progress, ignore the event immediately
if (_buildLock.CurrentCount == 0) // lock held by a rebuild
{
_logger.LogInformation(
"File system event {ChangeType} on {Path} ignored because a rebuild is already in progress",
fileSystemEventArgs.ChangeType, fileSystemEventArgs.FullPath);
return;
}
SnapshotRebuilding?.Invoke(this, fileSystemEventArgs); SnapshotRebuilding?.Invoke(this, fileSystemEventArgs);
CancellationTokenSource cts = new(); CancellationTokenSource cts = new();
@@ -169,9 +207,9 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
new PostEvictionCallbackRegistration new PostEvictionCallbackRegistration
{ {
EvictionCallback = EvictionCallback =
(key, value, reason, state) => (_, value, reason, _) =>
{ {
if (!(reason == EvictionReason.Expired || reason == EvictionReason.TokenExpired)) return; if (reason is not (EvictionReason.Expired or EvictionReason.TokenExpired)) return;
if (value is FileSystemEventArgs args) if (value is FileSystemEventArgs args)
{ {
@@ -179,8 +217,9 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
{ {
_logger.LogInformation("File {FilePath} is locked, skipping snapshot update", args.FullPath); _logger.LogInformation("File {FilePath} is locked, skipping snapshot update", args.FullPath);
using var rebounce = _debouncerCache.CreateEntry(args.FullPath) using var rebounce = _debouncerCache.CreateEntry(args.FullPath)
.SetAbsoluteExpiration(TimeSpan.FromMilliseconds(DebounceMs)) .AddExpirationToken(new CancellationChangeToken(cts.Token))
.SetValue(args); .SetValue(args);
cts.CancelAfter(TimeSpan.FromMilliseconds(DebounceMs));
} }
} }
@@ -193,6 +232,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
_logger.LogDebug("File system event {EventType} on {Path} at {Time}", fileSystemEventArgs.ChangeType, _logger.LogDebug("File system event {EventType} on {Path} at {Time}", fileSystemEventArgs.ChangeType,
fileSystemEventArgs.FullPath, DateTime.Now.ToString("HH:mm:ss.fff")); fileSystemEventArgs.FullPath, DateTime.Now.ToString("HH:mm:ss.fff"));
} }
private static bool IsFileLocked(string filePath) private static bool IsFileLocked(string filePath)
{ {
FileStream? stream = null; FileStream? stream = null;
@@ -210,6 +250,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
{ {
stream?.Close(); stream?.Close();
} }
return false; return false;
} }
@@ -229,9 +270,19 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
public Task AddToSnapshotAsync(FileEntry entry) public Task AddToSnapshotAsync(FileEntry entry)
{ {
// Update lookup tables // Update lookup tables
_cache[entry.Path] = new SnapshotEntry(entry.Path, entry.Hash, entry.Size, entry.Titles); if (entry.Hash != null)
{
var lastModified = File.GetLastWriteTimeUtc(entry.Path);
_cache[entry.Path] = new SnapshotEntry(entry.Path, entry.Hash, entry.Size, lastModified, entry.Titles);
_hashCache[entry.Hash] = entry.Path; _hashCache[entry.Hash] = entry.Path;
_sizeLookup[entry.Hash] = entry.Size; _sizeLookup[entry.Hash] = entry.Size;
}
else
{
_logger.LogWarning("Cannot add entry {Path} to snapshot: no hash", entry.Path);
return Task.CompletedTask;
}
if (entry.Path.Contains(ArchivePathSeparator)) if (entry.Path.Contains(ArchivePathSeparator))
{ {
var filename = entry.Path.Split(ArchivePathSeparator)[0]; var filename = entry.Path.Split(ArchivePathSeparator)[0];
@@ -240,83 +291,88 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
foreach (var ncaMetadataWithHash in entry.Titles) foreach (var ncaMetadataWithHash in entry.Titles)
{ {
if (ncaMetadataWithHash.Hash == null)
{
_logger.LogWarning("Cannot add entry {Path} to snapshot: no hash", entry.Path);
continue;
}
_hashCache[ncaMetadataWithHash.Hash] = entry.Path; _hashCache[ncaMetadataWithHash.Hash] = entry.Path;
_sizeLookup[ncaMetadataWithHash.Hash] = entry.Size; _sizeLookup[ncaMetadataWithHash.Hash] = entry.Size;
_logger.LogInformation("Added entry {titleId} to snapshot (hash={hash})", ncaMetadataWithHash.TitleId, ncaMetadataWithHash.Hash); _logger.LogInformation("Added entry {titleId} to snapshot (hash={hash})", ncaMetadataWithHash.TitleId, ncaMetadataWithHash.Hash);
} }
// Persist snapshot to disk // Persist snapshot to disk
PersistSnapshotAsync(); PersistSnapshotAsync();
return Task.CompletedTask; return Task.CompletedTask;
} }
/* ==============================================================
* 6️⃣ Snapshot build / persistence helpers
* ============================================================== */
/// Builds _cache and _hashCache based on directory configuration /// Builds _cache and _hashCache based on directory configuration
public Task BuildSnapshotAsync() public Task BuildSnapshotAsync()
{
// Acquire the rebuild lock if we cannot, skip this build.
if (!_buildLock.Wait(0))
{
_logger.LogInformation("BuildSnapshotAsync called while rebuild in progress, ignoring.");
return Task.CompletedTask;
}
try
{ {
_logger.LogInformation("Building snapshot"); _logger.LogInformation("Building snapshot");
var index = LoadSnapshotIndex(); var index = LoadSnapshotIndex();
var latestModifiedUtcParallel = FileSystemExtensions.GetLatestModifiedUtcParallel(_options.RootDirectories); var latestModifiedUtcParallel = FileSystemExtensions.GetLatestModifiedUtcParallel(_options.RootDirectories);
var fileInfo = new FileInfo(_snapshotPath); var fileInfo = new FileInfo(_snapshotPath);
bool snapshotVerified = true; bool snapshotVerified = fileInfo.Exists;
if (latestModifiedUtcParallel.HasValue && latestModifiedUtcParallel.Value < fileInfo.LastWriteTimeUtc) if (latestModifiedUtcParallel.HasValue && latestModifiedUtcParallel.Value < fileInfo.LastWriteTimeUtc)
{ {
if (index.Count != 0) if (index.Count != 0)
{ {
// directory may have been added with older roms, verify that the snapshot is still up to date
foreach (var dir in _options.RootDirectories) foreach (var dir in _options.RootDirectories)
{ {
// check first entry is in index // Snapshot is older than the latest modified file in the directory
var entry = BuildSnapshot(dir).FirstOrDefault(); var lastOrDefault = BuildSnapshot(dir).LastOrDefault();
if (entry != null) if (lastOrDefault != null && !index.TryGetValue(lastOrDefault.Path, out _))
{
if (!index.TryGetValue(entry.Path, out var cached))
{ {
snapshotVerified = false; snapshotVerified = false;
_logger.LogInformation("Snapshot does not contain first entry in directory {Directory}", dir); _logger.LogInformation("Snapshot does not contain first entry in directory {Directory}", dir);
} }
} }
} }
}
if (snapshotVerified) if (!snapshotVerified)
{ {
_logger.LogInformation("Snapshot is up to date");
return Task.CompletedTask;
}
}
else
{
_logger.LogInformation("Snapshot is up to date but index is empty");
}
}
_logger.LogInformation("Rebuilding snapshot (root dirs: {Count})", _options.RootDirectories.Count); _logger.LogInformation("Rebuilding snapshot (root dirs: {Count})", _options.RootDirectories.Count);
var entries = new List<FileEntry>(); var entries = new List<FileEntry>();
var snapshotChanged = false;
foreach (var dir in _options.RootDirectories) foreach (var dir in _options.RootDirectories)
{ {
_ = Task.Run(() => foreach (var entry in BuildSnapshot(dir))
{ {
_logger.LogInformation("Rebuilding directory {Directory}", dir); if (entry != null) entries.Add(entry);
var buildSnapshot = BuildSnapshot(dir); }
var fileEntries = buildSnapshot.ToList();
snapshotChanged = snapshotChanged || fileEntries.Count != 0;
entries.AddRange(fileEntries.Where(entry => entry != null)!);
});
} }
var snapshotEmptied = fileInfo.Exists && index.Count == 0 && _options.RootDirectories.Count == 0; var currentHash = ComputeSnapshotHash(entries);
// Replace the entire snapshot if (entries.Count > 0 || fileInfo.Exists && index.Count == 0)
var currentSnapshotHash = ComputeSnapshotHash(entries);
if (snapshotChanged || snapshotEmptied)
{
_logger.LogInformation("Snapshot rebuilt");
SnapshotRebuilt?.Invoke(this, EventArgs.Empty); SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
} }
PersistSnapshotAsync();
}
finally
{
_buildLock.Release();
}
return Task.CompletedTask; return Task.CompletedTask;
} }
public void GetArchiveName(string titleId) public void GetArchiveName(string titleId)
{ {
;
} }
// Returns List of FileEntry that do not have a hash in the cache // Returns List of FileEntry that do not have a hash in the cache
@@ -324,7 +380,11 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
private IEnumerable<FileEntry?> BuildSnapshot(string dir) private IEnumerable<FileEntry?> BuildSnapshot(string dir)
{ {
if (!Directory.Exists(dir)) yield break; if (!Directory.Exists(dir)) yield break;
foreach (var file in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories)) foreach (var file in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories).OrderBy(file =>
{
var fileInfo = new FileInfo(file);
return fileInfo.LastWriteTimeUtc;
}))
{ {
var hash = string.Empty; var hash = string.Empty;
var ext = Path.GetExtension(file).ToLowerInvariant(); var ext = Path.GetExtension(file).ToLowerInvariant();
@@ -341,6 +401,16 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
var titles = new List<(string, long, NcaMetadataWithHash)>(); var titles = new List<(string, long, NcaMetadataWithHash)>();
if (_options.RomExtensions.Contains(ext)) if (_options.RomExtensions.Contains(ext))
{ {
var fileInfo = new FileInfo(file);
var ncaMetadataWithHash = fileInfo.GetNcaMetadataWithHash();
if (ncaMetadataWithHash != null)
{
//var titleInfo = _titleDatabaseService.GetAsync(ncaMetadataWithHash.TitleId).Result;
var fileEntryFromFileName = new FileEntry(file, fileInfo.Length, ncaMetadataWithHash.Hash, [ncaMetadataWithHash]);
AddToSnapshotAsync(fileEntryFromFileName);
yield return fileEntryFromFileName;
}
using var nspStream = File.OpenRead(file); using var nspStream = File.OpenRead(file);
hash = ComputeFirstStreamHash(nspStream); hash = ComputeFirstStreamHash(nspStream);
@@ -406,7 +476,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
else else
{ {
_logger.LogInformation("Added {File} to snapshot (hash={Hash})", file, hash); _logger.LogInformation("Added {File} to snapshot (hash={Hash})", file, hash);
yield return new FileEntry(file, titles.Select((tuple, i) => tuple.Item2).FirstOrDefault(), hash, titles.Select((tuple, i) => tuple.Item3).ToList()); yield return new FileEntry(file, titles.Select((tuple, _) => tuple.Item2).FirstOrDefault(), hash, titles.Select((tuple, _) => tuple.Item3).ToList());
} }
} }
} }
@@ -416,48 +486,48 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
await Task.CompletedTask; await Task.CompletedTask;
} }
private string ComputeFirstStreamHash(Stream nspStream) private string ComputeFirstStreamHash(Stream nspStream) => _nspExtractor.ExtractHashFromStream(nspStream);
{
return _nspExtractor.ExtractHashFromStream(nspStream);
}
private void UpdateSnapshot() => BuildSnapshotAsync(); private void UpdateSnapshot() => BuildSnapshotAsync();
IEnumerable<FileEntry> GetEntries() private IEnumerable<FileEntry> GetEntries()
{ {
foreach (var snapshotEntry in _cache) foreach (var kv in _cache.OrderByDescending(pair => pair.Value.LastModified))
{ yield return new FileEntry(kv.Key, kv.Value.Size, kv.Value.Hash, kv.Value.NcaMetadataWithHash);
_sizeLookup.TryGetValue(snapshotEntry.Value.Hash, out var size);
var fileEntry = new FileEntry(snapshotEntry.Key, snapshotEntry.Value.Size, snapshotEntry.Value.Hash, snapshotEntry.Value.NcaMetadataWithHash);
yield return fileEntry;
}
} }
private Task PersistSnapshotAsync() private Task PersistSnapshotAsync()
{ {
if (_debouncerCache.TryGetValue(_jsonPath, out var value)) if (_debouncerCache.TryGetValue(_jsonPath, out _))
{ {
_logger.LogInformation("Sliding debounce in progress, skipping snapshot persistence"); _logger.LogInformation("Sliding debounce in progress, skipping snapshot persistence");
return Task.CompletedTask; return Task.CompletedTask;
} }
var entries = GetEntries().ToList();
var newHash = ComputeSnapshotHash(entries);
var snapshot = GetSnapshot(); var snapshot = GetSnapshot();
var entries = GetEntries();
var fileEntries = entries.ToList();
var newHash = ComputeSnapshotHash(fileEntries);
if (snapshot.Hash == newHash) return Task.CompletedTask; if (snapshot.Hash == newHash) return Task.CompletedTask;
CancellationTokenSource cts = new(); var cancellationTokenSource = new CancellationTokenSource();
_logger.LogInformation("Snapshot hash changed persisting new snapshot"); using var cacheEntry = _debouncerCache.CreateEntry(_jsonPath)
using var debouncedPersistence = _debouncerCache.CreateEntry(_jsonPath); .AddExpirationToken(new CancellationChangeToken(cancellationTokenSource.Token))
debouncedPersistence.AddExpirationToken(new CancellationChangeToken(cts.Token)); .SetValue(entries)
//debouncedPersistence.AbsoluteExpirationRelativeToNow = TimeSpan.FromMilliseconds(DebounceMs); .SetOptions(new MemoryCacheEntryOptions
debouncedPersistence.Value = fileEntries;
debouncedPersistence.PostEvictionCallbacks.Add(new PostEvictionCallbackRegistration
{ {
EvictionCallback = (key, entriesCallback, reason, state) => PostEvictionCallbacks =
{ {
if (entriesCallback is IEnumerable<FileEntry> entriesToPersist && key is string filePath) new PostEvictionCallbackRegistration
{ {
EvictionCallback = (key, value, reason, _) =>
{
if (reason is not (EvictionReason.Expired or EvictionReason.TokenExpired))
return;
var filePath = (string)key;
if (_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout)) if (_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout))
{
try
{ {
if (IsFileLocked(filePath)) if (IsFileLocked(filePath))
{ {
@@ -465,25 +535,24 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
} }
else else
{ {
File.WriteAllText(filePath, File.WriteAllText(filePath, JsonSerializer.Serialize(value, _jsonSerializerOptions));
JsonSerializer.Serialize(entriesToPersist, _jsonSerializerOptions)); _logger.LogInformation("Persisted snapshot to {FilePath}", filePath);
_snapshotFileSemaphore.Release();
_logger.LogInformation("Persisted snapshot");
SnapshotRebuilt?.Invoke(this, EventArgs.Empty); SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
} }
} }
else finally
{ {
_logger.LogInformation("Failed to persist file {FilePath} due to timeout", filePath); _snapshotFileSemaphore.Release();
}
}
} }
} }
} }
}); });
cts.CancelAfter(TimeSpan.FromMilliseconds(DebounceMs)); cancellationTokenSource.CancelAfter(TimeSpan.FromMilliseconds(DebounceMs));
return Task.CompletedTask; return Task.CompletedTask;
} }
private static string ComputeHash(string filePath) private static string ComputeHash(string filePath)
{ {
using var sha = SHA256.Create(); using var sha = SHA256.Create();
@@ -499,6 +568,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
var hash = sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(json)); var hash = sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(json));
return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant(); return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
} }
/// <summary> /// <summary>
/// From filesystem cache, load each entry and build the lookups /// From filesystem cache, load each entry and build the lookups
/// Check for duplicate hashes /// Check for duplicate hashes
@@ -518,12 +588,19 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
// Reindex the cache // Reindex the cache
foreach (var fileEntry in entries) foreach (var fileEntry in entries)
{ {
if (fileEntry.Hash == null)
{
_logger.LogError("Entry {Path} has no hash", fileEntry.Path);
continue;
}
if (_hashCache.TryGetValue(fileEntry.Hash, out var value)) if (_hashCache.TryGetValue(fileEntry.Hash, out var value))
{ {
_logger.LogWarning("Duplicate hash found in snapshot: {Hash}, {OldPath}, {newPath}", fileEntry.Hash, value, fileEntry.Path); _logger.LogWarning("Duplicate hash found in snapshot: {Hash}, {OldPath}, {newPath}", fileEntry.Hash, value, fileEntry.Path);
} }
if (!File.Exists(fileEntry.Path)) var nspOrArchivePath = fileEntry.Path.Split(ArchivePathSeparator)[0];
if (!File.Exists(nspOrArchivePath))
{ {
_logger.LogWarning("Nonexistent entry found: {Path}", fileEntry.Path); _logger.LogWarning("Nonexistent entry found: {Path}", fileEntry.Path);
continue; continue;
@@ -543,30 +620,35 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
{ {
_logger.LogInformation("Entry {Path} is not contained in any root directory", fileEntry.Path); _logger.LogInformation("Entry {Path} is not contained in any root directory", fileEntry.Path);
continue; continue;
}; }
if (_options.RomExtensions.Contains(Path.GetExtension(fileEntry.Path))) if (_options.RomExtensions.Contains(Path.GetExtension(fileEntry.Path)))
{ {
var fileInfo = new FileInfo(fileEntry.Path);
if (fileEntry.Path.Contains(ArchivePathSeparator)) if (fileEntry.Path.Contains(ArchivePathSeparator))
{ {
var filename = fileEntry.Path.Split(ArchivePathSeparator)[0]; var filename = fileEntry.Path.Split(ArchivePathSeparator)[0];
_cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileEntry.Titles!); // ReSharper disable once RedundantSuppressNullableWarningExpression
_cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileInfo.LastWriteTimeUtc, fileEntry.Titles!);
_archiveLookup[filename] = fileEntry.Path; _archiveLookup[filename] = fileEntry.Path;
} }
else else
{ {
_cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileEntry.Titles!); // ReSharper disable once RedundantSuppressNullableWarningExpression
_cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileInfo.LastWriteTimeUtc, fileEntry.Titles!);
fileEntries.TryAdd(fileEntry.Path, fileEntry); fileEntries.TryAdd(fileEntry.Path, fileEntry);
_hashCache[fileEntry.Hash] = fileEntry.Path; _hashCache[fileEntry.Hash] = fileEntry.Path;
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
if (fileEntry.Titles == null) continue; if (fileEntry.Titles == null) continue;
foreach (var ncaMetadataWithHash in fileEntry.Titles) foreach (var ncaMetadataWithHash in fileEntry.Titles)
{ {
if (ncaMetadataWithHash.Hash == null) continue;
_hashCache[ncaMetadataWithHash.Hash] = fileEntry.Path; _hashCache[ncaMetadataWithHash.Hash] = fileEntry.Path;
} }
} }
} }
} }
_logger.LogInformation("Loaded snapshot index {Count} entries", fileEntries.Count); _logger.LogInformation("Loaded snapshot index {Count} entries", fileEntries.Count);
return fileEntries; return fileEntries;
} }
@@ -578,6 +660,14 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
} }
public void RebuildSnapshot() public void RebuildSnapshot()
{
// Fast path: if we already have the lock, just log and exit.
if (!_buildLock.Wait(0))
{
_logger.LogInformation("RebuildSnapshot called while a rebuild is already in progress, ignoring.");
return;
}
try
{ {
// 1️⃣ Flush the old inmemory snapshot // 1️⃣ Flush the old inmemory snapshot
_cache.Clear(); _cache.Clear();
@@ -591,6 +681,12 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
PersistSnapshotAsync().Wait(); // same PersistSnapshotAsync().Wait(); // same
SnapshotRebuilt?.Invoke(this, EventArgs.Empty); SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
} }
finally
{
_buildLock.Release();
}
}
#endregion #endregion
public ROMSnapshot GetSnapshot() public ROMSnapshot GetSnapshot()
@@ -635,7 +731,10 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
} }
} }
private sealed record SnapshotEntry(string Path, string Hash, long Size, List<NcaMetadataWithHash> NcaMetadataWithHash); /// <summary>
/// Represents a single ROM/archive entry in the snapshot cache.
/// </summary>
private sealed record SnapshotEntry(string Path, string Hash, long Size, DateTime LastModified, List<NcaMetadataWithHash> NcaMetadataWithHash);
// File: TinfoilVibeServer/Services/SnapshotService.cs (inside SnapshotService class) // File: TinfoilVibeServer/Services/SnapshotService.cs (inside SnapshotService class)
@@ -685,8 +784,8 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
public class ROMSnapshot public class ROMSnapshot
{ {
public string? Hash { get; set; } public string? Hash { get; init; }
public IReadOnlyList<FileEntry> Files { get; set; } = new List<FileEntry>(); public IReadOnlyList<FileEntry> Files { get; init; } = new List<FileEntry>();
} }
public async Task StartAsync(CancellationToken cancellationToken) public async Task StartAsync(CancellationToken cancellationToken)
@@ -0,0 +1,31 @@
namespace TinfoilVibeServer.Utilities;
public class DependentStream(Stream innerStream, IDisposable? parentContainer) : Stream
{
public override void Flush() => innerStream.Flush();
public override int Read(byte[] buffer, int offset, int count) => innerStream.Read(buffer, offset, count);
public override long Seek(long offset, SeekOrigin origin) => innerStream.Seek(offset, origin);
public override void SetLength(long value) => innerStream.SetLength(value);
public override void Write(byte[] buffer, int offset, int count) => innerStream.Write(buffer, offset, count);
public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken)
{
return innerStream.CopyToAsync(destination, bufferSize, cancellationToken);
}
public override bool CanRead => innerStream.CanRead;
public override bool CanSeek => innerStream.CanSeek;
public override bool CanWrite => innerStream.CanWrite;
public override long Length => innerStream.Length;
public override long Position { get => innerStream.Position; set => innerStream.Position = value; }
protected override void Dispose(bool disposing)
{
parentContainer?.Dispose();
base.Dispose(disposing);
}
}
@@ -112,8 +112,7 @@ public static class FileSystemExtensions
/// <exception cref="IOException">Thrown if a file exists at the target path or the directory cannot be created.</exception> /// <exception cref="IOException">Thrown if a file exists at the target path or the directory cannot be created.</exception>
public static void EnsureDirectoryExists(string? path) public static void EnsureDirectoryExists(string? path)
{ {
if (path is null) ArgumentNullException.ThrowIfNull(path);
throw new ArgumentNullException(nameof(path));
if (string.IsNullOrWhiteSpace(path)) if (string.IsNullOrWhiteSpace(path))
throw new ArgumentException("Path must not be empty or whitespace.", nameof(path)); throw new ArgumentException("Path must not be empty or whitespace.", nameof(path));
@@ -1,7 +1,7 @@
using System.Globalization; using System.Globalization;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
namespace TinfoilVibeServer.Models; namespace TinfoilVibeServer.Utilities;
public static class IdHelper public static class IdHelper
@@ -0,0 +1,30 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using LibHac.Ncm;
using TinfoilVibeServer.Models;
namespace TinfoilVibeServer.Utilities;
public static class NcaMetadataWithHashHelper
{
public static NcaMetadataWithHash? GetNcaMetadataWithHash(this FileInfo fileInfo)
{
var match = Regex.Match(fileInfo.Name, @"^(.+)\[(\w{16})\]\[v(\d{1,7})\]\[(\w+).*\]\.nsp$");
if (!match.Success) return null;
var titleId = match.Groups[2].Value;
var applicationTitle = match.Groups[1].Value.Trim();
var version = int.Parse(match.Groups[3].Value) / 0x10000;
var nspType = match.Groups[4].Value.ToLowerInvariant() switch
{
"base" => ContentMetaType.Application,
"update" => ContentMetaType.Patch,
"dlc" => ContentMetaType.AddOnContent,
_ => ContentMetaType.Patch
};
var bytes = Encoding.UTF8.GetBytes(fileInfo.FullName);
var hash = SHA256.HashData(bytes);
return new NcaMetadataWithHash(titleId, applicationTitle, version, nspType, Convert.ToBase64String(hash));
}
}
+1
View File
@@ -8,6 +8,7 @@ services:
image: gitea.ecenshu.net/ecenshu/tinfoilvibeserver:latest image: gitea.ecenshu.net/ecenshu/tinfoilvibeserver:latest
container_name: tinfoilvibeserver container_name: tinfoilvibeserver
restart: unless-stopped restart: unless-stopped
user: "1000:1000"
env_file: env_file:
- .env - .env
environment: environment: