From cb60d768dfd84aece22f2d87d1b4830f0e12032d Mon Sep 17 00:00:00 2001 From: Huy Nguyen Date: Sun, 16 Nov 2025 01:27:43 +0000 Subject: [PATCH] Synchronous Snapshot Build and ordered persistence (#5) Fix build warnings Snapshot now persisted in lastmodified date descending, hopefully aligns with snapshot simple check against first entry in a directory not existing in the snapshot during build Building Snapshot should only be done synchrnonously and atomically Blacklist watched for changes Reviewed-on: https://gitea.ecenshu.net/ecenshu/TinfoilVibeServer/pulls/5 Co-authored-by: Huy Nguyen Co-committed-by: Huy Nguyen --- TinfoilVibeServer.sln.DotSettings | 3 +- TinfoilVibeServer/Authentication/AuthStore.cs | 52 +++- .../Controllers/IndexController.cs | 82 +----- .../Middleware/BasicAuthMiddleware.cs | 15 +- .../Services/IndexBuilderService.cs | 4 +- TinfoilVibeServer/Services/SnapshotService.cs | 267 +++++++++++++----- .../Utilities/DependentStream.cs | 31 ++ .../Utilities/FileSystemExtensions.cs | 3 +- 8 files changed, 279 insertions(+), 178 deletions(-) create mode 100644 TinfoilVibeServer/Utilities/DependentStream.cs diff --git a/TinfoilVibeServer.sln.DotSettings b/TinfoilVibeServer.sln.DotSettings index ba97206..47c14a4 100644 --- a/TinfoilVibeServer.sln.DotSettings +++ b/TinfoilVibeServer.sln.DotSettings @@ -1,4 +1,5 @@  IP NSP - PFS \ No newline at end of file + PFS + ROM \ No newline at end of file diff --git a/TinfoilVibeServer/Authentication/AuthStore.cs b/TinfoilVibeServer/Authentication/AuthStore.cs index 9adbce7..a2f65e6 100644 --- a/TinfoilVibeServer/Authentication/AuthStore.cs +++ b/TinfoilVibeServer/Authentication/AuthStore.cs @@ -34,10 +34,11 @@ public class AuthStore : IDisposable, IAuthStore public readonly ConcurrentDictionary Credentials = new(); public readonly ConcurrentDictionary> Fingerprints = new(); public readonly ConcurrentDictionary FailedAttempts = new(); - private readonly HashSet BlacklistIPs = new(); + private readonly HashSet _blacklistIPs = new(); - private readonly object _sync = new(); + private readonly Lock _sync = new(); private readonly FileSystemWatcher _credentialsWatcher; + private readonly FileSystemWatcher _blacklistWatcher; public AuthStore(ILogger logger, ConfigManager configManager, IHostEnvironment env) { @@ -56,6 +57,16 @@ public class AuthStore : IDisposable, IAuthStore }; _credentialsWatcher.Changed += (_, _) => OnCredentialsChanged(); _credentialsWatcher.EnableRaisingEvents = true; + _blacklistWatcher = new FileSystemWatcher + { + Path = (!string.IsNullOrEmpty(directoryName)) ? directoryName : AppContext.BaseDirectory, + Filter = Path.GetFileName(_configManager.Settings?.BlacklistFile) ?? throw new InvalidOperationException(), + NotifyFilter = NotifyFilters.LastWrite | NotifyFilters.Size | NotifyFilters.Attributes + }; + _blacklistWatcher.Changed += (_, _) => OnBlacklistChanged(); + _blacklistWatcher.EnableRaisingEvents = true; + _logger.LogInformation("Started watching credentials file {File}", credentialsFilePath); + _logger.LogInformation("Started watching blacklist file {File}", Path.Combine(env.ContentRootPath, "data", Path.GetFileName(_configManager.Settings?.BlacklistFile) ?? throw new InvalidOperationException())); } private static string DetermineCredentialsPath(string? settingsCredentialsFile, IHostEnvironment env) @@ -66,7 +77,7 @@ public class AuthStore : IDisposable, IAuthStore public void Dispose() { - _credentialsWatcher?.Dispose(); + _credentialsWatcher.Dispose(); } #region Loading helpers @@ -89,7 +100,7 @@ public class AuthStore : IDisposable, IAuthStore } // fingerprints - if (File.Exists(_configManager.Settings.FingerprintsFile)) + if (File.Exists(_configManager.Settings?.FingerprintsFile)) { var txt = File.ReadAllText(_configManager.Settings.FingerprintsFile); var dict = JsonSerializer.Deserialize>>(txt)!; @@ -98,24 +109,26 @@ public class AuthStore : IDisposable, IAuthStore } else { - FileSystemExtensions.EnsureDirectoryExists(Path.GetDirectoryName(Path.GetFullPath(_configManager.Settings.FingerprintsFile))); + if (_configManager.Settings?.FingerprintsFile != null) + FileSystemExtensions.EnsureDirectoryExists(Path.GetDirectoryName(Path.GetFullPath(_configManager.Settings.FingerprintsFile))); } // blacklist - if (File.Exists(_configManager.Settings.BlacklistFile)) + if (File.Exists(_configManager.Settings?.BlacklistFile)) { var txt = File.ReadAllText(_configManager.Settings.BlacklistFile); var arr = JsonSerializer.Deserialize(txt)!; foreach (var ip in arr) - BlacklistIPs.Add(ip); + _blacklistIPs.Add(ip); } else { - FileSystemExtensions.EnsureDirectoryExists(Path.GetDirectoryName(Path.GetFullPath(_configManager.Settings.BlacklistFile))); + if (_configManager.Settings?.BlacklistFile != null) + FileSystemExtensions.EnsureDirectoryExists(Path.GetDirectoryName(Path.GetFullPath(_configManager.Settings.BlacklistFile))); } _logger.LogInformation("Loaded {UserCount} users, {FpCount} fingerprints, {IpCount} IPs", - Credentials.Count, Fingerprints.Count, BlacklistIPs.Count); + Credentials.Count, Fingerprints.Count, _blacklistIPs.Count); } #endregion @@ -131,6 +144,16 @@ public class AuthStore : IDisposable, IAuthStore ReloadCredentials(); }); } + + private void OnBlacklistChanged() + { + // Small debounce – the file may still be locked by the editor. + Task.Run(async () => + { + await Task.Delay(200); + LoadAll(); + }); + } private void ReloadCredentials() { @@ -143,7 +166,6 @@ public class AuthStore : IDisposable, IAuthStore try { - var txt = File.ReadAllText(credentialsFilePath); var newDict = JsonSerializer.Deserialize>(txt)!; @@ -264,7 +286,7 @@ public class AuthStore : IDisposable, IAuthStore if (newCount < _configManager.Settings.MaxFailedAttempts + 1) return newCount; - BlacklistIPs.Add(ip); + _blacklistIPs.Add(ip); PersistBlacklist(); lock (_sync) { @@ -319,7 +341,7 @@ public class AuthStore : IDisposable, IAuthStore private void PersistBlacklist() { - var json = JsonSerializer.Serialize(BlacklistIPs.ToArray(), new JsonSerializerOptions { WriteIndented = true }); + var json = JsonSerializer.Serialize(_blacklistIPs.ToArray(), new JsonSerializerOptions { WriteIndented = true }); if (_configManager.Settings == null) { _logger.LogCritical("Blacklist file not set, cannot persist"); @@ -336,17 +358,17 @@ public class AuthStore : IDisposable, IAuthStore public bool IsIPBlacklisted(string ipAddress) { - return BlacklistIPs.Contains(ipAddress); + return _blacklistIPs.Contains(ipAddress); } public bool UnbanIp(string ipAddress) { - return BlacklistIPs.Remove(ipAddress); + return _blacklistIPs.Remove(ipAddress); } public bool BlacklistActive() { - return BlacklistIPs.Count > 0; + return _blacklistIPs.Count > 0; } #endregion diff --git a/TinfoilVibeServer/Controllers/IndexController.cs b/TinfoilVibeServer/Controllers/IndexController.cs index 0b2f33b..8fc9dbc 100644 --- a/TinfoilVibeServer/Controllers/IndexController.cs +++ b/TinfoilVibeServer/Controllers/IndexController.cs @@ -1,33 +1,15 @@ -using System; -using System.IO; -using System.Linq; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Configuration; +using Microsoft.AspNetCore.Mvc; using SharpCompress.Readers; using TinfoilVibeServer.Models; using TinfoilVibeServer.Services; +using TinfoilVibeServer.Utilities; namespace TinfoilVibeServer.Controllers; [ApiController] [Route("/")] -public sealed class IndexController : ControllerBase +public sealed class IndexController(ISnapshotService snapshotService, IndexBuilderService indexBuilderService) : ControllerBase { - private readonly ISnapshotService _snapshotService; - private readonly TitleDatabaseService _titleDb; - private readonly IConfiguration _configuration; - private readonly IndexBuilderService _indexBuilderService; - - public IndexController(ISnapshotService snapshotService, - TitleDatabaseService titleDb, - IConfiguration configuration, IndexBuilderService indexBuilderService) - { - _snapshotService = snapshotService; - _titleDb = titleDb; - _configuration = configuration; - _indexBuilderService = indexBuilderService; - } - // ------------------------------------------------------------ // GET / // ------------------------------------------------------------ @@ -40,10 +22,10 @@ public sealed class IndexController : ControllerBase { if (HttpContext.Request.Headers.CacheControl == "no-cache") { - _indexBuilderService.InvalidateIndex(this, EventArgs.Empty); + indexBuilderService.InvalidateIndex(this, EventArgs.Empty); } - var index = _indexBuilderService.Build(HttpContext); + var index = indexBuilderService.Build(HttpContext); return Ok(index); } @@ -75,7 +57,7 @@ public sealed class IndexController : ControllerBase var titleId = match.Groups["id"].Value.ToUpperInvariant(); // ---- 2️⃣ Find the file that contains this TitleId ------------ - var entry = _snapshotService.GetSnapshot().Files + var entry = snapshotService.GetSnapshot().Files .FirstOrDefault(e => { return e.Titles.FirstOrDefault(hash => hash.TitleId == titleId)?.TitleId == titleId; }); if (entry == null) @@ -84,7 +66,7 @@ public sealed class IndexController : ControllerBase // ---- 3️⃣ If the file is a normal NSP → send it ---------------- if (Path.GetExtension(entry.Path).Equals(".nsp", StringComparison.OrdinalIgnoreCase) - && !entry.Path.Contains(_snapshotService.GetArchivePathSeparator())) + && !entry.Path.Contains(snapshotService.GetArchivePathSeparator())) { if (System.IO.File.Exists(entry.Path)) { @@ -116,7 +98,7 @@ public sealed class IndexController : ControllerBase if (IsInsideArchive(entry.Path)) { // Example: file is inside an archive – use ArchiveHandler - var innerFileName = entry.Path.Split(_snapshotService.GetArchivePathSeparator()).Last(); + var innerFileName = entry.Path.Split(snapshotService.GetArchivePathSeparator()).Last(); var stream = StreamFromArchive(entry, titleId, out var streamContainer); if (stream == null) @@ -149,7 +131,7 @@ public sealed class IndexController : ControllerBase // (e.g. "Games/MyGame.nsp" is a regular file; "archive.7z/mygame.nsp" // would be inside an archive). For simplicity we only check // for common archive extensions. - var filePath = path.Split(_snapshotService.GetArchivePathSeparator()).First(); + var filePath = path.Split(snapshotService.GetArchivePathSeparator()).First(); return filePath.EndsWith(".zip", StringComparison.OrdinalIgnoreCase) || filePath.EndsWith(".7z", StringComparison.OrdinalIgnoreCase) || filePath.EndsWith(".rar", StringComparison.OrdinalIgnoreCase); @@ -163,9 +145,9 @@ public sealed class IndexController : ControllerBase private Stream? StreamFromArchive(FileEntry fileEntry, string titleId, out IDisposable? streamContainer) { // Example: file is inside an archive – use ArchiveHandler - var archivePath = fileEntry.Path.Split(_snapshotService.GetArchivePathSeparator()).First(); - _snapshotService.GetArchiveName(titleId); - var innerFileName = Path.GetFileName(fileEntry.Path.Split(_snapshotService.GetArchivePathSeparator()).Last()); + var archivePath = fileEntry.Path.Split(snapshotService.GetArchivePathSeparator()).First(); + snapshotService.GetArchiveName(titleId); + var innerFileName = Path.GetFileName(fileEntry.Path.Split(snapshotService.GetArchivePathSeparator()).Last()); // Use SharpCompress to open the archive and find the entry. // Only the 3 archive types we support are handled. @@ -212,44 +194,4 @@ public sealed class IndexController : ControllerBase streamContainer = null; return null; } - - public class DependentStream : Stream - { - private readonly Stream _innerStream; - private readonly IDisposable? _parentContainer; - - public DependentStream(Stream innerStream, IDisposable? parentContainer) - { - _innerStream = innerStream; - _parentContainer = parentContainer; - } - - public override void Flush() => _innerStream.Flush(); - - public override int Read(byte[] buffer, int offset, int count) => _innerStream.Read(buffer, offset, count); - - public override long Seek(long offset, SeekOrigin origin) => _innerStream.Seek(offset, origin); - - public override void SetLength(long value) => _innerStream.SetLength(value); - - public override void Write(byte[] buffer, int offset, int count) => _innerStream.Write(buffer, offset, count); - - - public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) - { - return _innerStream.CopyToAsync(destination, bufferSize, cancellationToken); - } - - public override bool CanRead => _innerStream.CanRead; - public override bool CanSeek => _innerStream.CanSeek; - public override bool CanWrite => _innerStream.CanWrite; - public override long Length => _innerStream.Length; - public override long Position { get => _innerStream.Position; set => _innerStream.Position = value; } - protected override void Dispose(bool disposing) - { - _parentContainer?.Dispose(); - base.Dispose(disposing); - } - } - } \ No newline at end of file diff --git a/TinfoilVibeServer/Middleware/BasicAuthMiddleware.cs b/TinfoilVibeServer/Middleware/BasicAuthMiddleware.cs index dda7896..5b810e4 100644 --- a/TinfoilVibeServer/Middleware/BasicAuthMiddleware.cs +++ b/TinfoilVibeServer/Middleware/BasicAuthMiddleware.cs @@ -4,24 +4,17 @@ using TinfoilVibeServer.Authentication; namespace TinfoilVibeServer.Middleware; /// -/// Minimal Basic‑Auth middleware that also checks UID, failure counters and a blacklist. +/// Minimal Basic‑Auth middleware that also checks UID, failure counters, and a blacklist. /// -public sealed class BasicAuthMiddleware +public sealed class BasicAuthMiddleware(RequestDelegate next) { - private readonly RequestDelegate _next; - - public BasicAuthMiddleware(RequestDelegate next) - { - _next = next; - } - public async Task InvokeAsync(HttpContext context, IAuthStore store, ILogger logger) { // ------------- 1) Bypass auth for every path except “/” ---------------- // PathString is a struct – compare its value directly. if (!context.Request.Path.Equals("/", StringComparison.Ordinal)) { - await _next(context); + await next(context); return; } @@ -96,7 +89,7 @@ public sealed class BasicAuthMiddleware // Authentication succeeded – attach username for downstream handlers if needed context.Items["User"] = username; logger.LogInformation("User {User} authenticated successfully (UID={UID})", username, uid); - await _next(context); + await next(context); } private static void Challenge(HttpContext ctx) diff --git a/TinfoilVibeServer/Services/IndexBuilderService.cs b/TinfoilVibeServer/Services/IndexBuilderService.cs index e56b97e..28520c5 100644 --- a/TinfoilVibeServer/Services/IndexBuilderService.cs +++ b/TinfoilVibeServer/Services/IndexBuilderService.cs @@ -117,9 +117,9 @@ public sealed class IndexBuilderService: IHostedService } } - var fileName =Uri.EscapeDataString($"{name}[{titleId}][v{versionNumberParsed}][{patchOrApp}].nsp"); + var fileName = Uri.EscapeDataString($"{name}[{titleId}][v{versionNumberParsed}][{patchOrApp}].nsp"); var url = $"{baseUri.ToString().TrimEnd('/')}/{fileName}"; - var isWellFormed = Uri.TryCreate(url, UriKind.Absolute, out Uri? parsedUri); + var isWellFormed = Uri.TryCreate(url, UriKind.Absolute, out var parsedUri); if (isWellFormed && parsedUri != null) { diff --git a/TinfoilVibeServer/Services/SnapshotService.cs b/TinfoilVibeServer/Services/SnapshotService.cs index 81e400d..ab962fb 100644 --- a/TinfoilVibeServer/Services/SnapshotService.cs +++ b/TinfoilVibeServer/Services/SnapshotService.cs @@ -27,9 +27,18 @@ public interface ISnapshotService public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedService { #region FileSystemWatcher + + /* ============================================================== + * 1️⃣ FileSystemWatcher + * ============================================================== */ private readonly List _watchers = new(); + #endregion + #region Snapshot options & helpers + /* ============================================================== + * 2️⃣ Snapshot options & helpers + * ============================================================== */ private readonly SnapshotOptions _options; private readonly INSPExtractor _nspExtractor; private readonly IArchiveHandler _archiveHandler; @@ -38,27 +47,43 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ private readonly string _jsonPath; private readonly string _snapshotPath; private readonly ConcurrentDictionary _cache = new(); + private readonly ConcurrentDictionary _hashCache = new(); + // Archive full path -> FileEntry.Path private readonly ConcurrentDictionary _archiveLookup = new(); + // hash -> file size private readonly ConcurrentDictionary _sizeLookup = new(); private readonly IMemoryCache _debouncerCache; public event EventHandler? SnapshotRebuilt; public event EventHandler? SnapshotRebuilding; - - private readonly SemaphoreSlim _snapshotFileSemaphore = new(1,1); + + private readonly SemaphoreSlim _snapshotFileSemaphore = new(1, 1); private const char ArchivePathSeparator = '|'; + public char GetArchivePathSeparator() => ArchivePathSeparator; - - public SnapshotService( + #endregion + + /* ============================================================== + * 3️⃣ Build‑time guard + * ============================================================== */ + /// + /// Allows only one rebuild at a time. + /// + private readonly SemaphoreSlim _buildLock = new(1, 1); + + /* ============================================================== + * 4️⃣ Constructor + * ============================================================== */ + public SnapshotService( IMemoryCache debouncerCache, IOptionsMonitor options, INSPExtractor nspExtractor, IArchiveHandler archiveHandler, ILogger logger, IHostEnvironment environment - ) + ) { _options = options.CurrentValue; _debouncerCache = debouncerCache; @@ -66,8 +91,8 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ _archiveHandler = archiveHandler; _logger = logger; _environment = environment; - _jsonPath = Path.Combine(Path.DirectorySeparatorChar.ToString(),"app","data", _options.SnapshotFile); - + _jsonPath = Path.Combine(Path.DirectorySeparatorChar.ToString(), "app", "data", _options.SnapshotFile); + FileSystemExtensions.EnsureDirectoryExists(Path.GetFullPath(Path.GetDirectoryName(_jsonPath) ?? throw new InvalidOperationException())); if (!File.Exists(_jsonPath)) { @@ -75,30 +100,30 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ File.WriteAllText(_jsonPath, "[]"); _snapshotFileSemaphore.Release(); } - _snapshotPath = Path.Combine(Path.DirectorySeparatorChar.ToString(),"app","data", _options.SnapshotBackupFile); + + _snapshotPath = Path.Combine(Path.DirectorySeparatorChar.ToString(), "app", "data", _options.SnapshotBackupFile); FileSystemExtensions.EnsureDirectoryExists(Path.GetFullPath(Path.GetDirectoryName(_snapshotPath) ?? throw new InvalidOperationException())); - + // 1️⃣ Register for *property* changes - options.OnChange((snapshotOptions, arg) => - { - _options.RootDirectories = snapshotOptions.RootDirectories; - }); - _options.PropertyChanged += (s, e) => OnOptionsChanged(e.PropertyName); + options.OnChange((snapshotOptions, _) => { _options.RootDirectories = snapshotOptions.RootDirectories; }); + _options.PropertyChanged += (_, e) => OnOptionsChanged(e.PropertyName); if (_options.RootDirectories.Count == 0) { _logger.LogInformation("No directories set to watch for ROMS/Archives"); } + foreach (var path in _options.RootDirectories) { AddWatchDirectory(path); } } + // --------- Private helpers --------- private void OnOptionsChanged(string? propertyName) { if (propertyName != nameof(SnapshotOptions.RootDirectories)) return; - + _logger.LogInformation("Root directories changed, rebuilding snapshot"); var fileSystemWatchers = _watchers.Where(watcher => !_options.RootDirectories.Contains(watcher.Path)); var systemWatchers = fileSystemWatchers.ToList(); @@ -119,9 +144,13 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ BuildSnapshotAsync(); // rebuild everything PersistSnapshotAsync(); } - + #region FileSystemWatcher + + /* ============================================================== + * 5️⃣ FileSystemWatcher helpers + * ============================================================== */ private void AddWatchDirectory(string path) { if (!Directory.Exists(path)) return; @@ -156,9 +185,18 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ private void ThrottleSnapshotUpdate(FileSystemEventArgs fileSystemEventArgs) { + // If a rebuild is in progress, ignore the event immediately + if (_buildLock.CurrentCount == 0) // lock held by a rebuild + { + _logger.LogInformation( + "File system event {ChangeType} on {Path} ignored because a rebuild is already in progress", + fileSystemEventArgs.ChangeType, fileSystemEventArgs.FullPath); + return; + } + SnapshotRebuilding?.Invoke(this, fileSystemEventArgs); CancellationTokenSource cts = new(); - + using var cacheEntry = _debouncerCache.CreateEntry(fileSystemEventArgs.FullPath) .AddExpirationToken(new CancellationChangeToken(cts.Token)) .SetValue(fileSystemEventArgs) @@ -169,9 +207,9 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ new PostEvictionCallbackRegistration { EvictionCallback = - (key, value, reason, state) => + (_, value, reason, _) => { - if (!(reason == EvictionReason.Expired || reason == EvictionReason.TokenExpired)) return; + if (reason is not (EvictionReason.Expired or EvictionReason.TokenExpired)) return; if (value is FileSystemEventArgs args) { @@ -179,8 +217,9 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ { _logger.LogInformation("File {FilePath} is locked, skipping snapshot update", args.FullPath); using var rebounce = _debouncerCache.CreateEntry(args.FullPath) - .SetAbsoluteExpiration(TimeSpan.FromMilliseconds(DebounceMs)) + .AddExpirationToken(new CancellationChangeToken(cts.Token)) .SetValue(args); + cts.CancelAfter(TimeSpan.FromMilliseconds(DebounceMs)); } } @@ -193,6 +232,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ _logger.LogDebug("File system event {EventType} on {Path} at {Time}", fileSystemEventArgs.ChangeType, fileSystemEventArgs.FullPath, DateTime.Now.ToString("HH:mm:ss.fff")); } + private static bool IsFileLocked(string filePath) { FileStream? stream = null; @@ -210,6 +250,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ { stream?.Close(); } + return false; } @@ -229,9 +270,19 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ public Task AddToSnapshotAsync(FileEntry entry) { // Update lookup tables - _cache[entry.Path] = new SnapshotEntry(entry.Path, entry.Hash, entry.Size, entry.Titles); - _hashCache[entry.Hash] = entry.Path; - _sizeLookup[entry.Hash] = entry.Size; + if (entry.Hash != null) + { + var lastModified = File.GetLastWriteTimeUtc(entry.Path); + _cache[entry.Path] = new SnapshotEntry(entry.Path, entry.Hash, entry.Size, lastModified, entry.Titles); + _hashCache[entry.Hash] = entry.Path; + _sizeLookup[entry.Hash] = entry.Size; + } + else + { + _logger.LogWarning("Cannot add entry {Path} to snapshot: no hash", entry.Path); + return Task.CompletedTask; + } + if (entry.Path.Contains(ArchivePathSeparator)) { var filename = entry.Path.Split(ArchivePathSeparator)[0]; @@ -240,62 +291,88 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ foreach (var ncaMetadataWithHash in entry.Titles) { + if (ncaMetadataWithHash.Hash == null) + { + _logger.LogWarning("Cannot add entry {Path} to snapshot: no hash", entry.Path); + continue; + } + _hashCache[ncaMetadataWithHash.Hash] = entry.Path; _sizeLookup[ncaMetadataWithHash.Hash] = entry.Size; _logger.LogInformation("Added entry {titleId} to snapshot (hash={hash})", ncaMetadataWithHash.TitleId, ncaMetadataWithHash.Hash); } + // Persist snapshot to disk PersistSnapshotAsync(); return Task.CompletedTask; } + /* ============================================================== + * 6️⃣ Snapshot build / persistence helpers + * ============================================================== */ /// Builds _cache and _hashCache based on directory configuration public Task BuildSnapshotAsync() { - _logger.LogInformation("Building snapshot"); - var index = LoadSnapshotIndex(); - var latestModifiedUtcParallel = FileSystemExtensions.GetLatestModifiedUtcParallel(_options.RootDirectories); - var fileInfo = new FileInfo(_snapshotPath); - bool snapshotVerified = fileInfo.Exists; - if (latestModifiedUtcParallel.HasValue && latestModifiedUtcParallel.Value < fileInfo.LastWriteTimeUtc) + // Acquire the rebuild lock – if we cannot, skip this build. + if (!_buildLock.Wait(0)) { - if (index.Count != 0) + _logger.LogInformation("BuildSnapshotAsync called while rebuild in progress, ignoring."); + return Task.CompletedTask; + } + + try + { + _logger.LogInformation("Building snapshot"); + var index = LoadSnapshotIndex(); + var latestModifiedUtcParallel = FileSystemExtensions.GetLatestModifiedUtcParallel(_options.RootDirectories); + var fileInfo = new FileInfo(_snapshotPath); + bool snapshotVerified = fileInfo.Exists; + if (latestModifiedUtcParallel.HasValue && latestModifiedUtcParallel.Value < fileInfo.LastWriteTimeUtc) { - foreach (var dir in _options.RootDirectories) + if (index.Count != 0) { - var firstEntry = BuildSnapshot(dir).FirstOrDefault(); - if (firstEntry != null && !index.TryGetValue(firstEntry.Path, out _)) + foreach (var dir in _options.RootDirectories) { - snapshotVerified = false; - _logger.LogInformation("Snapshot does not contain first entry in directory {Directory}", dir); + // Snapshot is older than the latest modified file in the directory + var lastOrDefault = BuildSnapshot(dir).LastOrDefault(); + if (lastOrDefault != null && !index.TryGetValue(lastOrDefault.Path, out _)) + { + snapshotVerified = false; + _logger.LogInformation("Snapshot does not contain first entry in directory {Directory}", dir); + } } } } - } - if (!snapshotVerified) - { - _logger.LogInformation("Rebuilding snapshot (root dirs: {Count})", _options.RootDirectories.Count); - var entries = new List(); - foreach (var dir in _options.RootDirectories) + if (!snapshotVerified) { - foreach (var entry in BuildSnapshot(dir)) + _logger.LogInformation("Rebuilding snapshot (root dirs: {Count})", _options.RootDirectories.Count); + var entries = new List(); + foreach (var dir in _options.RootDirectories) { - if (entry != null) entries.Add(entry); + foreach (var entry in BuildSnapshot(dir)) + { + if (entry != null) entries.Add(entry); + } } - } - var currentHash = ComputeSnapshotHash(entries); - if (entries.Count > 0 || fileInfo.Exists && index.Count == 0) - SnapshotRebuilt?.Invoke(this, EventArgs.Empty); - } - PersistSnapshotAsync(); + var currentHash = ComputeSnapshotHash(entries); + if (entries.Count > 0 || fileInfo.Exists && index.Count == 0) + SnapshotRebuilt?.Invoke(this, EventArgs.Empty); + } + + PersistSnapshotAsync(); + } + finally + { + _buildLock.Release(); + } return Task.CompletedTask; } public void GetArchiveName(string titleId) { - ; + } // Returns List of FileEntry that do not have a hash in the cache @@ -303,7 +380,11 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ private IEnumerable BuildSnapshot(string dir) { if (!Directory.Exists(dir)) yield break; - foreach (var file in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories)) + foreach (var file in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories).OrderBy(file => + { + var fileInfo = new FileInfo(file); + return fileInfo.LastWriteTimeUtc; + })) { var hash = string.Empty; var ext = Path.GetExtension(file).ToLowerInvariant(); @@ -329,6 +410,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ AddToSnapshotAsync(fileEntryFromFileName); yield return fileEntryFromFileName; } + using var nspStream = File.OpenRead(file); hash = ComputeFirstStreamHash(nspStream); @@ -369,7 +451,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ } if (titlesEnumerable == null) continue; - + titles = titlesEnumerable.ToList(); foreach (var title in titles) { @@ -394,11 +476,11 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ else { _logger.LogInformation("Added {File} to snapshot (hash={Hash})", file, hash); - yield return new FileEntry(file, titles.Select((tuple, i) => tuple.Item2).FirstOrDefault(), hash, titles.Select((tuple, i) => tuple.Item3).ToList()); + yield return new FileEntry(file, titles.Select((tuple, _) => tuple.Item2).FirstOrDefault(), hash, titles.Select((tuple, _) => tuple.Item3).ToList()); } } } - + private async Task ValidateSnapshotAsync(CancellationToken cancellationToken = default) { await Task.CompletedTask; @@ -410,7 +492,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ private IEnumerable GetEntries() { - foreach (var kv in _cache) + foreach (var kv in _cache.OrderByDescending(pair => pair.Value.LastModified)) yield return new FileEntry(kv.Key, kv.Value.Size, kv.Value.Hash, kv.Value.NcaMetadataWithHash); } @@ -423,6 +505,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ } var entries = GetEntries().ToList(); + var newHash = ComputeSnapshotHash(entries); var snapshot = GetSnapshot(); if (snapshot.Hash == newHash) return Task.CompletedTask; @@ -437,9 +520,9 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ { new PostEvictionCallbackRegistration { - EvictionCallback = (key, value, reason, state) => + EvictionCallback = (key, value, reason, _) => { - if (!(reason == EvictionReason.Expired || reason == EvictionReason.TokenExpired)) + if (reason is not (EvictionReason.Expired or EvictionReason.TokenExpired)) return; var filePath = (string)key; if (_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout)) @@ -453,6 +536,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ else { File.WriteAllText(filePath, JsonSerializer.Serialize(value, _jsonSerializerOptions)); + _logger.LogInformation("Persisted snapshot to {FilePath}", filePath); SnapshotRebuilt?.Invoke(this, EventArgs.Empty); } } @@ -484,6 +568,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ var hash = sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(json)); return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant(); } + /// /// From filesystem cache, load each entry and build the lookups /// Check for duplicate hashes @@ -503,6 +588,12 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ // Reindex the cache foreach (var fileEntry in entries) { + if (fileEntry.Hash == null) + { + _logger.LogError("Entry {Path} has no hash", fileEntry.Path); + continue; + } + if (_hashCache.TryGetValue(fileEntry.Hash, out var value)) { _logger.LogWarning("Duplicate hash found in snapshot: {Hash}, {OldPath}, {newPath}", fileEntry.Hash, value, fileEntry.Path); @@ -514,7 +605,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ _logger.LogWarning("Nonexistent entry found: {Path}", fileEntry.Path); continue; } - + var fileContainedInRootDirectories = false; foreach (var optionsRootDirectory in _options.RootDirectories) { @@ -529,54 +620,73 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ { _logger.LogInformation("Entry {Path} is not contained in any root directory", fileEntry.Path); continue; - }; - + } + if (_options.RomExtensions.Contains(Path.GetExtension(fileEntry.Path))) { + var fileInfo = new FileInfo(fileEntry.Path); if (fileEntry.Path.Contains(ArchivePathSeparator)) { var filename = fileEntry.Path.Split(ArchivePathSeparator)[0]; - _cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileEntry.Titles!); + // ReSharper disable once RedundantSuppressNullableWarningExpression + _cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileInfo.LastWriteTimeUtc, fileEntry.Titles!); _archiveLookup[filename] = fileEntry.Path; } else { - _cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileEntry.Titles!); + // ReSharper disable once RedundantSuppressNullableWarningExpression + _cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileInfo.LastWriteTimeUtc, fileEntry.Titles!); fileEntries.TryAdd(fileEntry.Path, fileEntry); _hashCache[fileEntry.Hash] = fileEntry.Path; // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract if (fileEntry.Titles == null) continue; foreach (var ncaMetadataWithHash in fileEntry.Titles) { + if (ncaMetadataWithHash.Hash == null) continue; _hashCache[ncaMetadataWithHash.Hash] = fileEntry.Path; } } } } + _logger.LogInformation("Loaded snapshot index {Count} entries", fileEntries.Count); return fileEntries; } catch (ArgumentException e) { _logger.LogError(e, "Failed to load snapshot"); - return new(); + return new(); } } public void RebuildSnapshot() { - // 1️⃣ Flush the old in‑memory snapshot - _cache.Clear(); - _hashCache.Clear(); - _archiveLookup.Clear(); - _sizeLookup.Clear(); - //_failedAttempts.Clear(); // if you keep per‑user counters + // Fast path: if we already have the lock, just log and exit. + if (!_buildLock.Wait(0)) + { + _logger.LogInformation("RebuildSnapshot called while a rebuild is already in progress, ignoring."); + return; + } + try + { + // 1️⃣ Flush the old in‑memory snapshot + _cache.Clear(); + _hashCache.Clear(); + _archiveLookup.Clear(); + _sizeLookup.Clear(); + //_failedAttempts.Clear(); // if you keep per‑user counters - // 2️⃣ Re‑build from disk again - BuildSnapshotAsync().Wait(); // synchronous – we already own the lock - PersistSnapshotAsync().Wait(); // same - SnapshotRebuilt?.Invoke(this, EventArgs.Empty); + // 2️⃣ Re‑build from disk again + BuildSnapshotAsync().Wait(); // synchronous – we already own the lock + PersistSnapshotAsync().Wait(); // same + SnapshotRebuilt?.Invoke(this, EventArgs.Empty); + } + finally + { + _buildLock.Release(); + } } + #endregion public ROMSnapshot GetSnapshot() @@ -612,7 +722,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ return new ROMSnapshot(); } - + public void Dispose() { foreach (var watcher in _watchers) @@ -621,7 +731,10 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ } } - private sealed record SnapshotEntry(string Path, string Hash, long Size, List NcaMetadataWithHash); + /// + /// Represents a single ROM/archive entry in the snapshot cache. + /// + private sealed record SnapshotEntry(string Path, string Hash, long Size, DateTime LastModified, List NcaMetadataWithHash); // File: TinfoilVibeServer/Services/SnapshotService.cs (inside SnapshotService class) @@ -671,8 +784,8 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ public class ROMSnapshot { - public string? Hash { get; set; } - public IReadOnlyList Files { get; set; } = new List(); + public string? Hash { get; init; } + public IReadOnlyList Files { get; init; } = new List(); } public async Task StartAsync(CancellationToken cancellationToken) diff --git a/TinfoilVibeServer/Utilities/DependentStream.cs b/TinfoilVibeServer/Utilities/DependentStream.cs new file mode 100644 index 0000000..e50a8c0 --- /dev/null +++ b/TinfoilVibeServer/Utilities/DependentStream.cs @@ -0,0 +1,31 @@ +namespace TinfoilVibeServer.Utilities; + +public class DependentStream(Stream innerStream, IDisposable? parentContainer) : Stream +{ + public override void Flush() => innerStream.Flush(); + + public override int Read(byte[] buffer, int offset, int count) => innerStream.Read(buffer, offset, count); + + public override long Seek(long offset, SeekOrigin origin) => innerStream.Seek(offset, origin); + + public override void SetLength(long value) => innerStream.SetLength(value); + + public override void Write(byte[] buffer, int offset, int count) => innerStream.Write(buffer, offset, count); + + + public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) + { + return innerStream.CopyToAsync(destination, bufferSize, cancellationToken); + } + + public override bool CanRead => innerStream.CanRead; + public override bool CanSeek => innerStream.CanSeek; + public override bool CanWrite => innerStream.CanWrite; + public override long Length => innerStream.Length; + public override long Position { get => innerStream.Position; set => innerStream.Position = value; } + protected override void Dispose(bool disposing) + { + parentContainer?.Dispose(); + base.Dispose(disposing); + } +} \ No newline at end of file diff --git a/TinfoilVibeServer/Utilities/FileSystemExtensions.cs b/TinfoilVibeServer/Utilities/FileSystemExtensions.cs index 7d5f73a..1d3815e 100644 --- a/TinfoilVibeServer/Utilities/FileSystemExtensions.cs +++ b/TinfoilVibeServer/Utilities/FileSystemExtensions.cs @@ -112,8 +112,7 @@ public static class FileSystemExtensions /// Thrown if a file exists at the target path or the directory cannot be created. public static void EnsureDirectoryExists(string? path) { - if (path is null) - throw new ArgumentNullException(nameof(path)); + ArgumentNullException.ThrowIfNull(path); if (string.IsNullOrWhiteSpace(path)) throw new ArgumentException("Path must not be empty or whitespace.", nameof(path));