diff --git a/.idea/.idea.TinfoilVibeServer/.idea/vcs.xml b/.idea/.idea.TinfoilVibeServer/.idea/vcs.xml index 94a25f7..c656bff 100644 --- a/.idea/.idea.TinfoilVibeServer/.idea/vcs.xml +++ b/.idea/.idea.TinfoilVibeServer/.idea/vcs.xml @@ -2,5 +2,6 @@ + \ No newline at end of file diff --git a/TinfoilVibeServer.sln.DotSettings.user b/TinfoilVibeServer.sln.DotSettings.user index 103bd5f..a8484bd 100644 --- a/TinfoilVibeServer.sln.DotSettings.user +++ b/TinfoilVibeServer.sln.DotSettings.user @@ -3,9 +3,17 @@ True ForceIncluded ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded @@ -15,35 +23,75 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded ForceIncluded ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded + <AssemblyExplorer> <Assembly Path="D:\Cloud\Git\TinfoilVibeServer\TinfoilVibeServer\libhac\src\LibHac\bin\Release\net8.0\LibHac.dll" /> + <Assembly Path="D:\Cloud\Git\TinfoilVibeServer\libhac\src\LibHac\bin\Release\net8.0\LibHac.dll" /> </AssemblyExplorer> <SessionState ContinuousTestingMode="0" Name="All tests from Solution" xmlns="urn:schemas-jetbrains-com:jetbrains-ut-session"> diff --git a/TinfoilVibeServer/Controllers/IndexController.cs b/TinfoilVibeServer/Controllers/IndexController.cs index f8c6387..5043574 100644 --- a/TinfoilVibeServer/Controllers/IndexController.cs +++ b/TinfoilVibeServer/Controllers/IndexController.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Configuration; +using SharpCompress.Readers; using TinfoilVibeServer.Models; using TinfoilVibeServer.Services; @@ -18,12 +19,12 @@ public sealed class IndexController : ControllerBase private readonly IndexBuilderService _indexBuilderService; public IndexController(ISnapshotService snapshotService, - TitleDatabaseService titleDb, - IConfiguration configuration, IndexBuilderService indexBuilderService) + TitleDatabaseService titleDb, + IConfiguration configuration, IndexBuilderService indexBuilderService) { _snapshotService = snapshotService; - _titleDb = titleDb; - _configuration = configuration; + _titleDb = titleDb; + _configuration = configuration; _indexBuilderService = indexBuilderService; } @@ -39,8 +40,9 @@ public sealed class IndexController : ControllerBase { if (HttpContext.Request.Headers.CacheControl == "no-cache") { - _snapshotService.RebuildSnapshot(); + _indexBuilderService.InvalidateIndex(this, EventArgs.Empty); } + var index = _indexBuilderService.Build(); return Ok(index); @@ -56,7 +58,7 @@ public sealed class IndexController : ControllerBase /// /// The relative file path requested. [HttpGet("{*path}")] - public IActionResult Download(string path) + public async Task Download(string path) { if (string.IsNullOrWhiteSpace(path)) return BadRequest("Missing url query parameter."); @@ -64,7 +66,7 @@ public sealed class IndexController : ControllerBase // ---- 1️⃣ Parse the brackets -------------------------------- // Expected format: [name][TitleId][v][patchOrApp].nsp var match = System.Text.RegularExpressions.Regex.Match(path, - @"\[(?.*?)\]\[(?[0-9a-fA-F]{8}[0-9a-fA-F]{8})\]\[(?[0-9a-fA-F]+)\]\[(?Base|Update)\]\.nsp", + @"(?.*?)\[(?[0-9a-fA-F]{8}[0-9a-fA-F]{8})\]\[v(?[0-9]+)\]\[(?Base|Update)\]\.nsp", System.Text.RegularExpressions.RegexOptions.IgnoreCase); if (!match.Success) @@ -74,41 +76,52 @@ public sealed class IndexController : ControllerBase // ---- 2️⃣ Find the file that contains this TitleId ------------ var entry = _snapshotService.GetSnapshot().Files - .FirstOrDefault(e => e.Title?.TitleId == titleId); + .FirstOrDefault(e => { return e.Titles.FirstOrDefault(hash => hash.TitleId == titleId)?.TitleId == titleId; }); if (entry == null) return NotFound("No file with that TitleId found."); // ---- 3️⃣ If the file is a normal NSP → send it ---------------- - if (Path.GetExtension(entry.Path).Equals(".nsp", StringComparison.OrdinalIgnoreCase)) + + if (Path.GetExtension(entry.Path).Equals(".nsp", StringComparison.OrdinalIgnoreCase) && !entry.Path.Contains(_snapshotService.GetArchivePathSeparator())) { - // Check if it is inside an archive. - // If the path contains a slash that is not the root separator - // it might be an entry inside an archive; we simply stream it. - // - For normal files, we can use SendFileAsync. - // - For archives, we stream the entry using ArchiveHandler. - - if (IsInsideArchive(entry.Path)) - { - // Example: file is inside an archive – use ArchiveHandler - var archivePath = Path.GetDirectoryName(entry.Path); - var innerFileName = Path.GetFileName(entry.Path); - var stream = StreamFromArchive(archivePath, innerFileName); - - if (stream == null) - return NotFound("Could not stream entry from archive."); - - return File(stream, "application/octet-stream", - Path.GetFileName(innerFileName)); - } - else + if (System.IO.File.Exists(entry.Path)) { // Regular file – just serve it. return PhysicalFile(entry.Path, "application/octet-stream", - Path.GetFileName(entry.Path)); + Path.GetFileName(entry.Path)); } } + // Check if it is inside an archive. + // If the path contains a slash that is not the root separator + // it might be an entry inside an archive; we simply stream it. + // - For normal files, we can use SendFileAsync. + // - For archives, we stream the entry using ArchiveHandler. + + if (IsInsideArchive(entry.Path)) + { + // Example: file is inside an archive – use ArchiveHandler + var innerFileName = entry.Path.Split(_snapshotService.GetArchivePathSeparator()).Last(); + var stream = StreamFromArchive(entry, titleId, out var streamContainer); + + if (stream == null) + return NotFound("Could not stream entry from archive."); + + var wrappedStream = new DependentStream(stream, streamContainer); + var contentDisposition = $"inline; filename=\"{innerFileName}\""; + + Response.ContentType = "application/octet-stream"; + Response.ContentLength = entry.Size; + Response.Headers["Content-Disposition"] = contentDisposition; + await using var entryStream = wrappedStream; + const int bufferSize = 10 * 1024 * 1024;// 81920; // 80 KB – default used by CopyToAsync + await entryStream.CopyToAsync(Response.Body, bufferSize, HttpContext.RequestAborted); + + // Once the copy completes, the `await using` disposes the archive. + return new EmptyResult(); // We already wrote the stream to Response.Body + } + return NotFound("Requested URL does not reference an NSP file."); } @@ -116,22 +129,30 @@ public sealed class IndexController : ControllerBase /// Very light‑weight helper – decides whether the file path /// represents a file inside an archive. /// - private bool IsInsideArchive(string path) => + private bool IsInsideArchive(string path) + { // If the path contains a separator that is not a root separator // (e.g. "Games/MyGame.nsp" is a regular file; "archive.7z/mygame.nsp" // would be inside an archive). For simplicity we only check // for common archive extensions. - path.EndsWith(".zip", StringComparison.OrdinalIgnoreCase) || - path.EndsWith(".7z", StringComparison.OrdinalIgnoreCase) || - path.EndsWith(".rar", StringComparison.OrdinalIgnoreCase); + var filePath = path.Split(_snapshotService.GetArchivePathSeparator()).First(); + return filePath.EndsWith(".zip", StringComparison.OrdinalIgnoreCase) || + filePath.EndsWith(".7z", StringComparison.OrdinalIgnoreCase) || + filePath.EndsWith(".rar", StringComparison.OrdinalIgnoreCase); + } /// /// If the NSP is inside an archive, this method opens the archive /// and returns the entry stream. It is deliberately minimal – /// if the archive can’t be opened we return null. /// - private Stream? StreamFromArchive(string archivePath, string innerFileName) + private Stream? StreamFromArchive(FileEntry fileEntry, string titleId, out IDisposable? streamContainer) { + // Example: file is inside an archive – use ArchiveHandler + var archivePath = fileEntry.Path.Split(_snapshotService.GetArchivePathSeparator()).First(); + _snapshotService.GetArchiveName(titleId); + var innerFileName = Path.GetFileName(fileEntry.Path.Split(_snapshotService.GetArchivePathSeparator()).Last()); + // Use SharpCompress to open the archive and find the entry. // Only the 3 archive types we support are handled. try @@ -139,28 +160,31 @@ public sealed class IndexController : ControllerBase // Check which archive type if (archivePath.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) { - using var zip = SharpCompress.Archives.Zip.ZipArchive.Open(archivePath); + var zip = SharpCompress.Archives.Zip.ZipArchive.Open(archivePath, new ReaderOptions { LeaveStreamOpen = true }); + streamContainer = zip; var entry = zip.Entries - .FirstOrDefault(e => e.Key.Equals(innerFileName, - StringComparison.OrdinalIgnoreCase)); + .FirstOrDefault(e => e.Key != null && e.Key.Equals(innerFileName, + StringComparison.OrdinalIgnoreCase)); if (entry != null) return entry.OpenEntryStream(); } else if (archivePath.EndsWith(".7z", StringComparison.OrdinalIgnoreCase)) { - using var sevenZip = SharpCompress.Archives.SevenZip.SevenZipArchive.Open(archivePath); + var sevenZip = SharpCompress.Archives.SevenZip.SevenZipArchive.Open(archivePath, new ReaderOptions { LeaveStreamOpen = true }); + streamContainer = sevenZip; var entry = sevenZip.Entries - .FirstOrDefault(e => e.Key.Equals(innerFileName, - StringComparison.OrdinalIgnoreCase)); + .FirstOrDefault(e => e.Key != null && e.Key.Equals(innerFileName, + StringComparison.OrdinalIgnoreCase)); if (entry != null) return entry.OpenEntryStream(); } else if (archivePath.EndsWith(".rar", StringComparison.OrdinalIgnoreCase)) { - using var rar = SharpCompress.Archives.Rar.RarArchive.Open(archivePath); + var rar = SharpCompress.Archives.Rar.RarArchive.Open(archivePath, new ReaderOptions { LeaveStreamOpen = true }); + streamContainer = rar; var entry = rar.Entries - .FirstOrDefault(e => e.Key.Equals(innerFileName, - StringComparison.OrdinalIgnoreCase)); + .FirstOrDefault(e => e.Key != null && e.Key.Equals(innerFileName, + StringComparison.OrdinalIgnoreCase)); if (entry != null) return entry.OpenEntryStream(); } @@ -170,6 +194,42 @@ public sealed class IndexController : ControllerBase // ignore – we will just return null and the controller // will respond with 404. } + + streamContainer = null; return null; } + + public class DependentStream : Stream + { + private readonly Stream _innerStream; + private readonly IDisposable? _parentContainer; + + public DependentStream(Stream innerStream, IDisposable? parentContainer) + { + _innerStream = innerStream; + _parentContainer = parentContainer; + } + + public override void Flush() => _innerStream.Flush(); + + public override int Read(byte[] buffer, int offset, int count) => _innerStream.Read(buffer, offset, count); + + public override long Seek(long offset, SeekOrigin origin) => _innerStream.Seek(offset, origin); + + public override void SetLength(long value) => _innerStream.SetLength(value); + + public override void Write(byte[] buffer, int offset, int count) => _innerStream.Write(buffer, offset, count); + + public override bool CanRead => _innerStream.CanRead; + public override bool CanSeek => _innerStream.CanSeek; + public override bool CanWrite => _innerStream.CanWrite; + public override long Length => _innerStream.Length; + public override long Position { get => _innerStream.Position; set => _innerStream.Position = value; } + protected override void Dispose(bool disposing) + { + _parentContainer?.Dispose(); + base.Dispose(disposing); + } + } + } \ No newline at end of file diff --git a/TinfoilVibeServer/Middleware/BasicAuthMiddleware.cs b/TinfoilVibeServer/Middleware/BasicAuthMiddleware.cs index a0f245c..11ab6f2 100644 --- a/TinfoilVibeServer/Middleware/BasicAuthMiddleware.cs +++ b/TinfoilVibeServer/Middleware/BasicAuthMiddleware.cs @@ -17,6 +17,14 @@ public sealed class BasicAuthMiddleware public async Task InvokeAsync(HttpContext context, IAuthStore store, ILogger logger) { + // ------------- 1) Bypass auth for every path except “/” ---------------- + // PathString is a struct – compare its value directly. + if (!context.Request.Path.Equals("/", StringComparison.Ordinal)) + { + await _next(context); + return; + } + logger.LogInformation("Incoming request from {IP} – {Method} {Path}", context.Connection.RemoteIpAddress, context.Request.Method, context.Request.Path); var ip = context.Connection.RemoteIpAddress?.ToString() ?? "unknown"; diff --git a/TinfoilVibeServer/Models/FileEntry.cs b/TinfoilVibeServer/Models/FileEntry.cs index 57fc593..d73f050 100644 --- a/TinfoilVibeServer/Models/FileEntry.cs +++ b/TinfoilVibeServer/Models/FileEntry.cs @@ -6,8 +6,8 @@ namespace TinfoilVibeServer.Models; /// One line in the snapshot – the JSON will be an array of these. /// public sealed record FileEntry( - string Path, - long Size, - string Hash, // SHA‑256 hex - NcaMetadataWithHash? Title // null unless file is an NSP/XCI or an archive containing one + string Path, // nsp or archive path + long Size, // size of nsp or full archive + string Hash, // SHA‑256 hex of first NCA of first NCP in NSP or archive + List Titles // Details of all NSP Roms in the Path ); \ No newline at end of file diff --git a/TinfoilVibeServer/Models/SnapshotOptions.cs b/TinfoilVibeServer/Models/SnapshotOptions.cs index 417060a..46e651f 100644 --- a/TinfoilVibeServer/Models/SnapshotOptions.cs +++ b/TinfoilVibeServer/Models/SnapshotOptions.cs @@ -17,16 +17,16 @@ public sealed class SnapshotOptions : INotifyPropertyChanged } } } - private List _whitelistExtensions = new(); - public List WhitelistExtensions + private List _archiveExtensions = new(); + public List ArchiveExtensions { - get => _whitelistExtensions; + get => _archiveExtensions; set { - if (_whitelistExtensions != value) + if (_archiveExtensions != value) { - _whitelistExtensions = value; - OnPropertyChanged(nameof(_whitelistExtensions)); + _archiveExtensions = value; + OnPropertyChanged(nameof(_archiveExtensions)); } } } diff --git a/TinfoilVibeServer/Program.cs b/TinfoilVibeServer/Program.cs index b2196ba..8dbe16a 100644 --- a/TinfoilVibeServer/Program.cs +++ b/TinfoilVibeServer/Program.cs @@ -1,3 +1,4 @@ +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Options; using TinfoilVibeServer.Authentication; using TinfoilVibeServer.Middleware; @@ -24,11 +25,13 @@ builder.Services.AddSingleton(sp => var keySet = KeySetHolder.KeySet; // already loaded by ConfigManager return new NSPExtractor(keySet, logger); }); -builder.Services.AddSingleton(); +builder.Services.AddSingleton(); +builder.Services.AddSingleton(sp => sp.GetRequiredService()); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); +builder.Services.AddHostedService(provider => provider.GetRequiredService()); builder.Services.AddHostedService(provider => provider.GetRequiredService()).AddHttpClient(); builder.Services.AddHostedService(provider => provider.GetRequiredService()); builder.Services.AddControllers(); // add MVC @@ -47,6 +50,7 @@ app.MapControllers(); // routes the /index.json & /download endpoints app.MapGet("/debug", () => new SnapshotService( + app.Services.GetRequiredService(), app.Services.GetRequiredService>(), app.Services.GetRequiredService(), app.Services.GetRequiredService(), diff --git a/TinfoilVibeServer/Properties/launchSettings.json b/TinfoilVibeServer/Properties/launchSettings.json index c876394..7158c2e 100644 --- a/TinfoilVibeServer/Properties/launchSettings.json +++ b/TinfoilVibeServer/Properties/launchSettings.json @@ -5,7 +5,7 @@ "commandName": "Project", "dotnetRunMessages": true, "launchBrowser": false, - "applicationUrl": "http://localhost:5253", + "applicationUrl": "http://192.168.1.145:80", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } diff --git a/TinfoilVibeServer/Services/ArchiveHandler.cs b/TinfoilVibeServer/Services/ArchiveHandler.cs index 434c0d0..69a8ae4 100644 --- a/TinfoilVibeServer/Services/ArchiveHandler.cs +++ b/TinfoilVibeServer/Services/ArchiveHandler.cs @@ -3,7 +3,9 @@ using SharpCompress.Archives; using SharpCompress.Archives.Zip; using SharpCompress.Archives.Rar; using SharpCompress.Archives.SevenZip; +using SharpCompress.Common; using TinfoilVibeServer.Models; +using TinfoilVibeServer.Utilities; using ZipArchive = SharpCompress.Archives.Zip.ZipArchive; namespace TinfoilVibeServer.Services; @@ -13,7 +15,12 @@ public interface IArchiveHandler /// /// Return TitleInfo if an embedded Nintendo archive is found; otherwise null. /// - NcaMetadataWithHash? TryExtractTitleInfo(string filePath); + IEnumerable<(string, long, NcaMetadataWithHash)> TryExtractTitleInfos(string filePath); + + /// + /// Return TitleInfo if an embedded Nintendo archive is found; otherwise null. + /// + IEnumerable<(string, long, NcaMetadataWithHash)> TryExtractTitleInfos(Stream archiveStream, string archiveType); } /// @@ -35,7 +42,7 @@ public sealed class ArchiveHandler : IArchiveHandler /// /// Return TitleInfo if an embedded Nintendo archive is found; otherwise null. /// - public NcaMetadataWithHash? TryExtractTitleInfo(string filePath) + public IEnumerable<(string, long, NcaMetadataWithHash)> TryExtractTitleInfos(string filePath) { _logger.LogInformation("Examining archive {File} for embedded NSP", filePath); var ext = Path.GetExtension(filePath).ToLowerInvariant(); @@ -53,7 +60,7 @@ public sealed class ArchiveHandler : IArchiveHandler default: { _logger.LogWarning("Unsupported archive type {Extension} – skipping", ext); - return null; + return []; } } } @@ -61,28 +68,31 @@ public sealed class ArchiveHandler : IArchiveHandler { _logger.LogError("Error opening archive {File}: {Exception}", filePath, ex.Message); // Graceful fallback – return null - return null; + return []; } } - private NcaMetadataWithHash? HandleZip(string path) + public IEnumerable<(string, long, NcaMetadataWithHash)> TryExtractTitleInfos(Stream archiveStream, string archiveType) + { + throw new NotImplementedException(); + } + + private IEnumerable<(string, long, NcaMetadataWithHash)> HandleZip(string path) { using var archive = ZipArchive.Open(path); foreach (var entry in archive.Entries) { - if (!entry.IsDirectory && IsRomArchive(entry.Key)) - { - var temp = Path.GetTempFileName(); - entry.WriteToFile(temp); - var title = _nspExtractor.ExtractFromFile(temp); // instance call - File.Delete(temp); - return title; - } + if (entry.IsDirectory || entry.Key == null || !IsRomArchive(entry.Key)) continue; + + var temp = Path.GetTempFileName(); + entry.WriteToFile(temp); + var title = _nspExtractor.ExtractFromFile(temp); // instance call + File.Delete(temp); + if (title != null) yield return (entry.Key, entry.Size, title); } - return null; } - private NcaMetadataWithHash? Handle7z(string path) + private IEnumerable<(string, long, NcaMetadataWithHash)> Handle7z(string path) { using var archive = SevenZipArchive.Open(path); foreach (var entry in archive.Entries) @@ -93,35 +103,71 @@ public sealed class ArchiveHandler : IArchiveHandler entry.WriteToFile(temp); var title = _nspExtractor.ExtractFromFile(temp); // instance call File.Delete(temp); - return title; + if (title != null) yield return (entry.Key, entry.Size, title); } } - return null; } - private NcaMetadataWithHash? HandleRar(string path) + private IEnumerable<(string, long, NcaMetadataWithHash)> HandleRar(string path) { + var titles = new List<(string, long, NcaMetadataWithHash)>(); + var entryCount = 0; try { using var archive = RarArchive.Open(path); + entryCount = archive.Entries.Count; foreach (var entry in archive.Entries) { - if (!entry.IsDirectory && IsRomArchive(entry.Key)) + if (entry.IsDirectory || entry.Key == null || !IsRomArchive(entry.Key)) continue; + { + try + { + using var streamWrapper = new SeekableBufferedStream(entry.OpenEntryStream(), entry.Size, 64 * 1024 * 1024, false); + var title = _nspExtractor.ExtractFromStream(streamWrapper); + if (title != null) titles.Add((entry.Key, entry.Size, title)); + } + catch (Exception e) + { + if (e.Message.StartsWith("Failed to extract NSP")) + { + _logger.LogError("Failed to extract title info from archive {Archive}: {Exception}", path, e.Message); + } + else + { + throw; + } + } + } + } + } + catch (Exception exception) + { + if (titles.Count > 0 && titles.Count == entryCount) + { + // broken archive but managed to read some titles + _logger.LogInformation("Failed to fully process archive with SharpCompress, but found {Count} titles: {Exception}", titles.Count, exception.Message); + return titles; + } + + // Fallback to SharpSevenZip (if needed) + _logger.LogInformation( + "Failed to open archive with SharpCompress, falling back to SharpSevenZip {Exception}", + exception.Message); + using var archive = SevenZipArchive.Open(path); + foreach (var entry in archive.Entries) + { + if (entry is { IsDirectory: false, Key: not null } && IsRomArchive(entry.Key)) { var temp = Path.GetTempFileName(); entry.WriteToFile(temp); var title = _nspExtractor.ExtractFromFile(temp); // instance call File.Delete(temp); - return title; + if (title != null) titles.Add((entry.Key, entry.Size, title)); } } - return null; - } - catch (SharpCompress.Common.ArchiveException) - { - // Fallback to SharpSevenZip (if needed) - return null; } + + return titles; } private bool IsRomArchive(string entryName) diff --git a/TinfoilVibeServer/Services/IndexBuilderService.cs b/TinfoilVibeServer/Services/IndexBuilderService.cs index 9d32d6e..f5de0e2 100644 --- a/TinfoilVibeServer/Services/IndexBuilderService.cs +++ b/TinfoilVibeServer/Services/IndexBuilderService.cs @@ -4,6 +4,7 @@ using System.Globalization; using System.Linq; using System.Security.Cryptography; using System.Text.Json; +using System.Text.RegularExpressions; using LibHac.Ncm; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Logging; @@ -23,6 +24,7 @@ public sealed class IndexBuilderService: IHostedService private readonly ILogger _logger; private readonly string _cachePath; + private readonly SemaphoreSlim _lock = new(1, 1); public IndexBuilderService( ISnapshotService snapshotService, TitleDatabaseService titleDb, @@ -41,9 +43,10 @@ public sealed class IndexBuilderService: IHostedService // 1️⃣ Load cache if it exists var cached = LoadCache(); var snapshot = _snapshotService.GetSnapshot(); - if (string.IsNullOrEmpty(snapshot.Hash)) + + if (string.IsNullOrEmpty(snapshot.Hash) || snapshot.Files.Count == 0) { - _snapshotService.BuildSnapshot(); + _snapshotService.BuildSnapshotAsync(); snapshot = _snapshotService.GetSnapshot(); } @@ -57,33 +60,14 @@ public sealed class IndexBuilderService: IHostedService _logger.LogInformation("Building index (snapshot size={Count})", snapshot.Files.Count); // 3️⃣ Build new index from snapshot entries - var files = snapshot.Files - .Where(e => e.Title != null) - .Select(e => - { - var titleId = e.Title.TitleId; - var name = _titleDb.TryGetTitle(titleId, out var t) - ? t.Name - : "Unknown"; - - var vProcessed = e.Title.Version * 0x10000; - var patchOrApp = e.Title.ContentMetaType == ContentMetaType.Application ? "Base" : "Update"; - if (e.Title.ContentMetaType == ContentMetaType.Patch) - { - name = _titleDb.TryGetTitle(e.Title.ApplicationTitle, out var appTitle) ? appTitle.Name : "Unknown"; - } - var url = $"{name}[{titleId}][{vProcessed}][{patchOrApp}].nsp"; - - return new FileDto(url, e.Size); - }) - .ToList(); + var files = ParseSnapshotFiles(snapshot); var directories = _configuration.GetSection("Directories") .Get() ?? Array.Empty(); var success = _configuration["SuccessMessage"] ?? string.Empty; - var index = new IndexDto(files, directories.ToList(), success); + var index = new IndexDto(files.SelectMany(inner => inner).ToList(), directories.ToList(), success); // 4️⃣ Persist cache PersistCache(snapshotHash, index); @@ -91,6 +75,57 @@ public sealed class IndexBuilderService: IHostedService return index; } + private List> ParseSnapshotFiles(SnapshotService.ROMSnapshot snapshot) + { + var files = snapshot.Files + .Where(e => e.Titles.Count > 0) + .Select(e => + { + var fileDtos = new List(); + foreach (var title in e.Titles) + { + var titleId = title.TitleId; + var name = _titleDb.TryGetTitle(titleId, out var t) + ? t.Name + : "Unknown"; + + var versionNumberParsed = (title.ContentMetaType == ContentMetaType.Application) ? title.Version : title.Version * 0x10000; + var patchOrApp = title.ContentMetaType == ContentMetaType.Application ? "Base" : "Update"; + if (title.ContentMetaType == ContentMetaType.Patch || + title.ContentMetaType == ContentMetaType.AddOnContent) + { + // Patch should use the application title name + name = _titleDb.TryGetTitle(title.ApplicationTitle, out var appTitle) + ? appTitle.Name + : "Unknown"; + //_logger.LogInformation("Patch title {TitleId} uses application title {ApplicationTitle}", titleId, title.ApplicationTitle); + } + + // if still unknown, its probably a demo, use filename extraction + if (name == "Unknown") + { + var match = Regex.Match(Path.GetFileNameWithoutExtension(e.Path), "^(.+?)\\s*(?:\\[.*)?$", + RegexOptions.None); + if (match.Success) + { + name = match.Groups[1].Value; + _logger.LogInformation("Name not found for {TitleId}, using filename: {Name}", titleId, + name); + } + } + + var url = $"http://192.168.1.145/{name}[{titleId}][v{versionNumberParsed}][{patchOrApp}].nsp"; + + fileDtos.Add(new FileDto(url, e.Size)); + } + + return fileDtos; + + }) + .ToList(); + return files; + } + private IndexCache? LoadCache() { if (!File.Exists(_cachePath)) return null; @@ -120,11 +155,24 @@ public sealed class IndexBuilderService: IHostedService public Task StartAsync(CancellationToken cancellationToken) { Build(); + this._snapshotService.SnapshotRebuilt += InvalidateIndex; return Task.CompletedTask; } + public void InvalidateIndex(object? sender, EventArgs e) + { + if (!File.Exists(_cachePath)) return; + + File.Delete(_cachePath); + _logger.LogInformation("Index cache cleared"); + } + public Task StopAsync(CancellationToken cancellationToken) - => Task.CompletedTask; // nothing special to do on shutdown + { + _snapshotService.SnapshotRebuilt -= InvalidateIndex; + return Task.CompletedTask; // nothing special to do on shutdown + } + #endregion } \ No newline at end of file diff --git a/TinfoilVibeServer/Services/NSPExtractor.cs b/TinfoilVibeServer/Services/NSPExtractor.cs index 4bd4487..d51f323 100644 --- a/TinfoilVibeServer/Services/NSPExtractor.cs +++ b/TinfoilVibeServer/Services/NSPExtractor.cs @@ -1,10 +1,4 @@ -// File: Services/NSPExtactor.cs -// *** UPDATED *** -using System; -using System.IO; -using System.Linq; -using System.Collections.Generic; -using System.Security.Cryptography; +using System.Security.Cryptography; using LibHac.Common; using LibHac.Fs; using LibHac.Fs.Fsa; @@ -13,8 +7,8 @@ using LibHac.Tools.FsSystem; using LibHac.Tools.FsSystem.NcaUtils; using LibHac.Common.Keys; using LibHac.Ncm; +using LibHac.Tools.Fs; using LibHac.Tools.Ncm; -using TinfoilVibeServer.Models; namespace TinfoilVibeServer.Services { @@ -61,22 +55,73 @@ namespace TinfoilVibeServer.Services /// public NcaMetadataWithHash? ExtractFromStream(Stream stream) { - _logger.LogInformation("Extracting NSP from stream (length={Length}", stream.Length); - if (!IsPfs0FileSystem(stream)) - return null; - + if (!stream.CanSeek) return null; stream.Seek(0, SeekOrigin.Begin); - + + _logger.LogInformation("Extracting NSP/XCI from stream (length={Length})", stream.Length); using var storage = new StreamStorage(stream, false); - var partition = new PartitionFileSystem(); - partition.Initialize(storage).ThrowIfFailure(); + if (IsPfs0FileSystem(stream)) + { + return ExtractNSPFromStream(storage); + } + if (IsXciFileSystem(stream)) + { + var xci = new Xci(_keySet, storage); + List ncaEntries; + if (xci.HasPartition(XciPartitionType.Secure)) + { + _logger.LogInformation("Processing as XCI"); + var partition = xci.OpenPartition(XciPartitionType.Secure); + ncaEntries = partition + .EnumerateEntries("*.cnmt.nca", SearchOptions.RecurseSubdirectories) + .Where(e => e.Type == DirectoryEntryType.File) + .ToList(); + byte[]? hash = null; + foreach (var dirEntry in ncaEntries) + { + using var fileRef = new UniqueRef(); + partition.OpenFile(ref fileRef.Ref, dirEntry.FullPath.ToU8Span(), OpenMode.Read).ThrowIfFailure(); + using var ncaFile = fileRef.Release(); + using var ncaFileStorage = new FileStorage(ncaFile); + + var nca = new Nca(_keySet, ncaFileStorage); + if (hash == null) + { + // Hash the *first* NCA stream – the stream we just opened + using var sha256 = SHA256.Create(); + using var ncaStream = ncaFile.AsStream(); + hash = sha256.ComputeHash(ncaStream); + } + + if (nca.Header.ContentType != NcaContentType.Meta) + continue; // only the meta NCA contains title metadata + + string titleId = nca.Header.TitleId.ToString("X16"); + var (contentMetaType, applicationTitleId, titleVersion) = GetMetaData(nca); + + _logger.LogInformation("Meta NCA found – TitleId={TitleId} Version={Version}", titleId, titleVersion); + // XCI can never be a patch? + return new NcaMetadataWithHash(titleId, applicationTitleId.ToString("X16"), titleVersion.Major, ContentMetaType.Application, BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant()); + } + } + } + + return null; // no meta NCA found + } + + private NcaMetadataWithHash? ExtractNSPFromStream(StreamStorage storage) + { + List ncaEntries; + _logger.LogInformation("Processing as NSP"); + var partition = new PartitionFileSystem(); + partition.Initialize(storage).ThrowIfFailure(); // Find the first *.nca that contains the meta header - var ncaEntries = partition + ncaEntries = partition .EnumerateEntries("*.nca", SearchOptions.RecurseSubdirectories) .Where(e => e.Type == DirectoryEntryType.File) .ToList(); - + byte[]? hash = null; foreach (var dirEntry in ncaEntries) { using var fileRef = new UniqueRef(); @@ -85,30 +130,31 @@ namespace TinfoilVibeServer.Services using var ncaFileStorage = new FileStorage(ncaFile); var nca = new Nca(_keySet, ncaFileStorage); + if (hash == null) + { + // Hash the *first* NCA stream – the stream we just opened + using var sha256 = SHA256.Create(); + using var ncaStream = ncaFile.AsStream(); + hash = sha256.ComputeHash(ncaStream); + } + if (nca.Header.ContentType != NcaContentType.Meta) continue; // only the meta NCA contains title metadata _logger.LogInformation("Meta NCA found – TitleId={TitleId} Version={Version}", nca.Header.TitleId, nca.Header.Version); string titleId = nca.Header.TitleId.ToString("X16"); - int version = nca.Header.Version; - bool isPatch = nca.IsPatch; - bool isApp = nca.IsProgram && !isPatch; - - // Hash the *first* NCA stream – the stream we just opened - using var ncaStream = ncaFile.AsStream(); - using var sha256 = SHA256.Create(); - var hash = sha256.ComputeHash(ncaStream); - - var (contentMetaType,applicationTitle) = GetMetaDataType(nca); + + var (contentMetaType,applicationTitle,titleVersion) = GetMetaData(nca); if (contentMetaType != null) - return new NcaMetadataWithHash(titleId, applicationTitle.ToString("X16"), version, contentMetaType.Value, BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant()); + return new NcaMetadataWithHash(titleId, applicationTitle.ToString("X16"), titleVersion.Minor, contentMetaType.Value, BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant()); } - return null; // no meta NCA found + return null; } - private static (ContentMetaType?,ulong) GetMetaDataType(Nca nca) + + private static (ContentMetaType?,ulong, TitleVersion) GetMetaData(Nca nca) { - if (nca.Header.ContentType != NcaContentType.Meta) return (null,0); + if (nca.Header.ContentType != NcaContentType.Meta) return (null,0, new TitleVersion(0, true)); using var openFileSystem = nca.OpenFileSystem(0, IntegrityCheckLevel.ErrorOnInvalid); foreach (var entry in openFileSystem.EnumerateEntries("*.cnmt", SearchOptions.Default)) { @@ -121,11 +167,12 @@ namespace TinfoilVibeServer.Services var cnmt = new Cnmt(asStream); var applicationTitle = cnmt.ApplicationTitleId; - return (cnmt.Type,applicationTitle); + return (cnmt.Type,applicationTitle, cnmt.TitleVersion); } - return (null,0); + return (null,0, new TitleVersion(0, true)); } + /// /// Quick sanity check that the stream looks like a PFS0 file system. /// @@ -136,7 +183,7 @@ namespace TinfoilVibeServer.Services if (!stream.CanSeek) return false; stream.Seek(0, SeekOrigin.Begin); - var storage = new StreamStorage(stream, false); + var storage = new StreamStorage(stream, true); var partition = new PartitionFileSystem(); partition.Initialize(storage).ThrowIfFailure(); return true; @@ -147,6 +194,32 @@ namespace TinfoilVibeServer.Services return false; } } + private bool IsXciFileSystem(Stream stream) + { + try + { + if (!stream.CanSeek) return false; + stream.Seek(0, SeekOrigin.Begin); + + var storage = new StreamStorage(stream, true); + try + { + var xciBlock = new Xci(_keySet, storage); + _logger.LogInformation("XCI found"); + return xciBlock.HasPartition(XciPartitionType.Secure); + } + catch + { + // ignored + } + return false; + } + catch (Exception e) + { + _logger.LogError("Failed to extract XCI: {Exception}", e.Message); + return false; + } + } public string ExtractHashFromStream(Stream nspStream) { @@ -208,13 +281,14 @@ namespace TinfoilVibeServer.Services public ContentMetaType ContentMetaType { get; set; } public string Hash { get; } - public NcaMetadataWithHash(string titleId, string applicationTitle, int version, ContentMetaType contentMetaType, string hash) + public NcaMetadataWithHash(string titleId, string applicationTitle, int version, + ContentMetaType contentMetaType, string hash) { TitleId = titleId; ApplicationTitle = applicationTitle; Version = version; ContentMetaType = contentMetaType; - Hash = hash; + Hash = hash; } } } \ No newline at end of file diff --git a/TinfoilVibeServer/Services/SnapshotService.cs b/TinfoilVibeServer/Services/SnapshotService.cs index c787d1b..a6c1962 100644 --- a/TinfoilVibeServer/Services/SnapshotService.cs +++ b/TinfoilVibeServer/Services/SnapshotService.cs @@ -1,6 +1,7 @@ using System.Collections.Concurrent; using System.Security.Cryptography; using System.Text.Json; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Options; using TinfoilVibeServer.Models; using TinfoilVibeServer.Utilities; @@ -11,55 +12,81 @@ public interface ISnapshotService event EventHandler SnapshotRebuilt; // raised after a rebuild void RebuildSnapshot(); SnapshotService.ROMSnapshot GetSnapshot(); - void BuildSnapshot(); + + Task AddToSnapshotAsync(FileEntry entry); + Task BuildSnapshotAsync(); + void GetArchiveName(string titleId); + char GetArchivePathSeparator(); } /// /// Keeps an in‑memory snapshot, watches the filesystem for changes, and /// only re‑processes a file if its hash changed. /// -public sealed class SnapshotService : IDisposable, ISnapshotService +public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedService { + #region FileSystemWatcher + private readonly List _watchers = new(); + #endregion + private readonly SnapshotOptions _options; private readonly INSPExtractor _nspExtractor; private readonly IArchiveHandler _archiveHandler; private readonly ILogger _logger; private readonly string _jsonPath; private readonly string _snapshotPath; - private readonly List _watchers = new(); - private readonly ConcurrentDictionary _cache = new(); - private string? _currentSnapshotHash; - private readonly Timer _debounceTimer; + private readonly ConcurrentDictionary _cache = new(); + private readonly ConcurrentDictionary _hashCache = new(); + // Archive full path -> FileEntry.Path + private readonly ConcurrentDictionary _archiveLookup = new(); + // hash -> file size + private readonly ConcurrentDictionary _sizeLookup = new(); + private readonly IMemoryCache _debouncerCache; public event EventHandler? SnapshotRebuilt; - + public event EventHandler? SnapshotRebuilding; + + private readonly SemaphoreSlim _snapshotFileSemaphore = new(1,1); + private const char ArchivePathSeparator = '|'; + public char GetArchivePathSeparator() => ArchivePathSeparator; + public SnapshotService( + IMemoryCache debouncerCache, IOptionsMonitor options, INSPExtractor nspExtractor, IArchiveHandler archiveHandler, ILogger logger) { _options = options.CurrentValue; + _debouncerCache = debouncerCache; _nspExtractor = nspExtractor; _archiveHandler = archiveHandler; _logger = logger; _jsonPath = Path.Combine(AppContext.BaseDirectory, _options.SnapshotFile); - FileSystemExtensions.EnsureDirectoryExists(Path.GetDirectoryName(_jsonPath)); + + // Debounce timer for persisting snapshot + long debounceTime = 200; + var entryOptions = new MemoryCacheEntryOptions() + .SetSlidingExpiration(TimeSpan.FromSeconds(debounceTime)).RegisterPostEvictionCallback((key, value, reason, + state) => + { + + _logger.LogInformation("Should persist the snapshot {Key}, {Reason}", key, reason); + }); // <‑‑ sliding! + FileSystemExtensions.EnsureDirectoryExists(Path.GetFullPath(Path.GetDirectoryName(_jsonPath))); if (!File.Exists(_jsonPath)) { + _snapshotFileSemaphore.Wait(); File.WriteAllText(_jsonPath, "[]"); + _snapshotFileSemaphore.Release(); } _snapshotPath = Path.Combine(AppContext.BaseDirectory, _options.SnapshotBackupFile); - FileSystemExtensions.EnsureDirectoryExists(Path.GetDirectoryName(_snapshotPath)); + FileSystemExtensions.EnsureDirectoryExists(Path.GetFullPath(Path.GetDirectoryName(_snapshotPath))); // 1️⃣ Register for *property* changes _options.PropertyChanged += (s, e) => OnOptionsChanged(e.PropertyName); - BuildSnapshot(); // initial scan - File.WriteAllText(_snapshotPath, JsonSerializer.Serialize(GetSnapshot())); - _debounceTimer = new Timer(_ => DebounceElapsed(), null, Timeout.Infinite, Timeout.Infinite); - foreach (var path in _options.RootDirectories) { - InitializeFileSystemWatcher(path); + AddWatchDirectory(path); } } // --------- Private helpers --------- @@ -81,15 +108,18 @@ public sealed class SnapshotService : IDisposable, ISnapshotService foreach (var newWatchedDirectory in newWatchedDirectories) { - InitializeFileSystemWatcher(newWatchedDirectory); + AddWatchDirectory(newWatchedDirectory); } - BuildSnapshot(); // rebuild everything - PersistSnapshot(); + BuildSnapshotAsync(); // rebuild everything + PersistSnapshotAsync(); } } - private void InitializeFileSystemWatcher(string path) + + + #region FileSystemWatcher + private void AddWatchDirectory(string path) { if (!Directory.Exists(path)) return; var watcher = new FileSystemWatcher @@ -104,32 +134,84 @@ public sealed class SnapshotService : IDisposable, ISnapshotService watcher.Deleted += OnChanged; watcher.Renamed += OnRenamed; watcher.EnableRaisingEvents = true; - + _logger.LogInformation("Watching {Path}", path); _watchers.Add(watcher); } - #region FileSystemWatcher + private void RemoveWatchDirectory(string path) + { + var fileSystemWatchers = _watchers.FirstOrDefault(watcher => watcher.Path == path); + if (fileSystemWatchers == null) return; + fileSystemWatchers.EnableRaisingEvents = false; + fileSystemWatchers.Dispose(); + _logger.LogInformation("Stopped watching {Path}", path); + _watchers.Remove(fileSystemWatchers); + } private void OnChanged(object? _, FileSystemEventArgs e) => ThrottleSnapshotUpdate(e); private void OnRenamed(object? _, RenamedEventArgs e) => ThrottleSnapshotUpdate(e); private void ThrottleSnapshotUpdate(FileSystemEventArgs fileSystemEventArgs) { - lock (_lock) - { - _debounceTimer.Change(_debounceMs, Timeout.Infinite); // reset the timer - _logger.LogDebug("File system event {EventType} on {Path} at {Time}", fileSystemEventArgs.ChangeType, fileSystemEventArgs.FullPath, DateTime.Now.ToString("HH:mm:ss")); - } - /*Task.Run(async () => - { - await Task.Delay(250); - _logger.LogDebug("File system event {EventType} on {Path}", fileSystemEventArgs.ChangeType, fileSystemEventArgs.FullPath); - UpdateSnapshot(); - });*/ + SnapshotRebuilding?.Invoke(this, fileSystemEventArgs); + using var cacheEntry = _debouncerCache.CreateEntry(fileSystemEventArgs.FullPath) + //.SetAbsoluteExpiration(TimeSpan.FromMilliseconds(DebounceMs)) + .SetValue(fileSystemEventArgs) + .SetOptions(new MemoryCacheEntryOptions + { + PostEvictionCallbacks = + { + new PostEvictionCallbackRegistration + { + EvictionCallback = + (key, value, reason, state) => + { + if (reason != EvictionReason.Expired) return; + + if (value is FileSystemEventArgs args) + { + if (IsFileLocked(args.FullPath)) + { + _logger.LogInformation("File {FilePath} is locked, skipping snapshot update", args.FullPath); + using var rebounce = _debouncerCache.CreateEntry(args.FullPath) + .SetAbsoluteExpiration(TimeSpan.FromMilliseconds(DebounceMs)) + .SetValue(args); + } + } + + RebuildSnapshot(); + } + } + } + }); + cacheEntry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMilliseconds(DebounceMs); + + _logger.LogDebug("File system event {EventType} on {Path} at {Time}", fileSystemEventArgs.ChangeType, + fileSystemEventArgs.FullPath, DateTime.Now.ToString("HH:mm:ss")); } - - private readonly object _lock = new object(); - private int _debounceMs = 200; + private static bool IsFileLocked(string filePath) + { + FileStream? stream = null; + var file = new FileInfo(filePath); + + try + { + stream = file.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.None); + } + catch (IOException) + { + return true; + } + finally + { + stream?.Close(); + } + return false; + } + + private const int DebounceMs = 400; + private readonly JsonSerializerOptions _jsonSerializerOptions = new() { IncludeFields = true }; + private int SnapshotFileLockTimeout { get; } = 1000; private void DebounceElapsed() { @@ -140,15 +222,66 @@ public sealed class SnapshotService : IDisposable, ISnapshotService #region Snapshot logic - public void BuildSnapshot() + public Task AddToSnapshotAsync(FileEntry entry) { + // Update lookup tables + _cache[entry.Path] = new SnapshotEntry(entry.Path, entry.Hash, entry.Size, entry.Titles); + _hashCache[entry.Hash] = entry.Path; + _sizeLookup[entry.Hash] = entry.Size; + if (entry.Path.Contains(ArchivePathSeparator)) + { + var filename = entry.Path.Split(ArchivePathSeparator)[0]; + _archiveLookup[filename] = entry.Path; + } + + foreach (var ncaMetadataWithHash in entry.Titles) + { + _hashCache[ncaMetadataWithHash.Hash] = entry.Path; + _sizeLookup[ncaMetadataWithHash.Hash] = entry.Size; + _logger.LogInformation("Added entry {titleId} to snapshot (hash={hash})", ncaMetadataWithHash.TitleId, ncaMetadataWithHash.Hash); + } + // Persist snapshot to disk + PersistSnapshotAsync(); + return Task.CompletedTask; + } + + /// Builds _cache and _hashCache based on directory configuration + public Task BuildSnapshotAsync() + { + _logger.LogInformation("Building snapshot"); var index = LoadSnapshotIndex(); var latestModifiedUtcParallel = FileSystemExtensions.GetLatestModifiedUtcParallel(_options.RootDirectories); var fileInfo = new FileInfo(_snapshotPath); + bool snapshotVerified = true; if (latestModifiedUtcParallel.HasValue && latestModifiedUtcParallel.Value < fileInfo.LastWriteTimeUtc) { - _logger.LogInformation("Snapshot is up to date"); - return; + if (index.Count != 0) + { + // directory may have been added with older roms, verify that the snapshot is still up to date + foreach (var dir in _options.RootDirectories) + { + // check first entry is in index + var entry = BuildSnapshot(dir).FirstOrDefault(); + if (entry != null) + { + if (!index.TryGetValue(entry.Path, out var cached)) + { + snapshotVerified = false; + _logger.LogInformation("Snapshot does not contain first entry in directory {Directory}", dir); + } + } + } + + if (snapshotVerified) + { + _logger.LogInformation("Snapshot is up to date"); + return Task.CompletedTask; + } + } + else + { + _logger.LogInformation("Snapshot is up to date but index is empty"); + } } _logger.LogInformation("Rebuilding snapshot (root dirs: {Count})", _options.RootDirectories.Count); var entries = new List(); @@ -156,65 +289,122 @@ public sealed class SnapshotService : IDisposable, ISnapshotService var snapshotChanged = false; foreach (var dir in _options.RootDirectories) { - if (!Directory.Exists(dir)) continue; - foreach (var file in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories)) + _ = Task.Run(() => { - var ext = Path.GetExtension(file).ToLowerInvariant(); - - if (!(_options.WhitelistExtensions.Contains(ext) || _options.RomExtensions.Contains(ext))) - continue; - - if (index.TryGetValue(file, out var value)) - { - entries.Add(value); - continue; - } - - // 3) extract title if applicable - string hash; - NcaMetadataWithHash? title = null; - if (_options.RomExtensions.Contains(ext)) - { - using var nspStream = File.OpenRead(file); - hash = ComputeFirstStreamHash(nspStream); - - // 2) use cached title if unchanged - if (index.TryGetValue(file, out var cached) && cached.Hash == hash) - { - entries.Add(cached); - continue; - } - - title = _nspExtractor.ExtractFromStream(nspStream); - } - else - { - hash = ComputeFirstStreamHash(file); - title = _archiveHandler.TryExtractTitleInfo(file); - } - - if (title == null) - { - _logger.LogInformation("Failed to process {File}", file); - } - // 4) update cache - _cache[file] = new CachedFile(file, hash, title); - - // 5) add to snapshot - entries.Add(new FileEntry(file, new FileInfo(file).Length, hash, title)); - _logger.LogInformation("Added {File} to snapshot (hash={Hash}", file, hash); - snapshotChanged = true; - } + _logger.LogInformation("Rebuilding directory {Directory}", dir); + var buildSnapshot = BuildSnapshot(dir); + var fileEntries = buildSnapshot.ToList(); + snapshotChanged = snapshotChanged || fileEntries.Count != 0; + entries.AddRange(fileEntries.Where(entry => entry != null)!); + }); } // Replace the entire snapshot - _currentSnapshotHash = ComputeSnapshotHash(entries); - File.WriteAllText(_jsonPath, JsonSerializer.Serialize(entries)); + ComputeSnapshotHash(entries); if (snapshotChanged) { _logger.LogInformation("Snapshot rebuilt"); SnapshotRebuilt?.Invoke(this, EventArgs.Empty); } + return Task.CompletedTask; + } + + public void GetArchiveName(string titleId) + { + ; + } + + // Returns List of FileEntry that do not have a hash in the cache + // Each entry that has not been added to the lookup table is added to the cache + private IEnumerable BuildSnapshot(string dir) + { + FileEntry entry; + if (!Directory.Exists(dir)) yield break; + foreach (var file in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories)) + { + string hash = string.Empty; + var ext = Path.GetExtension(file).ToLowerInvariant(); + + if (!(_options.ArchiveExtensions.Contains(ext) || _options.RomExtensions.Contains(ext))) + continue; + + if (_cache.ContainsKey(file) || _hashCache.ContainsKey(hash)) + { + continue; + } + + // 3) extract title if applicable + var titles = new List<(string, long, NcaMetadataWithHash)>(); + if (_options.RomExtensions.Contains(ext)) + { + using var nspStream = File.OpenRead(file); + hash = ComputeFirstStreamHash(nspStream); + + if (_hashCache.ContainsKey(hash)) + { + continue; + } + + var nspStreamLength = nspStream.Length; + var title = _nspExtractor.ExtractFromStream(nspStream); + if (title != null) + { + var archiveEntry = new FileEntry(file, nspStreamLength, hash, [title]); + AddToSnapshotAsync(archiveEntry); + titles.Add((title.TitleId, nspStreamLength, title)); + yield return archiveEntry; + } + } + else + { + if (_options.ArchiveExtensions.Contains(ext)) + { + if (_archiveLookup.ContainsKey(file)) continue; + hash = ComputeFirstStreamHash(file); + if (_hashCache.ContainsKey(hash)) + { + yield return null; + } + + IEnumerable<(string, long, NcaMetadataWithHash)>? titlesEnumerable = null; + try + { + titlesEnumerable = _archiveHandler.TryExtractTitleInfos(file); + } + catch (Exception e) + { + _logger.LogError(e, "Failed to extract title info from archive {Archive}", file); + } + + if (titlesEnumerable == null) continue; + + titles = titlesEnumerable.ToList(); + foreach (var title in titles) + { + var archiveEntry = new FileEntry(file + ArchivePathSeparator + title.Item1, title.Item2, title.Item3.Hash, [title.Item3]); + AddToSnapshotAsync(archiveEntry); + yield return archiveEntry; + } + /*var fileEntry = new FileEntry(file, new FileInfo(file).Length, hash, titles.Select((tuple, i) => tuple.Item3).ToList()); + AddToSnapshotAsync(fileEntry); + yield return fileEntry;*/ + } + else + { + continue; + } + } + + if (titles.Count == 0) + { + _logger.LogInformation("Failed to process {File}", file); + } + else + { + _logger.LogInformation("Added {File} to snapshot (hash={Hash})", file, hash); + yield return new FileEntry(file, titles.Select((tuple, i) => tuple.Item2).FirstOrDefault(), hash, titles.Select((tuple, i) => tuple.Item3).ToList()); + } + } } private string ComputeFirstStreamHash(Stream nspStream) @@ -222,20 +412,65 @@ public sealed class SnapshotService : IDisposable, ISnapshotService return _nspExtractor.ExtractHashFromStream(nspStream); } - private void UpdateSnapshot() => BuildSnapshot(); + private void UpdateSnapshot() => BuildSnapshotAsync(); - private void PersistSnapshot() + IEnumerable GetEntries() { - var snapshot = GetSnapshot(); - var newHash = ComputeSnapshotHash(snapshot.Files); - if (_currentSnapshotHash != newHash) + foreach (var snapshotEntry in _cache) { - _logger.LogInformation("Snapshot hash changed – persisting new snapshot"); - _currentSnapshotHash = newHash; - File.WriteAllText(_jsonPath, JsonSerializer.Serialize(snapshot.Files)); - File.WriteAllText(_snapshotPath, JsonSerializer.Serialize(snapshot.Files)); + _sizeLookup.TryGetValue(snapshotEntry.Value.Hash, out var size); + var fileEntry = new FileEntry(snapshotEntry.Key, snapshotEntry.Value.Size, snapshotEntry.Value.Hash, snapshotEntry.Value.NcaMetadataWithHash); + yield return fileEntry; } } + private Task PersistSnapshotAsync() + { + if (_debouncerCache.TryGetValue(_jsonPath, out var value)) + { + _logger.LogInformation("Sliding debounce in progress, skipping snapshot persistence"); + return Task.CompletedTask; + } + var snapshot = GetSnapshot(); + var entries = GetEntries(); + var fileEntries = entries.ToList(); + var newHash = ComputeSnapshotHash(fileEntries); + if (snapshot.Hash == newHash) return Task.CompletedTask; + + _logger.LogInformation("Snapshot hash changed – persisting new snapshot"); + using var debouncedPersistence = _debouncerCache.CreateEntry(_jsonPath); + debouncedPersistence.SlidingExpiration = TimeSpan.FromMilliseconds(DebounceMs); + debouncedPersistence.Value = fileEntries; + debouncedPersistence.PostEvictionCallbacks.Add(new PostEvictionCallbackRegistration + { + EvictionCallback = (key, entriesCallback, reason, state) => + { + if (entriesCallback is IEnumerable entriesToPersist && key is string filePath) + { + if (_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout)) + { + if (IsFileLocked(filePath)) + { + _logger.LogInformation("File {FilePath} is locked, skipping snapshot persistence", filePath); + } + else + { + File.WriteAllText(filePath, + JsonSerializer.Serialize(entriesToPersist, _jsonSerializerOptions)); + _snapshotFileSemaphore.Release(); + _logger.LogInformation("Persisted snapshot"); + SnapshotRebuilt?.Invoke(this, EventArgs.Empty); + } + } + else + { + _logger.LogInformation("Failed to persist file {FilePath} due to timeout", filePath); + } + } + } + }); + return Task.CompletedTask; + } + private static string ComputeHash(string filePath) { @@ -252,38 +487,103 @@ public sealed class SnapshotService : IDisposable, ISnapshotService var hash = sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(json)); return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant(); } + /// + /// From filesystem cache, load each entry and build the lookups + /// + /// private Dictionary LoadSnapshotIndex() { - if (!File.Exists(_jsonPath)) return new(); - + if (!File.Exists(_jsonPath)) return new Dictionary(); + _snapshotFileSemaphore.Wait(); var json = File.ReadAllText(_jsonPath); - var entries = JsonSerializer.Deserialize>(json, new JsonSerializerOptions(){IncludeFields = true})!; - return entries.ToDictionary(e => e.Path, e => e); + _snapshotFileSemaphore.Release(); + var entries = JsonSerializer.Deserialize>(json, _jsonSerializerOptions)!; + try + { + var fileEntries = new Dictionary(); + // Reindex the cache + foreach (var fileEntry in entries) + { + if (_hashCache.TryGetValue(fileEntry.Hash, out var value)) + { + _logger.LogWarning("Duplicate hash found in snapshot: {Hash}, {OldPath}, {newPath}", fileEntry.Hash, value, fileEntry.Path); + } + + if (_options.RomExtensions.Contains(Path.GetExtension(fileEntry.Path))) + { + if (fileEntry.Path.Contains(ArchivePathSeparator)) + { + var filename = fileEntry.Path.Split(ArchivePathSeparator)[0]; + _cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileEntry.Titles); + _archiveLookup[filename] = fileEntry.Path; + } + else + { + _cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileEntry.Titles); + fileEntries.TryAdd(fileEntry.Path, fileEntry); + _hashCache[fileEntry.Hash] = fileEntry.Path; + // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract + if (fileEntry.Titles == null) continue; + foreach (var ncaMetadataWithHash in fileEntry.Titles) + { + _hashCache[ncaMetadataWithHash.Hash] = fileEntry.Path; + } + } + } + } + return fileEntries; + } + catch (ArgumentException e) + { + _logger.LogError(e, "Failed to load snapshot"); + return new(); + } + } + + + public void RebuildSnapshot() + { + // Build a fresh snapshot and persist it. + BuildSnapshotAsync(); // private method inside the same class + PersistSnapshotAsync(); // private method inside the same class + SnapshotRebuilt?.Invoke(this, EventArgs.Empty); } #endregion public ROMSnapshot GetSnapshot() { - if (!File.Exists(_jsonPath)) return new(); - var json = File.ReadAllText(_jsonPath); - var hash = ComputeHash(_jsonPath); - var romSnapshot = new ROMSnapshot() + if (!File.Exists(_jsonPath)) return new ROMSnapshot(); + + if (_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout)) { - Hash = hash, - Files = JsonSerializer.Deserialize>(json, - new JsonSerializerOptions() { IncludeFields = true })! - }; - return romSnapshot; - } + try + { + var json = File.ReadAllText(_jsonPath); + var hash = ComputeHash(_jsonPath); + var romSnapshot = new ROMSnapshot + { + Hash = hash, + Files = JsonSerializer.Deserialize>(json, _jsonSerializerOptions)! + }; + return romSnapshot; + } + catch (Exception e) + { + _logger.LogError(e, "Failed to load snapshot"); + } + finally + { + _snapshotFileSemaphore.Release(); + } + } + else + { + _logger.LogWarning("Failed to load snapshot due to timeout"); + } - public void RebuildSnapshot() - { - // Build a fresh snapshot and persist it. - BuildSnapshot(); // private method inside the same class - PersistSnapshot(); // private method inside the same class - SnapshotRebuilt?.Invoke(this, EventArgs.Empty); + return new ROMSnapshot(); } - + public void Dispose() { foreach (var watcher in _watchers) @@ -292,7 +592,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService } } - private sealed record CachedFile(string Path, string Hash, NcaMetadataWithHash? NcaMetadataWithHash); + private sealed record SnapshotEntry(string Path, string Hash, long Size, List NcaMetadataWithHash); // File: TinfoilVibeServer/Services/SnapshotService.cs (inside SnapshotService class) @@ -342,4 +642,20 @@ public sealed class SnapshotService : IDisposable, ISnapshotService public string Hash { get; set; } public IReadOnlyList Files { get; set; } = new List(); } + + public async Task StartAsync(CancellationToken cancellationToken) + { + _logger.LogInformation("Starting snapshot service"); + _ = Task.Run(async () => + { + await BuildSnapshotAsync(); + await PersistSnapshotAsync(); + }, cancellationToken); // initial scan + new Timer(_ => DebounceElapsed(), null, Timeout.Infinite, Timeout.Infinite); + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + Dispose(); + } } \ No newline at end of file diff --git a/TinfoilVibeServer/Services/TitleDatabaseService.cs b/TinfoilVibeServer/Services/TitleDatabaseService.cs index 6194130..d29340a 100644 --- a/TinfoilVibeServer/Services/TitleDatabaseService.cs +++ b/TinfoilVibeServer/Services/TitleDatabaseService.cs @@ -32,18 +32,9 @@ public sealed class TitleDatabaseService : IHostedService private readonly ISnapshotService _snapshotService; private readonly Dictionary _titleIdToPath = new Dictionary(); - // 1️⃣ Cache for the JSON data (key = TitleId) - /* - private readonly ConcurrentDictionary _titleData - = new(); - - // 2️⃣ Reverse lookup: TitleId → real file‑system path - private readonly ConcurrentDictionary _titleIdToPath - = new(); - */ - + // Regex to find a 16‑digit hex TitleId in a filename - private static readonly Regex _titleIdRegex = new( + private static readonly Regex TitleIdRegex = new( @"([0-9a-fA-F]{8})([0-9a-fA-F]{8})", RegexOptions.Compiled); #endregion @@ -80,7 +71,15 @@ public sealed class TitleDatabaseService : IHostedService Path.Combine(AppContext.BaseDirectory, "Games") }; // Reload cache immediately when a snapshot rebuild occurs - _snapshotService.SnapshotRebuilt += (_, _) => ReloadCacheAsync(); + _snapshotService.SnapshotRebuilt += SnapshotServiceOnSnapshotRebuilt; + _options.OnChange(OptionsChanged); + } + + private void OptionsChanged(TitleDbOptions arg1, string? arg2) + { + // todo: handle ttl changes + // todo: handle country code change + // todo: handle language change } #endregion @@ -119,9 +118,9 @@ public sealed class TitleDatabaseService : IHostedService var dict = await LoadFromDiskAsync() ?? new Dictionary(); // Set the new entry – the sliding expiration will now - // automatically move 30 s (or whatever you configured) forward + // automatically move 30 s (or whatever you configured) forward // every time the entry is accessed via Get/Set. - var titleInfoDtos = _cache.Set(CacheKey, dict, entryOptions); + _cache.Set(CacheKey, dict, entryOptions); _logger.LogInformation("Title DB reloaded – {Count} items cached (TTL={TTL}s).", dict.Count, ttlSec); } @@ -158,19 +157,6 @@ public sealed class TitleDatabaseService : IHostedService return dto; } - /*public async Task AddOrUpdateAsync(string titleId, TitleInfoDto dto) - { - var all = await GetAllAsync(); // slides the entry - all[id] = dto; - await PersistAsync(all); // writes to disk & triggers rebuild - } - - public async Task RemoveAsync(string titleId) - { - var all = await GetAllAsync(); // slides the entry - if (all.Remove(id)) - await PersistAsync(all); // writes to disk & triggers rebuild - }*/ /* ---------------------------------------------------------------- */ /* 3️⃣ Persist to disk & notify snapshot service /* ---------------------------------------------------------------- */ @@ -184,7 +170,19 @@ public sealed class TitleDatabaseService : IHostedService /* ---------------------------------------------------------------- */ /* 4️⃣ Dispose /* ---------------------------------------------------------------- */ - public void Dispose() => _snapshotService.SnapshotRebuilt -= (_, _) => ReloadCacheAsync(); + public void Dispose() => _snapshotService.SnapshotRebuilt -= SnapshotServiceOnSnapshotRebuilt; + + private async void SnapshotServiceOnSnapshotRebuilt(object? o, EventArgs eventArgs) + { + try + { + await ReloadCacheAsync(); + } + catch (Exception e) + { + _logger.LogCritical(e, "Failed to reload title database cache"); + } + } #region Public API @@ -240,6 +238,7 @@ public sealed class TitleDatabaseService : IHostedService /// /// Read the JSON file and populate _titleData. + /// Also remap titleId for XCI files based on cnmts.json /// private async Task> ReadTitleDbAsync(string filePath, CancellationToken ct) { diff --git a/TinfoilVibeServer/TinfoilVibeServer.csproj b/TinfoilVibeServer/TinfoilVibeServer.csproj index 5740f29..cb50206 100644 --- a/TinfoilVibeServer/TinfoilVibeServer.csproj +++ b/TinfoilVibeServer/TinfoilVibeServer.csproj @@ -10,8 +10,10 @@ + + diff --git a/TinfoilVibeServer/Utilities/SeekableBufferedStream.cs b/TinfoilVibeServer/Utilities/SeekableBufferedStream.cs new file mode 100644 index 0000000..e942ca8 --- /dev/null +++ b/TinfoilVibeServer/Utilities/SeekableBufferedStream.cs @@ -0,0 +1,251 @@ +using System.Buffers; + +namespace TinfoilVibeServer.Utilities; + +/// +/// A read‑only, seekable wrapper around a non‑seekable stream. +/// It buffers the source data on demand in chunks so that you can seek +/// back and forth without reading the whole source at once. +/// +public sealed class SeekableBufferedStream : Stream +{ + private const int DefaultChunkSize = 128 * 1024 * 1024; // 128 MiB + + private readonly Stream _source; + private readonly ArrayPool _pool; + private readonly int _chunkSize; + private readonly bool _disposeSource; + + // Buffer block – holds a rented byte[] and the number of bytes actually read. + private readonly struct BufferBlock + { + public readonly byte[] Data; + public readonly int Length; + public BufferBlock(byte[] data, int length) { Data = data; Length = length; } + } + + private readonly List _blocks = new(); + private readonly long _specifiedLength = 0; + private long _bufferedLength; // total number of bytes buffered so far + private long _position; // current logical position in the stream + private bool _eof; // true when the source stream has been exhausted + + #region ctor / dispose + + /// + /// Creates a new instance. + /// + /// The underlying source stream. Must be readable. + /// Length of underlying stream if known before using + /// Size of each buffer chunk (bytes). 128 MiB by default. + /// If true, disposing this wrapper will also dispose the source stream. + public SeekableBufferedStream(Stream source, long specifiedLength = 0, int chunkSize = DefaultChunkSize, bool disposeSource = false) + { + if (source == null) throw new ArgumentNullException(nameof(source)); + if (!source.CanRead) throw new ArgumentException("Source stream must be readable.", nameof(source)); + if (chunkSize <= 0) throw new ArgumentOutOfRangeException(nameof(chunkSize), "Chunk size must be positive."); + if (specifiedLength <= 0) throw new ArgumentOutOfRangeException(nameof(specifiedLength), "Specified length must be positive."); + + _source = source; + _specifiedLength = specifiedLength; + _pool = ArrayPool.Shared; + _chunkSize = chunkSize; + _disposeSource = disposeSource; + } + + protected override void Dispose(bool disposing) + { + if (disposing) + { + foreach (var block in _blocks) + _pool.Return(block.Data, clearArray: true); + _blocks.Clear(); + + if (_disposeSource) + _source.Dispose(); + } + base.Dispose(disposing); + } + + #endregion + + #region helpers + + /// + /// Ensures that at least bytes are buffered. + /// Reads from the source stream until the requested offset is reached or EOF is hit. + /// + private void EnsureBuffered(long requiredOffset) + { + if (_eof || _bufferedLength >= requiredOffset) + return; + + while (_bufferedLength < requiredOffset && !_eof) + { + var buf = _pool.Rent(_chunkSize); + int read = _source.Read(buf, 0, _chunkSize); + if (read == 0) + { + _eof = true; + _pool.Return(buf, clearArray: true); + break; + } + + _blocks.Add(new BufferBlock(buf, read)); + _bufferedLength += read; + } + } + + /// + /// Finds the block that contains and the offset inside that block. + /// + private void GetBlockAndOffset(long pos, out int blockIndex, out int offsetInBlock) + { + long accumulated = 0; + for (int i = 0; i < _blocks.Count; i++) + { + int blockLen = _blocks[i].Length; + if (pos < accumulated + blockLen) + { + blockIndex = i; + offsetInBlock = (int)(pos - accumulated); + return; + } + accumulated += blockLen; + } + + // This should never happen because we always call EnsureBuffered before accessing. + throw new InvalidOperationException("Requested position is outside buffered range."); + } + + #endregion + + #region Stream overrides + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + public override long Length + { + get + { + // If we were given a length, we can return that. + if (_specifiedLength > 0) return _specifiedLength; + + // If we already hit EOF, we know the length. + if (_eof) return _bufferedLength; + + // If the underlying stream is seekable, we can ask it directly. + if (_source.CanSeek) + return _source.Length; + + // Otherwise we need to drain the source to discover its length. + while (!_eof) + EnsureBuffered(_bufferedLength + _chunkSize); + return _bufferedLength; + } + } + + public override long Position + { + get => _position; + set + { + if (value < 0) throw new ArgumentOutOfRangeException(nameof(value)); + if (value > Length) throw new ArgumentOutOfRangeException(nameof(value)); + _position = value; + } + } + + public override int Read(byte[] buffer, int offset, int count) + { + if (buffer == null) throw new ArgumentNullException(nameof(buffer)); + if (offset < 0 || count < 0 || offset + count > buffer.Length) + throw new ArgumentOutOfRangeException(); + + // If we are already at or beyond the logical end, nothing to read. + if (_position >= Length) + return 0; + + // We will read at most `count` bytes but not past the logical end. + long maxRead = Math.Min(count, Length - _position); + EnsureBuffered(_position + maxRead); + + int bytesRead = 0; + while (bytesRead < maxRead) + { + GetBlockAndOffset(_position, out int blockIdx, out int blockOffset); + var block = _blocks[blockIdx]; + int available = block.Length - blockOffset; + int toCopy = (int)Math.Min(available, maxRead - bytesRead); + + Buffer.BlockCopy(block.Data, blockOffset, buffer, offset + bytesRead, toCopy); + + _position += toCopy; + bytesRead += toCopy; + } + + return bytesRead; + } + + public override long Seek(long offset, SeekOrigin origin) + { + long newPos = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => _position + offset, + SeekOrigin.End => Length + offset, + _ => throw new ArgumentException("Invalid SeekOrigin", nameof(origin)) + }; + + if (newPos < 0) throw new IOException("Attempted to seek before the beginning of the stream."); + + // Make sure we have buffered data up to the new position. + EnsureBuffered(newPos); + _position = newPos; + return _position; + } + + public override void SetLength(long value) => throw new NotSupportedException(); + + public override void Flush() { /* No-op – read‑only stream */ } + + public override void Write(byte[] buffer, int offset, int count) => + throw new NotSupportedException(); + + public override void WriteByte(byte value) => throw new NotSupportedException(); + + #endregion + + #region async helpers (optional) + + public override async ValueTask ReadAsync(Memory destination, CancellationToken cancellationToken = default) + { + // If we are already at or beyond the logical end, nothing to read. + if (_position >= Length) + return 0; + + long maxRead = Math.Min(destination.Length, Length - _position); + EnsureBuffered(_position + maxRead); + + int bytesRead = 0; + while (bytesRead < maxRead) + { + GetBlockAndOffset(_position, out int blockIdx, out int blockOffset); + var block = _blocks[blockIdx]; + int available = block.Length - blockOffset; + int toCopy = (int)Math.Min(available, maxRead - bytesRead); + + // We copy synchronously – no async source involved + destination.Slice(bytesRead, toCopy).Span + .CopyTo(block.Data.AsSpan(blockOffset, toCopy)); + + _position += toCopy; + bytesRead += toCopy; + } + + return bytesRead; + } + + #endregion +} \ No newline at end of file diff --git a/TinfoilVibeServer/appsettings.json b/TinfoilVibeServer/appsettings.json index da1fc0d..7e1e8ea 100644 --- a/TinfoilVibeServer/appsettings.json +++ b/TinfoilVibeServer/appsettings.json @@ -13,8 +13,8 @@ "BlacklistFile": "blacklist.json", "MaxFailedAttempts": 5, "Snapshot" : { - "RootDirectories": [ "\\\\NAS\\Roms\\Switch", "Z:\\imgs\\roms\\Switch" ], - "WhitelistExtensions": [ ".bin", ".jpg", ".png", ".txt" ], + "RootDirectories": [ "Z:\\downloads\\roms\\switch", "Z:\\imgs\\roms\\Switch" ], + "ArchiveExtensions": [ ".zip", ".rar", ".7z" ], "RomExtensions": [ ".xci", ".nsp", ".xcz" ], "CacheTtl": 60, "SnapshotFile": "index.tfl", @@ -23,7 +23,7 @@ "TitleDb": { "CountryCode": "AU", "Language": "en", - "TtlSeconds" : 30, + "TtlSeconds" : 90, "SnapshotFile" : "snapshot.json" }, diff --git a/TinfoilVibeServerTest/Tests/SnapshotServiceTests.cs b/TinfoilVibeServerTest/Tests/SnapshotServiceTests.cs index 913bbc6..1ff2273 100644 --- a/TinfoilVibeServerTest/Tests/SnapshotServiceTests.cs +++ b/TinfoilVibeServerTest/Tests/SnapshotServiceTests.cs @@ -2,6 +2,7 @@ using System.IO; using System.Threading.Tasks; using LibHac.Ncm; +using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using Moq; @@ -21,6 +22,7 @@ namespace TinfoilVibeServerTest.Tests private Mock _archiveHander; private Mock> _mockOptions; private SnapshotOptions _options; + private MemoryCache _memoryCache; [SetUp] public void SetUp() @@ -42,69 +44,92 @@ namespace TinfoilVibeServerTest.Tests _loggerMock = new Mock>(); _archiveHander = new Mock(); _nspExtractorMock = new Mock(); + var memoryCacheOptions = Options.Create(new MemoryCacheOptions()); + _memoryCache = new MemoryCache(memoryCacheOptions); + + _nspExtractorMock.Setup(extractor => extractor.ExtractHashFromStream(It.IsAny())).Returns("HASH"); _nspExtractorMock.Setup(extractor => extractor.ExtractFromStream(It.IsAny())).Returns( new NcaMetadataWithHash(titleId: "0000000000000000","0000000000000000", version: 1, ContentMetaType.Application, "HASH")); //Settings.RootDirs = new List { "TestData/Root1", "TestData/Root2" }; - _service = new SnapshotService(_mockOptions.Object, _nspExtractorMock.Object, _archiveHander.Object, _loggerMock.Object); + _service = new SnapshotService(_memoryCache, _mockOptions.Object, _nspExtractorMock.Object, _archiveHander.Object, _loggerMock.Object); } [TearDown] public void TearDown() { _service.Dispose(); + _memoryCache.Dispose(); } [Test] public async Task BuildSnapshot_WhenFilesChanged_ShouldPersist() { // Arrange - await File.WriteAllTextAsync(_options.SnapshotFile, "[]"); var initialHash = _service.GetSnapshot()?.Hash; - // Add a file to Root1 - var newFile = Path.Combine(_options.RootDirectories.First(), "new.nsp"); - FileSystemExtensions.EnsureDirectoryExists(Path.GetDirectoryName(newFile)); - // Create a new valid NSP file - // copy to temp to touch modified date - foreach (var file in Directory.GetFiles("../../../Data/")) + var rebuilding = false; + var rebuilt = false; + CancellationTokenSource snapshotRebuilding = new(); + _service.SnapshotRebuilding += (sender, args) => { - var filename = Path.GetFileName(file); - var destFilename = Path.Combine(Path.GetTempPath(), filename); - File.Copy(file, destFilename, true); - var info = new FileInfo(destFilename) - { - LastWriteTimeUtc = DateTime.UtcNow - }; - info.CopyTo(Path.Combine(_options.RootDirectories.First(),filename), true); - } - - // Act + rebuilding = true; + snapshotRebuilding.Cancel(); + }; + CancellationTokenSource snapshotPersisting = new(); _service.SnapshotRebuilt+= (sender, args) => { + rebuilt = true; + snapshotPersisting.Cancel(); // Assert var newHash = _service.GetSnapshot()?.Hash; Assert.That(newHash, Is.Not.EqualTo(initialHash)); - - }; - Task.Delay(300).Wait(); - _loggerMock.Verify( - l => l.Log( - LogLevel.Information, - It.IsAny(), - It.Is((v, t) => v.ToString().Contains("Snapshot rebuilt")), - null, - It.IsAny>()), Times.Once); + Timer timer = new(state => + { + snapshotPersisting.Cancel(); + snapshotRebuilding.Cancel(); + }, null, 20*1000, 0); + await File.WriteAllTextAsync(_options.SnapshotFile, "[]", snapshotPersisting.Token); + // Add a file to Root1 + var newFile = Path.Combine(_options.RootDirectories.First(), "new.nsp"); + + // Act + await File.WriteAllTextAsync(newFile,"TEST"); + + Task.Delay(4000).Wait(); + try + { + while (_memoryCache.Count > 0) + { + Task.Delay(200).Wait(snapshotRebuilding.Token); + } + } + catch (OperationCanceledException) + { + Assert.That(rebuilding, Is.True); + } + + try + { + while (_memoryCache.Count > 0) + { + Task.Delay(200).Wait(snapshotPersisting.Token); + } + } + catch (OperationCanceledException e) + { + Assert.That(rebuilt, Is.True); + } } [Test] public async Task BuildSnapshot_NoChange_ShouldNotPersist() { // Act - _service.BuildSnapshot(); + _service.BuildSnapshotAsync(); // Act again – snapshot should be identical - _service.BuildSnapshot(); + _service.BuildSnapshotAsync(); // Assert _loggerMock.Verify(