diff --git a/TinfoilVibeServer.sln.DotSettings.user b/TinfoilVibeServer.sln.DotSettings.user index e06bf7e..36c18d7 100644 --- a/TinfoilVibeServer.sln.DotSettings.user +++ b/TinfoilVibeServer.sln.DotSettings.user @@ -87,7 +87,10 @@ ForceIncluded ForceIncluded ForceIncluded + ForceIncluded + ForceIncluded ForceIncluded + ForceIncluded ForceIncluded ForceIncluded diff --git a/TinfoilVibeServer/Models/FileEntry.cs b/TinfoilVibeServer/Models/FileEntry.cs index d73f050..1c22ec2 100644 --- a/TinfoilVibeServer/Models/FileEntry.cs +++ b/TinfoilVibeServer/Models/FileEntry.cs @@ -8,6 +8,6 @@ namespace TinfoilVibeServer.Models; public sealed record FileEntry( string Path, // nsp or archive path long Size, // size of nsp or full archive - string Hash, // SHA‑256 hex of first NCA of first NCP in NSP or archive + string? Hash, // SHA‑256 hex of first NCA of first NCP in NSP or archive List Titles // Details of all NSP Roms in the Path ); \ No newline at end of file diff --git a/TinfoilVibeServer/Models/NcaMetadataWithHash.cs b/TinfoilVibeServer/Models/NcaMetadataWithHash.cs new file mode 100644 index 0000000..12550ba --- /dev/null +++ b/TinfoilVibeServer/Models/NcaMetadataWithHash.cs @@ -0,0 +1,26 @@ +using LibHac.Ncm; + +namespace TinfoilVibeServer.Models; + +/// +/// DTO returned by the extractor – contains all data the snapshot needs. +/// +public sealed class NcaMetadataWithHash +{ + public string TitleId { get; } + public string ApplicationTitle { get; set; } + public int Version { get; } + + public ContentMetaType ContentMetaType { get; set; } + public string? Hash { get; } + + public NcaMetadataWithHash(string titleId, string applicationTitle, int version, + ContentMetaType contentMetaType, string? hash = null) + { + TitleId = titleId; + ApplicationTitle = applicationTitle; + Version = version; + ContentMetaType = contentMetaType; + Hash = hash; + } +} \ No newline at end of file diff --git a/TinfoilVibeServer/Services/IndexBuilderService.cs b/TinfoilVibeServer/Services/IndexBuilderService.cs index c00b2b3..e56b97e 100644 --- a/TinfoilVibeServer/Services/IndexBuilderService.cs +++ b/TinfoilVibeServer/Services/IndexBuilderService.cs @@ -119,9 +119,11 @@ public sealed class IndexBuilderService: IHostedService var fileName =Uri.EscapeDataString($"{name}[{titleId}][v{versionNumberParsed}][{patchOrApp}].nsp"); var url = $"{baseUri.ToString().TrimEnd('/')}/{fileName}"; - if (Uri.IsWellFormedUriString(url, UriKind.Absolute)) + var isWellFormed = Uri.TryCreate(url, UriKind.Absolute, out Uri? parsedUri); + + if (isWellFormed && parsedUri != null) { - fileDtos.Add(new FileDto(url, e.Size)); + fileDtos.Add(new FileDto(parsedUri.AbsoluteUri, e.Size)); } else { diff --git a/TinfoilVibeServer/Services/NSPExtractor.cs b/TinfoilVibeServer/Services/NSPExtractor.cs index d972519..9343680 100644 --- a/TinfoilVibeServer/Services/NSPExtractor.cs +++ b/TinfoilVibeServer/Services/NSPExtractor.cs @@ -1,5 +1,4 @@ -using System.ComponentModel.DataAnnotations; -using System.Security.Cryptography; +using System.Security.Cryptography; using LibHac.Common; using LibHac.Fs; using LibHac.Fs.Fsa; @@ -11,6 +10,7 @@ using LibHac.Ncm; using LibHac.Tools.Fs; using LibHac.Tools.Ncm; using Microsoft.Extensions.Options; +using TinfoilVibeServer.Models; using Path = System.IO.Path; namespace TinfoilVibeServer.Services @@ -322,27 +322,4 @@ namespace TinfoilVibeServer.Services { public string? KeyFile { get; set; } } - - /// - /// DTO returned by the extractor – contains all data the snapshot needs. - /// - public sealed class NcaMetadataWithHash - { - public string TitleId { get; } - public string ApplicationTitle { get; set; } - public int Version { get; } - - public ContentMetaType ContentMetaType { get; set; } - public string Hash { get; } - - public NcaMetadataWithHash(string titleId, string applicationTitle, int version, - ContentMetaType contentMetaType, string hash) - { - TitleId = titleId; - ApplicationTitle = applicationTitle; - Version = version; - ContentMetaType = contentMetaType; - Hash = hash; - } - } } \ No newline at end of file diff --git a/TinfoilVibeServer/Services/SnapshotService.cs b/TinfoilVibeServer/Services/SnapshotService.cs index 1801df0..81e400d 100644 --- a/TinfoilVibeServer/Services/SnapshotService.cs +++ b/TinfoilVibeServer/Services/SnapshotService.cs @@ -79,7 +79,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ FileSystemExtensions.EnsureDirectoryExists(Path.GetFullPath(Path.GetDirectoryName(_snapshotPath) ?? throw new InvalidOperationException())); // 1️⃣ Register for *property* changes - options.OnChange(snapshotOptions => + options.OnChange((snapshotOptions, arg) => { _options.RootDirectories = snapshotOptions.RootDirectories; }); @@ -256,61 +256,40 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ var index = LoadSnapshotIndex(); var latestModifiedUtcParallel = FileSystemExtensions.GetLatestModifiedUtcParallel(_options.RootDirectories); var fileInfo = new FileInfo(_snapshotPath); - bool snapshotVerified = true; + bool snapshotVerified = fileInfo.Exists; if (latestModifiedUtcParallel.HasValue && latestModifiedUtcParallel.Value < fileInfo.LastWriteTimeUtc) { if (index.Count != 0) { - // directory may have been added with older roms, verify that the snapshot is still up to date foreach (var dir in _options.RootDirectories) { - // check first entry is in index - var entry = BuildSnapshot(dir).FirstOrDefault(); - if (entry != null) + var firstEntry = BuildSnapshot(dir).FirstOrDefault(); + if (firstEntry != null && !index.TryGetValue(firstEntry.Path, out _)) { - if (!index.TryGetValue(entry.Path, out var cached)) - { - snapshotVerified = false; - _logger.LogInformation("Snapshot does not contain first entry in directory {Directory}", dir); - } + snapshotVerified = false; + _logger.LogInformation("Snapshot does not contain first entry in directory {Directory}", dir); } } + } + } - if (snapshotVerified) + if (!snapshotVerified) + { + _logger.LogInformation("Rebuilding snapshot (root dirs: {Count})", _options.RootDirectories.Count); + var entries = new List(); + foreach (var dir in _options.RootDirectories) + { + foreach (var entry in BuildSnapshot(dir)) { - _logger.LogInformation("Snapshot is up to date"); - return Task.CompletedTask; + if (entry != null) entries.Add(entry); } } - else - { - _logger.LogInformation("Snapshot is up to date but index is empty"); - } - } - _logger.LogInformation("Rebuilding snapshot (root dirs: {Count})", _options.RootDirectories.Count); - var entries = new List(); - - var snapshotChanged = false; - foreach (var dir in _options.RootDirectories) - { - _ = Task.Run(() => - { - _logger.LogInformation("Rebuilding directory {Directory}", dir); - var buildSnapshot = BuildSnapshot(dir); - var fileEntries = buildSnapshot.ToList(); - snapshotChanged = snapshotChanged || fileEntries.Count != 0; - entries.AddRange(fileEntries.Where(entry => entry != null)!); - }); + var currentHash = ComputeSnapshotHash(entries); + if (entries.Count > 0 || fileInfo.Exists && index.Count == 0) + SnapshotRebuilt?.Invoke(this, EventArgs.Empty); } - var snapshotEmptied = fileInfo.Exists && index.Count == 0 && _options.RootDirectories.Count == 0; - // Replace the entire snapshot - var currentSnapshotHash = ComputeSnapshotHash(entries); - if (snapshotChanged || snapshotEmptied) - { - _logger.LogInformation("Snapshot rebuilt"); - SnapshotRebuilt?.Invoke(this, EventArgs.Empty); - } + PersistSnapshotAsync(); return Task.CompletedTask; } @@ -341,6 +320,15 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ var titles = new List<(string, long, NcaMetadataWithHash)>(); if (_options.RomExtensions.Contains(ext)) { + var fileInfo = new FileInfo(file); + var ncaMetadataWithHash = fileInfo.GetNcaMetadataWithHash(); + if (ncaMetadataWithHash != null) + { + //var titleInfo = _titleDatabaseService.GetAsync(ncaMetadataWithHash.TitleId).Result; + var fileEntryFromFileName = new FileEntry(file, fileInfo.Length, ncaMetadataWithHash.Hash, [ncaMetadataWithHash]); + AddToSnapshotAsync(fileEntryFromFileName); + yield return fileEntryFromFileName; + } using var nspStream = File.OpenRead(file); hash = ComputeFirstStreamHash(nspStream); @@ -416,74 +404,71 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ await Task.CompletedTask; } - private string ComputeFirstStreamHash(Stream nspStream) - { - return _nspExtractor.ExtractHashFromStream(nspStream); - } + private string ComputeFirstStreamHash(Stream nspStream) => _nspExtractor.ExtractHashFromStream(nspStream); private void UpdateSnapshot() => BuildSnapshotAsync(); - IEnumerable GetEntries() + private IEnumerable GetEntries() { - foreach (var snapshotEntry in _cache) - { - _sizeLookup.TryGetValue(snapshotEntry.Value.Hash, out var size); - var fileEntry = new FileEntry(snapshotEntry.Key, snapshotEntry.Value.Size, snapshotEntry.Value.Hash, snapshotEntry.Value.NcaMetadataWithHash); - yield return fileEntry; - } + foreach (var kv in _cache) + yield return new FileEntry(kv.Key, kv.Value.Size, kv.Value.Hash, kv.Value.NcaMetadataWithHash); } + private Task PersistSnapshotAsync() { - if (_debouncerCache.TryGetValue(_jsonPath, out var value)) + if (_debouncerCache.TryGetValue(_jsonPath, out _)) { _logger.LogInformation("Sliding debounce in progress, skipping snapshot persistence"); return Task.CompletedTask; - } + } + + var entries = GetEntries().ToList(); + var newHash = ComputeSnapshotHash(entries); var snapshot = GetSnapshot(); - var entries = GetEntries(); - var fileEntries = entries.ToList(); - var newHash = ComputeSnapshotHash(fileEntries); if (snapshot.Hash == newHash) return Task.CompletedTask; - - CancellationTokenSource cts = new(); - _logger.LogInformation("Snapshot hash changed – persisting new snapshot"); - using var debouncedPersistence = _debouncerCache.CreateEntry(_jsonPath); - debouncedPersistence.AddExpirationToken(new CancellationChangeToken(cts.Token)); - //debouncedPersistence.AbsoluteExpirationRelativeToNow = TimeSpan.FromMilliseconds(DebounceMs); - debouncedPersistence.Value = fileEntries; - debouncedPersistence.PostEvictionCallbacks.Add(new PostEvictionCallbackRegistration - { - EvictionCallback = (key, entriesCallback, reason, state) => + + var cancellationTokenSource = new CancellationTokenSource(); + using var cacheEntry = _debouncerCache.CreateEntry(_jsonPath) + .AddExpirationToken(new CancellationChangeToken(cancellationTokenSource.Token)) + .SetValue(entries) + .SetOptions(new MemoryCacheEntryOptions { - if (entriesCallback is IEnumerable entriesToPersist && key is string filePath) + PostEvictionCallbacks = { - if (_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout)) + new PostEvictionCallbackRegistration { - if (IsFileLocked(filePath)) + EvictionCallback = (key, value, reason, state) => { - _logger.LogInformation("File {FilePath} is locked, skipping snapshot persistence", filePath); + if (!(reason == EvictionReason.Expired || reason == EvictionReason.TokenExpired)) + return; + var filePath = (string)key; + if (_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout)) + { + try + { + if (IsFileLocked(filePath)) + { + _logger.LogInformation("File {FilePath} is locked, skipping snapshot persistence", filePath); + } + else + { + File.WriteAllText(filePath, JsonSerializer.Serialize(value, _jsonSerializerOptions)); + SnapshotRebuilt?.Invoke(this, EventArgs.Empty); + } + } + finally + { + _snapshotFileSemaphore.Release(); + } + } } - else - { - File.WriteAllText(filePath, - JsonSerializer.Serialize(entriesToPersist, _jsonSerializerOptions)); - _snapshotFileSemaphore.Release(); - _logger.LogInformation("Persisted snapshot"); - SnapshotRebuilt?.Invoke(this, EventArgs.Empty); - } - } - else - { - _logger.LogInformation("Failed to persist file {FilePath} due to timeout", filePath); } } - } - }); - cts.CancelAfter(TimeSpan.FromMilliseconds(DebounceMs)); + }); + cancellationTokenSource.CancelAfter(TimeSpan.FromMilliseconds(DebounceMs)); return Task.CompletedTask; } - private static string ComputeHash(string filePath) { using var sha = SHA256.Create(); @@ -523,7 +508,8 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ _logger.LogWarning("Duplicate hash found in snapshot: {Hash}, {OldPath}, {newPath}", fileEntry.Hash, value, fileEntry.Path); } - if (!File.Exists(fileEntry.Path)) + var nspOrArchivePath = fileEntry.Path.Split(ArchivePathSeparator)[0]; + if (!File.Exists(nspOrArchivePath)) { _logger.LogWarning("Nonexistent entry found: {Path}", fileEntry.Path); continue; diff --git a/TinfoilVibeServer/Models/IdHelper.cs b/TinfoilVibeServer/Utilities/IdHelper.cs similarity index 98% rename from TinfoilVibeServer/Models/IdHelper.cs rename to TinfoilVibeServer/Utilities/IdHelper.cs index ced038d..f3b1122 100644 --- a/TinfoilVibeServer/Models/IdHelper.cs +++ b/TinfoilVibeServer/Utilities/IdHelper.cs @@ -1,7 +1,7 @@ using System.Globalization; using System.Text.RegularExpressions; -namespace TinfoilVibeServer.Models; +namespace TinfoilVibeServer.Utilities; public static class IdHelper diff --git a/TinfoilVibeServer/Utilities/NcaMetadataWithHashHelper.cs b/TinfoilVibeServer/Utilities/NcaMetadataWithHashHelper.cs new file mode 100644 index 0000000..4b072c3 --- /dev/null +++ b/TinfoilVibeServer/Utilities/NcaMetadataWithHashHelper.cs @@ -0,0 +1,30 @@ +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using LibHac.Ncm; +using TinfoilVibeServer.Models; + +namespace TinfoilVibeServer.Utilities; + +public static class NcaMetadataWithHashHelper +{ + public static NcaMetadataWithHash? GetNcaMetadataWithHash(this FileInfo fileInfo) + { + var match = Regex.Match(fileInfo.Name, @"^(.+)\[(\w{16})\]\[v(\d{1,7})\]\[(\w+).*\]\.nsp$"); + if (!match.Success) return null; + var titleId = match.Groups[2].Value; + var applicationTitle = match.Groups[1].Value.Trim(); + var version = int.Parse(match.Groups[3].Value) / 0x10000; + var nspType = match.Groups[4].Value.ToLowerInvariant() switch + { + "base" => ContentMetaType.Application, + "update" => ContentMetaType.Patch, + "dlc" => ContentMetaType.AddOnContent, + _ => ContentMetaType.Patch + }; + var bytes = Encoding.UTF8.GetBytes(fileInfo.FullName); + var hash = SHA256.HashData(bytes); + + return new NcaMetadataWithHash(titleId, applicationTitle, version, nspType, Convert.ToBase64String(hash)); + } +} \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index 96f3a47..4a6b91e 100644 --- a/compose.yaml +++ b/compose.yaml @@ -8,6 +8,7 @@ services: image: gitea.ecenshu.net/ecenshu/tinfoilvibeserver:latest container_name: tinfoilvibeserver restart: unless-stopped + user: "1000:1000" env_file: - .env environment: