using System.Collections.Concurrent; using System.Security.Cryptography; using System.Text.Json; using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Options; using TinfoilVibeServer.Models; using TinfoilVibeServer.Utilities; namespace TinfoilVibeServer.Services; public interface ISnapshotService { event EventHandler SnapshotRebuilt; // raised after a rebuild void RebuildSnapshot(); SnapshotService.ROMSnapshot GetSnapshot(); Task AddToSnapshotAsync(FileEntry entry); Task BuildSnapshotAsync(); void GetArchiveName(string titleId); char GetArchivePathSeparator(); } /// /// Keeps an in‑memory snapshot, watches the filesystem for changes, and /// only re‑processes a file if its hash changed. /// public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedService { #region FileSystemWatcher private readonly List _watchers = new(); #endregion private readonly SnapshotOptions _options; private readonly INSPExtractor _nspExtractor; private readonly IArchiveHandler _archiveHandler; private readonly ILogger _logger; private readonly string _jsonPath; private readonly string _snapshotPath; private readonly ConcurrentDictionary _cache = new(); private readonly ConcurrentDictionary _hashCache = new(); // Archive full path -> FileEntry.Path private readonly ConcurrentDictionary _archiveLookup = new(); // hash -> file size private readonly ConcurrentDictionary _sizeLookup = new(); private readonly IMemoryCache _debouncerCache; public event EventHandler? SnapshotRebuilt; public event EventHandler? SnapshotRebuilding; private readonly SemaphoreSlim _snapshotFileSemaphore = new(1,1); private const char ArchivePathSeparator = '|'; public char GetArchivePathSeparator() => ArchivePathSeparator; public SnapshotService( IMemoryCache debouncerCache, IOptionsMonitor options, INSPExtractor nspExtractor, IArchiveHandler archiveHandler, ILogger logger) { _options = options.CurrentValue; _debouncerCache = debouncerCache; _nspExtractor = nspExtractor; _archiveHandler = archiveHandler; _logger = logger; _jsonPath = Path.Combine(AppContext.BaseDirectory, _options.SnapshotFile); // Debounce timer for persisting snapshot long debounceTime = 200; var entryOptions = new MemoryCacheEntryOptions() .SetSlidingExpiration(TimeSpan.FromSeconds(debounceTime)).RegisterPostEvictionCallback((key, value, reason, state) => { _logger.LogInformation("Should persist the snapshot {Key}, {Reason}", key, reason); }); // <‑‑ sliding! FileSystemExtensions.EnsureDirectoryExists(Path.GetFullPath(Path.GetDirectoryName(_jsonPath))); if (!File.Exists(_jsonPath)) { _snapshotFileSemaphore.Wait(); File.WriteAllText(_jsonPath, "[]"); _snapshotFileSemaphore.Release(); } _snapshotPath = Path.Combine(AppContext.BaseDirectory, _options.SnapshotBackupFile); FileSystemExtensions.EnsureDirectoryExists(Path.GetFullPath(Path.GetDirectoryName(_snapshotPath))); // 1️⃣ Register for *property* changes _options.PropertyChanged += (s, e) => OnOptionsChanged(e.PropertyName); foreach (var path in _options.RootDirectories) { AddWatchDirectory(path); } } // --------- Private helpers --------- private void OnOptionsChanged(string propertyName) { if (propertyName == nameof(SnapshotOptions.RootDirectories)) { var fileSystemWatchers = _watchers.Where(watcher => !_options.RootDirectories.Contains(watcher.Path)); foreach (var watcher in fileSystemWatchers) { watcher.EnableRaisingEvents = false; watcher.Dispose(); _watchers.Remove(watcher); } var newWatchedDirectories = _options.RootDirectories.Where(newWatchedDirectory => !_watchers.Any(watcher => string.Equals(watcher.Path, newWatchedDirectory, StringComparison.OrdinalIgnoreCase))); foreach (var newWatchedDirectory in newWatchedDirectories) { AddWatchDirectory(newWatchedDirectory); } BuildSnapshotAsync(); // rebuild everything PersistSnapshotAsync(); } } #region FileSystemWatcher private void AddWatchDirectory(string path) { if (!Directory.Exists(path)) return; var watcher = new FileSystemWatcher { Path = path, IncludeSubdirectories = true, NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.Size | NotifyFilters.LastWrite }; watcher.Created += OnChanged; watcher.Changed += OnChanged; watcher.Deleted += OnChanged; watcher.Renamed += OnRenamed; watcher.EnableRaisingEvents = true; _logger.LogInformation("Watching {Path}", path); _watchers.Add(watcher); } private void RemoveWatchDirectory(string path) { var fileSystemWatchers = _watchers.FirstOrDefault(watcher => watcher.Path == path); if (fileSystemWatchers == null) return; fileSystemWatchers.EnableRaisingEvents = false; fileSystemWatchers.Dispose(); _logger.LogInformation("Stopped watching {Path}", path); _watchers.Remove(fileSystemWatchers); } private void OnChanged(object? _, FileSystemEventArgs e) => ThrottleSnapshotUpdate(e); private void OnRenamed(object? _, RenamedEventArgs e) => ThrottleSnapshotUpdate(e); private void ThrottleSnapshotUpdate(FileSystemEventArgs fileSystemEventArgs) { SnapshotRebuilding?.Invoke(this, fileSystemEventArgs); using var cacheEntry = _debouncerCache.CreateEntry(fileSystemEventArgs.FullPath) //.SetAbsoluteExpiration(TimeSpan.FromMilliseconds(DebounceMs)) .SetValue(fileSystemEventArgs) .SetOptions(new MemoryCacheEntryOptions { PostEvictionCallbacks = { new PostEvictionCallbackRegistration { EvictionCallback = (key, value, reason, state) => { if (reason != EvictionReason.Expired) return; if (value is FileSystemEventArgs args) { if (IsFileLocked(args.FullPath)) { _logger.LogInformation("File {FilePath} is locked, skipping snapshot update", args.FullPath); using var rebounce = _debouncerCache.CreateEntry(args.FullPath) .SetAbsoluteExpiration(TimeSpan.FromMilliseconds(DebounceMs)) .SetValue(args); } } RebuildSnapshot(); } } } }); cacheEntry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMilliseconds(DebounceMs); _logger.LogDebug("File system event {EventType} on {Path} at {Time}", fileSystemEventArgs.ChangeType, fileSystemEventArgs.FullPath, DateTime.Now.ToString("HH:mm:ss")); } private static bool IsFileLocked(string filePath) { FileStream? stream = null; var file = new FileInfo(filePath); try { stream = file.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.None); } catch (IOException) { return true; } finally { stream?.Close(); } return false; } private const int DebounceMs = 400; private readonly JsonSerializerOptions _jsonSerializerOptions = new() { IncludeFields = true }; private int SnapshotFileLockTimeout { get; } = 1000; private void DebounceElapsed() { UpdateSnapshot(); } #endregion #region Snapshot logic public Task AddToSnapshotAsync(FileEntry entry) { // Update lookup tables _cache[entry.Path] = new SnapshotEntry(entry.Path, entry.Hash, entry.Size, entry.Titles); _hashCache[entry.Hash] = entry.Path; _sizeLookup[entry.Hash] = entry.Size; if (entry.Path.Contains(ArchivePathSeparator)) { var filename = entry.Path.Split(ArchivePathSeparator)[0]; _archiveLookup[filename] = entry.Path; } foreach (var ncaMetadataWithHash in entry.Titles) { _hashCache[ncaMetadataWithHash.Hash] = entry.Path; _sizeLookup[ncaMetadataWithHash.Hash] = entry.Size; _logger.LogInformation("Added entry {titleId} to snapshot (hash={hash})", ncaMetadataWithHash.TitleId, ncaMetadataWithHash.Hash); } // Persist snapshot to disk PersistSnapshotAsync(); return Task.CompletedTask; } /// Builds _cache and _hashCache based on directory configuration public Task BuildSnapshotAsync() { _logger.LogInformation("Building snapshot"); var index = LoadSnapshotIndex(); var latestModifiedUtcParallel = FileSystemExtensions.GetLatestModifiedUtcParallel(_options.RootDirectories); var fileInfo = new FileInfo(_snapshotPath); bool snapshotVerified = true; if (latestModifiedUtcParallel.HasValue && latestModifiedUtcParallel.Value < fileInfo.LastWriteTimeUtc) { if (index.Count != 0) { // directory may have been added with older roms, verify that the snapshot is still up to date foreach (var dir in _options.RootDirectories) { // check first entry is in index var entry = BuildSnapshot(dir).FirstOrDefault(); if (entry != null) { if (!index.TryGetValue(entry.Path, out var cached)) { snapshotVerified = false; _logger.LogInformation("Snapshot does not contain first entry in directory {Directory}", dir); } } } if (snapshotVerified) { _logger.LogInformation("Snapshot is up to date"); return Task.CompletedTask; } } else { _logger.LogInformation("Snapshot is up to date but index is empty"); } } _logger.LogInformation("Rebuilding snapshot (root dirs: {Count})", _options.RootDirectories.Count); var entries = new List(); var snapshotChanged = false; foreach (var dir in _options.RootDirectories) { _ = Task.Run(() => { _logger.LogInformation("Rebuilding directory {Directory}", dir); var buildSnapshot = BuildSnapshot(dir); var fileEntries = buildSnapshot.ToList(); snapshotChanged = snapshotChanged || fileEntries.Count != 0; entries.AddRange(fileEntries.Where(entry => entry != null)!); }); } // Replace the entire snapshot ComputeSnapshotHash(entries); if (snapshotChanged) { _logger.LogInformation("Snapshot rebuilt"); SnapshotRebuilt?.Invoke(this, EventArgs.Empty); } return Task.CompletedTask; } public void GetArchiveName(string titleId) { ; } // Returns List of FileEntry that do not have a hash in the cache // Each entry that has not been added to the lookup table is added to the cache private IEnumerable BuildSnapshot(string dir) { FileEntry entry; if (!Directory.Exists(dir)) yield break; foreach (var file in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories)) { string hash = string.Empty; var ext = Path.GetExtension(file).ToLowerInvariant(); if (!(_options.ArchiveExtensions.Contains(ext) || _options.RomExtensions.Contains(ext))) continue; if (_cache.ContainsKey(file) || _hashCache.ContainsKey(hash)) { continue; } // 3) extract title if applicable var titles = new List<(string, long, NcaMetadataWithHash)>(); if (_options.RomExtensions.Contains(ext)) { using var nspStream = File.OpenRead(file); hash = ComputeFirstStreamHash(nspStream); if (_hashCache.ContainsKey(hash)) { continue; } var nspStreamLength = nspStream.Length; var title = _nspExtractor.ExtractFromStream(nspStream); if (title != null) { var archiveEntry = new FileEntry(file, nspStreamLength, hash, [title]); AddToSnapshotAsync(archiveEntry); titles.Add((title.TitleId, nspStreamLength, title)); yield return archiveEntry; } } else { if (_options.ArchiveExtensions.Contains(ext)) { if (_archiveLookup.ContainsKey(file)) continue; hash = ComputeFirstStreamHash(file); if (_hashCache.ContainsKey(hash)) { yield return null; } IEnumerable<(string, long, NcaMetadataWithHash)>? titlesEnumerable = null; try { titlesEnumerable = _archiveHandler.TryExtractTitleInfos(file); } catch (Exception e) { _logger.LogError(e, "Failed to extract title info from archive {Archive}", file); } if (titlesEnumerable == null) continue; titles = titlesEnumerable.ToList(); foreach (var title in titles) { var archiveEntry = new FileEntry(file + ArchivePathSeparator + title.Item1, title.Item2, title.Item3.Hash, [title.Item3]); AddToSnapshotAsync(archiveEntry); yield return archiveEntry; } /*var fileEntry = new FileEntry(file, new FileInfo(file).Length, hash, titles.Select((tuple, i) => tuple.Item3).ToList()); AddToSnapshotAsync(fileEntry); yield return fileEntry;*/ } else { continue; } } if (titles.Count == 0) { _logger.LogInformation("Failed to process {File}", file); } else { _logger.LogInformation("Added {File} to snapshot (hash={Hash})", file, hash); yield return new FileEntry(file, titles.Select((tuple, i) => tuple.Item2).FirstOrDefault(), hash, titles.Select((tuple, i) => tuple.Item3).ToList()); } } } private string ComputeFirstStreamHash(Stream nspStream) { return _nspExtractor.ExtractHashFromStream(nspStream); } private void UpdateSnapshot() => BuildSnapshotAsync(); IEnumerable GetEntries() { foreach (var snapshotEntry in _cache) { _sizeLookup.TryGetValue(snapshotEntry.Value.Hash, out var size); var fileEntry = new FileEntry(snapshotEntry.Key, snapshotEntry.Value.Size, snapshotEntry.Value.Hash, snapshotEntry.Value.NcaMetadataWithHash); yield return fileEntry; } } private Task PersistSnapshotAsync() { if (_debouncerCache.TryGetValue(_jsonPath, out var value)) { _logger.LogInformation("Sliding debounce in progress, skipping snapshot persistence"); return Task.CompletedTask; } var snapshot = GetSnapshot(); var entries = GetEntries(); var fileEntries = entries.ToList(); var newHash = ComputeSnapshotHash(fileEntries); if (snapshot.Hash == newHash) return Task.CompletedTask; _logger.LogInformation("Snapshot hash changed – persisting new snapshot"); using var debouncedPersistence = _debouncerCache.CreateEntry(_jsonPath); debouncedPersistence.SlidingExpiration = TimeSpan.FromMilliseconds(DebounceMs); debouncedPersistence.Value = fileEntries; debouncedPersistence.PostEvictionCallbacks.Add(new PostEvictionCallbackRegistration { EvictionCallback = (key, entriesCallback, reason, state) => { if (entriesCallback is IEnumerable entriesToPersist && key is string filePath) { if (_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout)) { if (IsFileLocked(filePath)) { _logger.LogInformation("File {FilePath} is locked, skipping snapshot persistence", filePath); } else { File.WriteAllText(filePath, JsonSerializer.Serialize(entriesToPersist, _jsonSerializerOptions)); _snapshotFileSemaphore.Release(); _logger.LogInformation("Persisted snapshot"); SnapshotRebuilt?.Invoke(this, EventArgs.Empty); } } else { _logger.LogInformation("Failed to persist file {FilePath} due to timeout", filePath); } } } }); return Task.CompletedTask; } private static string ComputeHash(string filePath) { using var sha = SHA256.Create(); using var stream = File.OpenRead(filePath); var hash = sha.ComputeHash(stream); return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant(); } private static string ComputeSnapshotHash(IEnumerable entries) { var json = JsonSerializer.Serialize(entries); using var sha = SHA256.Create(); var hash = sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(json)); return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant(); } /// /// From filesystem cache, load each entry and build the lookups /// /// private Dictionary LoadSnapshotIndex() { if (!File.Exists(_jsonPath)) return new Dictionary(); _snapshotFileSemaphore.Wait(); var json = File.ReadAllText(_jsonPath); _snapshotFileSemaphore.Release(); var entries = JsonSerializer.Deserialize>(json, _jsonSerializerOptions)!; try { var fileEntries = new Dictionary(); // Reindex the cache foreach (var fileEntry in entries) { if (_hashCache.TryGetValue(fileEntry.Hash, out var value)) { _logger.LogWarning("Duplicate hash found in snapshot: {Hash}, {OldPath}, {newPath}", fileEntry.Hash, value, fileEntry.Path); } if (_options.RomExtensions.Contains(Path.GetExtension(fileEntry.Path))) { if (fileEntry.Path.Contains(ArchivePathSeparator)) { var filename = fileEntry.Path.Split(ArchivePathSeparator)[0]; _cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileEntry.Titles); _archiveLookup[filename] = fileEntry.Path; } else { _cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileEntry.Titles); fileEntries.TryAdd(fileEntry.Path, fileEntry); _hashCache[fileEntry.Hash] = fileEntry.Path; // ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract if (fileEntry.Titles == null) continue; foreach (var ncaMetadataWithHash in fileEntry.Titles) { _hashCache[ncaMetadataWithHash.Hash] = fileEntry.Path; } } } } return fileEntries; } catch (ArgumentException e) { _logger.LogError(e, "Failed to load snapshot"); return new(); } } public void RebuildSnapshot() { // 1️⃣ Flush the old in‑memory snapshot _cache.Clear(); _hashCache.Clear(); _archiveLookup.Clear(); _sizeLookup.Clear(); //_failedAttempts.Clear(); // if you keep per‑user counters // 2️⃣ Re‑build from disk again BuildSnapshotAsync().Wait(); // synchronous – we already own the lock PersistSnapshotAsync().Wait(); // same SnapshotRebuilt?.Invoke(this, EventArgs.Empty); } #endregion public ROMSnapshot GetSnapshot() { if (!File.Exists(_jsonPath)) return new ROMSnapshot(); if (_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout)) { try { var json = File.ReadAllText(_jsonPath); var hash = ComputeHash(_jsonPath); var romSnapshot = new ROMSnapshot { Hash = hash, Files = JsonSerializer.Deserialize>(json, _jsonSerializerOptions)! }; return romSnapshot; } catch (Exception e) { _logger.LogError(e, "Failed to load snapshot"); } finally { _snapshotFileSemaphore.Release(); } } else { _logger.LogWarning("Failed to load snapshot due to timeout"); } return new ROMSnapshot(); } public void Dispose() { foreach (var watcher in _watchers) { watcher.Dispose(); } } private sealed record SnapshotEntry(string Path, string Hash, long Size, List NcaMetadataWithHash); // File: TinfoilVibeServer/Services/SnapshotService.cs (inside SnapshotService class) private string ComputeFirstStreamHash(string filePath) { // Only treat NSP/XCI/XCZ as “first‑stream” files var ext = Path.GetExtension(filePath).ToLowerInvariant(); if (ext is not ".nsp" and not ".xci" and not ".xcz") { // Open the NSP/XCI with LibHac and read the first stream. // The first stream is the first entry returned by GetContentInfos(). try { using var reader = new RomArchiveReader(filePath); var first = reader.GetEntries().FirstOrDefault(); if (first == null) return ComputeFullHash(filePath); using var firstStream = first.Stream; var hash = _nspExtractor.ExtractHashFromStream(firstStream); return hash; } catch { // On error, fall back to the full file hash using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); var ncaMetadataWithHash = _nspExtractor.ExtractFromStream(fs); return ncaMetadataWithHash?.Hash ?? string.Empty; } } else { using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read); var ncaMetadataWithHash = _nspExtractor.ExtractFromStream(fs); return ncaMetadataWithHash?.Hash ?? string.Empty; } } private static string ComputeFullHash(string filePath) { using var sha256 = SHA256.Create(); using var stream = File.OpenRead(filePath); var hash = sha256.ComputeHash(stream); return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant(); } public class ROMSnapshot { public string? Hash { get; set; } public IReadOnlyList Files { get; set; } = new List(); } public async Task StartAsync(CancellationToken cancellationToken) { _logger.LogInformation("Starting snapshot service"); _ = Task.Run(async () => { await BuildSnapshotAsync(); await PersistSnapshotAsync(); }, cancellationToken); // initial scan new Timer(_ => DebounceElapsed(), null, Timeout.Infinite, Timeout.Infinite); } public Task StopAsync(CancellationToken cancellationToken) { Dispose(); return Task.CompletedTask; } }