956 lines
39 KiB
C#
956 lines
39 KiB
C#
using System;
|
||
using System.Collections.Concurrent;
|
||
using System.Collections.Generic;
|
||
using System.Diagnostics;
|
||
using System.IO;
|
||
using System.Linq;
|
||
using System.Security.Cryptography;
|
||
using System.Text.Json;
|
||
using System.Threading;
|
||
using System.Threading.Tasks;
|
||
using Microsoft.Extensions.Caching.Memory;
|
||
using Microsoft.Extensions.Hosting;
|
||
using Microsoft.Extensions.Logging;
|
||
using Microsoft.Extensions.Options;
|
||
using Microsoft.Extensions.Primitives;
|
||
using TinfoilVibeServer.Models;
|
||
using TinfoilVibeServer.Utilities;
|
||
|
||
namespace TinfoilVibeServer.Services;
|
||
public interface ISnapshotService
|
||
{
|
||
event EventHandler SnapshotRebuilt; // event raised after a rebuild
|
||
void RebuildSnapshot();
|
||
Task RebuildSnapshotAsync(CancellationToken cancellationToken = default);
|
||
SnapshotService.ROMSnapshot GetSnapshot();
|
||
|
||
Task AddToSnapshotAsync(FileEntry entry, CancellationToken cancellationToken = default);
|
||
Task BuildSnapshotAsync(CancellationToken cancellationToken = default);
|
||
void GetArchiveName(string titleId);
|
||
char GetArchivePathSeparator();
|
||
void Start();
|
||
void Stop();
|
||
}
|
||
|
||
/// <summary>
|
||
/// Watches a folder for changes and rebuilds a snapshot when the first change after a debounce window occurs.
|
||
/// While a rebuild is in progress, subsequent file changes are ignored (they will be processed once the current
|
||
/// rebuild finishes and a new debounce window starts).
|
||
/// only re‑processes a file if its hash changed.
|
||
/// </summary>
|
||
public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedService
|
||
{
|
||
#region FileSystemWatcher
|
||
|
||
/* ==============================================================
|
||
* 1️⃣ FileSystemWatcher
|
||
* ============================================================== */
|
||
private readonly List<FileSystemWatcher> _watchers = [];
|
||
|
||
#endregion
|
||
|
||
#region Snapshot options & helpers
|
||
/* ==============================================================
|
||
* 2️⃣ Snapshot options & helpers
|
||
* ============================================================== */
|
||
private readonly SnapshotOptions _options;
|
||
private readonly INSPExtractor _nspExtractor;
|
||
private readonly IArchiveHandler _archiveHandler;
|
||
private readonly ILogger<SnapshotService> _logger;
|
||
private readonly IHostEnvironment _environment;
|
||
private readonly string _jsonPath;
|
||
private readonly string _snapshotPath;
|
||
private readonly ConcurrentDictionary<string, SnapshotEntry> _cache = new();
|
||
|
||
private readonly ConcurrentDictionary<string, string> _hashCache = new();
|
||
|
||
// Archive full path -> FileEntry.Path
|
||
private readonly ConcurrentDictionary<string, string> _archiveLookup = new();
|
||
|
||
// hash -> file size
|
||
private readonly ConcurrentDictionary<string, long> _sizeLookup = new();
|
||
private readonly IMemoryCache _debouncerCache;
|
||
public event EventHandler? SnapshotRebuilt;
|
||
public event EventHandler? SnapshotRebuilding;
|
||
|
||
private readonly SemaphoreSlim _snapshotFileSemaphore = new(1, 1);
|
||
|
||
private const char ArchivePathSeparator = '|';
|
||
|
||
// Cache key used to keep the debounce flag
|
||
private const string DebounceKey = "SnapshotService.IsDebouncing";
|
||
private const string BuildKey = "SnapshotService.IsBuilding";
|
||
private CancellationTokenSource _cancellation = new();
|
||
private Task? _currentBuildTask;
|
||
|
||
public char GetArchivePathSeparator() => ArchivePathSeparator;
|
||
#endregion
|
||
|
||
/* ==============================================================
|
||
* 3️⃣ Build‑time guard
|
||
* ============================================================== */
|
||
/// <summary>
|
||
/// Allows only one rebuild at a time.
|
||
/// </summary>
|
||
private readonly SemaphoreSlim _buildLock = new(1, 1);
|
||
|
||
/* ==============================================================
|
||
* 4️⃣ Constructor
|
||
* ============================================================== */
|
||
public SnapshotService(
|
||
IMemoryCache debouncerCache,
|
||
IOptionsMonitor<SnapshotOptions> options,
|
||
INSPExtractor nspExtractor,
|
||
IArchiveHandler archiveHandler,
|
||
ILogger<SnapshotService> logger,
|
||
IHostEnvironment environment
|
||
)
|
||
{
|
||
_options = options.CurrentValue;
|
||
_debouncerCache = debouncerCache;
|
||
_nspExtractor = nspExtractor;
|
||
_archiveHandler = archiveHandler;
|
||
_logger = logger;
|
||
_environment = environment;
|
||
_jsonPath = !Path.IsPathRooted(_options.SnapshotFile) ? Path.Combine(Path.DirectorySeparatorChar.ToString(), "app", "data", _options.SnapshotFile) : _options.SnapshotFile;
|
||
|
||
FileSystemExtensions.EnsureDirectoryExists(Path.GetFullPath(Path.GetDirectoryName(_jsonPath) ?? throw new InvalidOperationException()));
|
||
if (!File.Exists(_jsonPath))
|
||
{
|
||
_snapshotFileSemaphore.Wait();
|
||
File.WriteAllText(_jsonPath, "[]");
|
||
_snapshotFileSemaphore.Release();
|
||
}
|
||
|
||
_snapshotPath = !Path.IsPathRooted(_options.SnapshotBackupFile) ? Path.Combine(Path.DirectorySeparatorChar.ToString(), "app", "data", _options.SnapshotBackupFile) : _options.SnapshotBackupFile;
|
||
|
||
FileSystemExtensions.EnsureDirectoryExists(Path.GetFullPath(Path.GetDirectoryName(_snapshotPath) ?? throw new InvalidOperationException()));
|
||
|
||
// 1️⃣ Register for *property* changes
|
||
options.OnChange((snapshotOptions, _) => { _options.RootDirectories = snapshotOptions.RootDirectories; });
|
||
_options.PropertyChanged += (_, e) => OnOptionsChanged(e.PropertyName);
|
||
|
||
if (_options.RootDirectories.Count == 0)
|
||
{
|
||
_logger.LogInformation("No directories set to watch for ROMS/Archives");
|
||
}
|
||
|
||
foreach (var path in _options.RootDirectories)
|
||
{
|
||
AddWatchDirectory(path);
|
||
}
|
||
}
|
||
#region Public API
|
||
|
||
public void Start() => _watchers.ForEach(watcher => watcher.EnableRaisingEvents = true);
|
||
|
||
public void Stop()
|
||
{
|
||
foreach (var fileSystemWatcher in _watchers)
|
||
{
|
||
fileSystemWatcher.EnableRaisingEvents = false;
|
||
}
|
||
_cancellation.Cancel();
|
||
try { _currentBuildTask?.Wait(); }
|
||
catch
|
||
{
|
||
// ignored
|
||
}
|
||
}
|
||
|
||
#endregion
|
||
|
||
// --------- Private helpers ---------
|
||
private void OnOptionsChanged(string? propertyName)
|
||
{
|
||
if (propertyName != nameof(SnapshotOptions.RootDirectories)) return;
|
||
|
||
_logger.LogInformation("Root directories changed, rebuilding snapshot");
|
||
var fileSystemWatchers = _watchers.Where(watcher => !_options.RootDirectories.Contains(watcher.Path));
|
||
var systemWatchers = fileSystemWatchers.ToList();
|
||
foreach (var watcher in systemWatchers)
|
||
{
|
||
RemoveWatchDirectory(watcher.Path);
|
||
}
|
||
|
||
var newWatchedDirectories = _options.RootDirectories.Where(newWatchedDirectory =>
|
||
!_watchers.Any(watcher =>
|
||
string.Equals(watcher.Path, newWatchedDirectory, StringComparison.OrdinalIgnoreCase)));
|
||
|
||
foreach (var newWatchedDirectory in newWatchedDirectories)
|
||
{
|
||
AddWatchDirectory(newWatchedDirectory);
|
||
}
|
||
|
||
_ = BuildSnapshotAsync(_cancellation.Token); // rebuild everything
|
||
PersistSnapshotAsync(_cancellation.Token);
|
||
}
|
||
|
||
|
||
#region FileSystemWatcher helpers
|
||
|
||
/* ==============================================================
|
||
* 5️⃣ FileSystemWatcher helpers
|
||
* ============================================================== */
|
||
private void AddWatchDirectory(string path)
|
||
{
|
||
if (!Directory.Exists(path)) return;
|
||
var watcher = new FileSystemWatcher
|
||
{
|
||
Path = path,
|
||
IncludeSubdirectories = true,
|
||
NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName |
|
||
NotifyFilters.Size | NotifyFilters.LastWrite
|
||
};
|
||
watcher.Created += OnChanged;
|
||
watcher.Changed += OnChanged;
|
||
watcher.Deleted += OnChanged;
|
||
watcher.Renamed += OnRenamed;
|
||
watcher.EnableRaisingEvents = true;
|
||
_logger.LogInformation("Watching {Path}", path);
|
||
_watchers.Add(watcher);
|
||
}
|
||
|
||
private void RemoveWatchDirectory(string path)
|
||
{
|
||
var fileSystemWatchers = _watchers.FirstOrDefault(watcher => watcher.Path == path);
|
||
if (fileSystemWatchers == null) return;
|
||
fileSystemWatchers.EnableRaisingEvents = false;
|
||
fileSystemWatchers.Dispose();
|
||
_logger.LogInformation("Stopped watching {Path}", path);
|
||
_watchers.Remove(fileSystemWatchers);
|
||
}
|
||
|
||
private void OnChanged(object? _, FileSystemEventArgs e)
|
||
{
|
||
var fileInfo = new FileInfo(e.FullPath);
|
||
if (_options.ArchiveExtensions.Contains(fileInfo.Extension) || _options.RomExtensions.Contains(fileInfo.Extension))
|
||
{
|
||
ThrottleSnapshotUpdate(e);
|
||
}
|
||
}
|
||
|
||
private void OnRenamed(object? _, RenamedEventArgs e)
|
||
{
|
||
var fileInfo = new FileInfo(e.FullPath);
|
||
if (_options.ArchiveExtensions.Contains(fileInfo.Extension) || _options.RomExtensions.Contains(fileInfo.Extension))
|
||
{
|
||
ThrottleSnapshotUpdate(e);
|
||
}
|
||
}
|
||
|
||
/// <summary>
|
||
/// Rebuild the snapshot, if rebuild in process, cancel it and restart
|
||
/// </summary>
|
||
/// <param name="fileSystemEventArgs"></param>
|
||
private void ThrottleSnapshotUpdate(FileSystemEventArgs fileSystemEventArgs)
|
||
{
|
||
// If a rebuild is already underway, ignore the event
|
||
if (_currentBuildTask is { IsCompleted: false })
|
||
{
|
||
_logger.LogInformation(
|
||
"File system event {ChangeType} on {Path} triggered, but build is in progress, skipping snapshot update",
|
||
fileSystemEventArgs.ChangeType, fileSystemEventArgs.FullPath);
|
||
return;
|
||
}
|
||
|
||
// Schedule a rebuild only if we’re not already debouncing
|
||
if (_debouncerCache.TryGetValue(DebounceKey, out bool isDebouncing) && isDebouncing)
|
||
return;
|
||
|
||
// If a rebuild is in progress, ignore the event immediately
|
||
if (_buildLock.CurrentCount == 0) // lock held by a rebuild
|
||
{
|
||
_logger.LogInformation(
|
||
"File system event {ChangeType} on {Path} triggered, restart Build Task on next completed entry",
|
||
fileSystemEventArgs.ChangeType, fileSystemEventArgs.FullPath);
|
||
_cancellation.Cancel();
|
||
_buildLock.Wait();
|
||
_buildLock.Release();
|
||
_cancellation.Dispose();
|
||
_cancellation = new CancellationTokenSource();
|
||
}
|
||
|
||
CancellationTokenSource cts = new();
|
||
|
||
using var cacheEntry = _debouncerCache.CreateEntry(fileSystemEventArgs.FullPath)
|
||
.AddExpirationToken(new CancellationChangeToken(cts.Token))
|
||
.SetValue(fileSystemEventArgs)
|
||
.SetOptions(new MemoryCacheEntryOptions
|
||
{
|
||
PostEvictionCallbacks =
|
||
{
|
||
new PostEvictionCallbackRegistration
|
||
{
|
||
EvictionCallback =
|
||
(_, _, reason, _) =>
|
||
{
|
||
if (reason is not (EvictionReason.Expired or EvictionReason.TokenExpired)) return;
|
||
|
||
SnapshotRebuilding?.Invoke(this, fileSystemEventArgs);
|
||
// Kick off the rebuild asynchronously
|
||
_currentBuildTask = RebuildSnapshotAsync(_cancellation.Token);
|
||
}
|
||
}
|
||
}
|
||
});
|
||
cts.CancelAfter(TimeSpan.FromMilliseconds(DebounceMs));
|
||
|
||
_logger.LogDebug("File system event {EventType} on {Path} at {Time}", fileSystemEventArgs.ChangeType,
|
||
fileSystemEventArgs.FullPath, DateTime.Now.ToString("HH:mm:ss.fff"));
|
||
}
|
||
|
||
private const int DebounceMs = 400;
|
||
private readonly JsonSerializerOptions _jsonSerializerOptions = new() { IncludeFields = true };
|
||
private int SnapshotFileLockTimeout { get; } = 1000;
|
||
|
||
#endregion
|
||
|
||
#region Snapshot logic
|
||
|
||
public async Task AddToSnapshotAsync(FileEntry entry, CancellationToken cancellationToken = default)
|
||
{
|
||
// Update lookup tables
|
||
if (entry.Hash == null)
|
||
{
|
||
_logger.LogWarning("Cannot add entry {Path} to snapshot: no hash", entry.Path);
|
||
return;
|
||
}
|
||
|
||
var cacheUpdated = _cache.ContainsKey(entry.Path);
|
||
var hashMatch = _hashCache.ContainsKey(entry.Hash) && _hashCache[entry.Hash] == entry.Path;
|
||
if (cacheUpdated && hashMatch) return;
|
||
|
||
var lastModified = File.GetLastWriteTimeUtc(entry.Path.Contains(ArchivePathSeparator) ? entry.Path.Split(ArchivePathSeparator)[0] : entry.Path);
|
||
|
||
_cache[entry.Path] = new SnapshotEntry(entry.Path, entry.Hash, entry.Size, lastModified, entry.Titles);
|
||
_hashCache[entry.Hash] = entry.Path;
|
||
_sizeLookup[entry.Hash] = entry.Size;
|
||
|
||
if (entry.Path.Contains(ArchivePathSeparator))
|
||
{
|
||
var filename = entry.Path.Split(ArchivePathSeparator)[0];
|
||
_archiveLookup[filename] = entry.Path;
|
||
}
|
||
|
||
foreach (var ncaMetadataWithHash in entry.Titles)
|
||
{
|
||
if (ncaMetadataWithHash.Hash == null)
|
||
{
|
||
_logger.LogWarning("Cannot add entry {Path} to snapshot: no hash", entry.Path);
|
||
continue;
|
||
}
|
||
|
||
_hashCache[ncaMetadataWithHash.Hash] = entry.Path;
|
||
_sizeLookup[ncaMetadataWithHash.Hash] = entry.Size;
|
||
}
|
||
|
||
// Persist snapshot to disk
|
||
// If entry.Titles is null, treat it as an empty collection
|
||
var titleIds = string.Join(",", entry.Titles.Select(t => t.TitleId.ToString()));
|
||
_logger.LogInformation(cacheUpdated ? "Updated snapshot for {Path}, titleIds=[{TitleIds}]" : "Added {Path} to snapshot, titleIds=[{TitleIds}]", entry.Path, titleIds);
|
||
|
||
await PersistSnapshotAsync(cancellationToken);
|
||
}
|
||
|
||
/* ==============================================================
|
||
* 6️⃣ Snapshot build / persistence helpers
|
||
* ============================================================== */
|
||
/// Builds _cache and _hashCache based on directory configuration
|
||
/// <param name="cancellationToken"></param>
|
||
public async Task BuildSnapshotAsync(CancellationToken cancellationToken = default)
|
||
{
|
||
// Acquire the rebuild lock – if we cannot, skip this build.
|
||
if (!await _buildLock.WaitAsync(0, cancellationToken))
|
||
{
|
||
_logger.LogInformation("BuildSnapshotAsync called while rebuild in progress, ignoring.");
|
||
return;
|
||
}
|
||
|
||
try
|
||
{
|
||
_logger.LogInformation("Building snapshot");
|
||
var index = LoadSnapshotIndex();
|
||
var latestModifiedUtcParallel = FileSystemExtensions.GetLatestModifiedUtcParallel(_options.RootDirectories);
|
||
var fileInfo = new FileInfo(_snapshotPath);
|
||
var snapshotVerified = fileInfo.Exists;
|
||
if (latestModifiedUtcParallel.HasValue && latestModifiedUtcParallel.Value < fileInfo.LastWriteTimeUtc)
|
||
{
|
||
if (index.Count != 0)
|
||
{
|
||
foreach (var dir in _options.RootDirectories)
|
||
{
|
||
// Snapshot is older than the latest modified file in the directory
|
||
try
|
||
{
|
||
var lastOrDefault = BuildSnapshot(dir, cancellationToken).LastOrDefault();
|
||
if (lastOrDefault != null && !index.TryGetValue(lastOrDefault.Path, out _))
|
||
{
|
||
snapshotVerified = false;
|
||
_logger.LogInformation("Snapshot does not contain first entry in directory {Directory}", dir);
|
||
}
|
||
}
|
||
catch (OperationCanceledException operationCanceledException)
|
||
{
|
||
_logger.LogInformation("Build Cancelled while building snapshot from directory {Directory}: {Message}", dir, operationCanceledException.Message);
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
if (!snapshotVerified)
|
||
{
|
||
_logger.LogInformation("Rebuilding snapshot (root dirs: {Count})", _options.RootDirectories.Count);
|
||
var entries = new List<FileEntry>();
|
||
foreach (var dir in _options.RootDirectories)
|
||
{
|
||
foreach (var entry in BuildSnapshot(dir, cancellationToken))
|
||
{
|
||
if (entry != null) entries.Add(entry);
|
||
}
|
||
}
|
||
|
||
//var currentHash = ComputeSnapshotHash(entries);
|
||
if (entries.Count > 0 || fileInfo.Exists && index.Count == 0)
|
||
SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
|
||
}
|
||
|
||
await PersistSnapshotAsync(cancellationToken);
|
||
}
|
||
finally
|
||
{
|
||
_buildLock.Release();
|
||
}
|
||
}
|
||
|
||
public void GetArchiveName(string titleId)
|
||
{
|
||
|
||
}
|
||
|
||
// Returns List of FileEntry that do not have a hash in the cache
|
||
// Each entry that has not been added to the lookup table is added to the cache
|
||
private IEnumerable<FileEntry?> BuildSnapshot(string dir, CancellationToken cancellationToken = default)
|
||
{
|
||
var processedFiles = new HashSet<string>();
|
||
|
||
if (!Directory.Exists(dir)) yield break;
|
||
foreach (var file in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories).OrderBy(file =>
|
||
{
|
||
var fileInfo = new FileInfo(file);
|
||
return fileInfo.LastWriteTimeUtc;
|
||
}))
|
||
{
|
||
string? hash;
|
||
var ext = Path.GetExtension(file).ToLowerInvariant();
|
||
|
||
if (!(_options.ArchiveExtensions.Contains(ext) || _options.RomExtensions.Contains(ext)))
|
||
continue;
|
||
|
||
// 3) extract title if applicable
|
||
var titles = new List<(string, long, NcaMetadataWithHash)>();
|
||
if (_options.RomExtensions.Contains(ext))
|
||
{
|
||
var fileInfo = new FileInfo(file);
|
||
var ncaMetadataWithHash = fileInfo.GetNcaMetadataWithHash();
|
||
if (ncaMetadataWithHash != null)
|
||
{
|
||
//var titleInfo = _titleDatabaseService.GetAsync(ncaMetadataWithHash.TitleId).Result;
|
||
var fileEntryFromFileName = new FileEntry(file, fileInfo.Length, ncaMetadataWithHash.Hash, [ncaMetadataWithHash]);
|
||
var addToSnapshotAsync = AddToSnapshotAsync(fileEntryFromFileName, cancellationToken);
|
||
addToSnapshotAsync.Wait(cancellationToken);
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
yield return fileEntryFromFileName;
|
||
continue;
|
||
}
|
||
|
||
using var nspStream = File.OpenRead(file);
|
||
_logger.LogDebug("Extracting hash for {File}", file);
|
||
hash = ComputeFirstStreamHash(nspStream);
|
||
|
||
if (_hashCache.TryGetValue(hash, out var value) && file == _cache[value].Path)
|
||
{
|
||
continue;
|
||
}
|
||
|
||
var nspStreamLength = nspStream.Length;
|
||
var title = _nspExtractor.ExtractFromStream(nspStream);
|
||
if (title != null)
|
||
{
|
||
var romEntry = new FileEntry(file, nspStreamLength, hash, [title]);
|
||
var addToSnapshotAsync = AddToSnapshotAsync(romEntry, cancellationToken);
|
||
addToSnapshotAsync.Wait(cancellationToken);
|
||
titles.Add((title.TitleId, nspStreamLength, title));
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
yield return romEntry;
|
||
continue;
|
||
}
|
||
}
|
||
else
|
||
{
|
||
if (_options.ArchiveExtensions.Contains(ext))
|
||
{
|
||
if (_archiveLookup.ContainsKey(file)) continue;
|
||
if (processedFiles.Contains(file)) continue;
|
||
_logger.LogDebug("Extracting hash for {File}", file);
|
||
var stopwatch = Stopwatch.StartNew();
|
||
hash = ComputeFirstStreamHashAsync(file, cancellationToken).Result;
|
||
stopwatch.Stop();
|
||
if (!string.IsNullOrEmpty(hash))
|
||
{
|
||
_logger.LogDebug("Computed hash for {File} in {Time}ms", file, stopwatch.ElapsedMilliseconds);
|
||
if (_hashCache.TryGetValue(hash, out var value) && file == _cache[value].Path)
|
||
{
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
yield return null;
|
||
continue;
|
||
}
|
||
}
|
||
|
||
IEnumerable<(string, long, NcaMetadataWithHash)>? titlesEnumerable = null;
|
||
try
|
||
{
|
||
titlesEnumerable = TryExtractTitleInfosWithRetryAsync(file, cancellationToken).Result;
|
||
// if it was multipart, add multiparts to processedFiles
|
||
var directoryName = Path.GetDirectoryName(file);
|
||
if (directoryName != null)
|
||
{
|
||
var baseName = MultiPartRarHelper.GetBaseNameForRarVolume(Path.GetFileName(file));
|
||
var discoverVolumes = MultiPartRarHelper.DiscoverVolumes(directoryName, baseName);
|
||
if (discoverVolumes.Count > 1)
|
||
{
|
||
processedFiles.UnionWith(discoverVolumes);
|
||
}
|
||
}
|
||
}
|
||
catch (Exception e)
|
||
{
|
||
_logger.LogError(e, "Failed to extract title info from archive {Archive}", file);
|
||
}
|
||
|
||
if (titlesEnumerable == null) continue;
|
||
|
||
titles = titlesEnumerable.ToList();
|
||
foreach (var title in titles)
|
||
{
|
||
var archiveEntry = new FileEntry(file + ArchivePathSeparator + title.Item1, title.Item2, title.Item3.Hash, [title.Item3]);
|
||
var addToSnapshotAsync = AddToSnapshotAsync(archiveEntry, cancellationToken);
|
||
addToSnapshotAsync.Wait(cancellationToken);
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
yield return archiveEntry;
|
||
}
|
||
}
|
||
|
||
continue;
|
||
}
|
||
|
||
if (titles.Count == 0)
|
||
{
|
||
_logger.LogInformation("Failed to process {File}", file);
|
||
}
|
||
else
|
||
{
|
||
_logger.LogInformation("Added {File} to snapshot (hash={Hash})", file, hash);
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
yield return new FileEntry(file, titles.Select((tuple, _) => tuple.Item2).FirstOrDefault(), hash, titles.Select((tuple, _) => tuple.Item3).ToList());
|
||
}
|
||
}
|
||
}
|
||
|
||
private async Task<IEnumerable<(string, long, NcaMetadataWithHash)>?> TryExtractTitleInfosWithRetryAsync(string file, CancellationToken cancellationToken = default)
|
||
{
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
for (var attempt = 0; attempt < _options.MaxRetryCount; attempt++)
|
||
{
|
||
try
|
||
{
|
||
var stopwatch2 = Stopwatch.StartNew();
|
||
var titlesEnumerable = _archiveHandler.TryExtractTitleInfos(file);
|
||
stopwatch2.Stop();
|
||
_logger.LogDebug("Extracted title infos for {File} in {Time}ms", file, stopwatch2.ElapsedMilliseconds);
|
||
return titlesEnumerable;
|
||
}
|
||
catch (IOException ex) when (attempt < _options.MaxRetryCount - 1)
|
||
{
|
||
var delay = (int)((attempt + 1) * _options.DebounceTimeoutMs * _options.RetryMultiplier);
|
||
_logger.LogWarning(ex, "Failed to load {Path}. Attempt {Attempt}, Retrying after {Delay}.",
|
||
file, attempt + 1, delay);
|
||
await Task.Delay(delay, cancellationToken);
|
||
}
|
||
catch (IOException) when (attempt >= _options.MaxRetryCount - 1)
|
||
{
|
||
_logger.LogWarning("Load {Path} failed after {retries} attempts", file, attempt + 1);
|
||
}
|
||
}
|
||
return null;
|
||
}
|
||
|
||
private async Task ValidateSnapshotAsync(CancellationToken cancellationToken = default)
|
||
{
|
||
await Task.CompletedTask;
|
||
}
|
||
|
||
private string ComputeFirstStreamHash(Stream nspStream) => _nspExtractor.ExtractHashFromStream(nspStream);
|
||
|
||
private IEnumerable<FileEntry> GetEntries()
|
||
{
|
||
foreach (var kv in _cache.OrderByDescending(pair => pair.Value.LastModified))
|
||
yield return new FileEntry(kv.Key, kv.Value.Size, kv.Value.Hash, kv.Value.NcaMetadataWithHash);
|
||
}
|
||
|
||
private Task PersistSnapshotAsync(CancellationToken cancellationToken = default)
|
||
{
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
|
||
if (_debouncerCache.TryGetValue(_jsonPath, out _))
|
||
{
|
||
_logger.LogDebug("Sliding debounce in progress, skipping snapshot persistence");
|
||
return Task.CompletedTask;
|
||
}
|
||
|
||
var entries = GetEntries().ToList();
|
||
|
||
var newHash = ComputeSnapshotHash(entries);
|
||
var snapshot = GetSnapshot();
|
||
if (snapshot.Hash == newHash) return Task.CompletedTask;
|
||
|
||
var cancellationTokenSource = new CancellationTokenSource();
|
||
using var cacheEntry = _debouncerCache.CreateEntry(_jsonPath)
|
||
.AddExpirationToken(new CancellationChangeToken(cancellationTokenSource.Token))
|
||
.SetValue(entries)
|
||
.SetOptions(new MemoryCacheEntryOptions
|
||
{
|
||
PostEvictionCallbacks =
|
||
{
|
||
new PostEvictionCallbackRegistration
|
||
{
|
||
EvictionCallback = (key, value, reason, _) =>
|
||
{
|
||
if (reason is not (EvictionReason.Expired or EvictionReason.TokenExpired))
|
||
return;
|
||
var filePath = (string)key;
|
||
if (!_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout)) return;
|
||
|
||
try
|
||
{
|
||
if (FileLockHelper.IsFileLocked(filePath))
|
||
{
|
||
_logger.LogInformation("File {FilePath} is locked, skipping snapshot persistence", filePath);
|
||
}
|
||
else
|
||
{
|
||
File.WriteAllText(filePath, JsonSerializer.Serialize(value, _jsonSerializerOptions));
|
||
_logger.LogInformation("Persisted snapshot to {FilePath}", filePath);
|
||
SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
|
||
}
|
||
}
|
||
finally
|
||
{
|
||
_snapshotFileSemaphore.Release();
|
||
}
|
||
}
|
||
}
|
||
}
|
||
});
|
||
cancellationTokenSource.CancelAfter(TimeSpan.FromMilliseconds(DebounceMs));
|
||
return Task.CompletedTask;
|
||
}
|
||
|
||
private static string ComputeHash(string filePath)
|
||
{
|
||
using var sha = SHA256.Create();
|
||
using var stream = File.OpenRead(filePath);
|
||
var hash = SHA256.HashData(stream);
|
||
return Convert.ToHexStringLower(hash);
|
||
}
|
||
|
||
private static string ComputeSnapshotHash(IEnumerable<FileEntry> entries)
|
||
{
|
||
var json = JsonSerializer.Serialize(entries);
|
||
var hash = SHA256.HashData(System.Text.Encoding.UTF8.GetBytes(json));
|
||
return Convert.ToHexStringLower(hash);
|
||
}
|
||
|
||
/// <summary>
|
||
/// From filesystem cache, load each entry and build the lookups
|
||
/// Check for duplicate hashes
|
||
/// Check for nonexistent entries against filesystem
|
||
/// </summary>
|
||
/// <returns></returns>
|
||
private Dictionary<string, FileEntry> LoadSnapshotIndex()
|
||
{
|
||
if (!File.Exists(_jsonPath)) return new Dictionary<string, FileEntry>();
|
||
_snapshotFileSemaphore.Wait();
|
||
var json = File.ReadAllText(_jsonPath);
|
||
_snapshotFileSemaphore.Release();
|
||
var entries = JsonSerializer.Deserialize<List<FileEntry>>(json, _jsonSerializerOptions)!;
|
||
try
|
||
{
|
||
var fileEntries = new Dictionary<string, FileEntry>();
|
||
// Reindex the cache
|
||
foreach (var fileEntry in entries)
|
||
{
|
||
if (fileEntry.Hash == null)
|
||
{
|
||
_logger.LogError("Entry {Path} has no hash", fileEntry.Path);
|
||
continue;
|
||
}
|
||
|
||
if (_hashCache.TryGetValue(fileEntry.Hash, out var value))
|
||
{
|
||
_logger.LogWarning("Duplicate hash found in snapshot: {Hash}, {OldPath}, {newPath}", fileEntry.Hash, value, fileEntry.Path);
|
||
}
|
||
|
||
var nspOrArchivePath = fileEntry.Path.Split(ArchivePathSeparator)[0];
|
||
if (!File.Exists(nspOrArchivePath))
|
||
{
|
||
_logger.LogWarning("Nonexistent entry found: {Path}", fileEntry.Path);
|
||
continue;
|
||
}
|
||
|
||
var fileContainedInRootDirectories = false;
|
||
foreach (var optionsRootDirectory in _options.RootDirectories)
|
||
{
|
||
if (fileEntry.Path.StartsWith(optionsRootDirectory))
|
||
{
|
||
fileContainedInRootDirectories = true;
|
||
break;
|
||
}
|
||
}
|
||
|
||
if (!fileContainedInRootDirectories)
|
||
{
|
||
_logger.LogInformation("Entry {Path} is not contained in any root directory", fileEntry.Path);
|
||
continue;
|
||
}
|
||
|
||
if (_options.RomExtensions.Contains(Path.GetExtension(fileEntry.Path)))
|
||
{
|
||
if (fileEntry.Path.Contains(ArchivePathSeparator))
|
||
{
|
||
var fileInfo = new FileInfo(fileEntry.Path.Split(ArchivePathSeparator)[0]);
|
||
var filename = fileEntry.Path.Split(ArchivePathSeparator)[0];
|
||
// ReSharper disable once RedundantSuppressNullableWarningExpression
|
||
_cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileInfo.LastWriteTimeUtc, fileEntry.Titles!);
|
||
_archiveLookup[filename] = fileEntry.Path;
|
||
}
|
||
else
|
||
{
|
||
var fileInfo = new FileInfo(fileEntry.Path);
|
||
// ReSharper disable once RedundantSuppressNullableWarningExpression
|
||
_cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileInfo.LastWriteTimeUtc, fileEntry.Titles!);
|
||
fileEntries.TryAdd(fileEntry.Path, fileEntry);
|
||
_hashCache[fileEntry.Hash] = fileEntry.Path;
|
||
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
|
||
if (fileEntry.Titles == null) continue;
|
||
foreach (var ncaMetadataWithHash in fileEntry.Titles)
|
||
{
|
||
if (ncaMetadataWithHash.Hash == null) continue;
|
||
_hashCache[ncaMetadataWithHash.Hash] = fileEntry.Path;
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
_logger.LogInformation("Loaded snapshot index {Count} entries", fileEntries.Count);
|
||
return fileEntries;
|
||
}
|
||
catch (ArgumentException e)
|
||
{
|
||
_logger.LogError(e, "Failed to load snapshot");
|
||
return new();
|
||
}
|
||
}
|
||
|
||
public void RebuildSnapshot()
|
||
{
|
||
RebuildSnapshotAsync().Wait();
|
||
}
|
||
|
||
public async Task RebuildSnapshotAsync(CancellationToken cancellationToken = default)
|
||
{
|
||
// Fast path: if we already have the lock, just log and exit.
|
||
if (!await _buildLock.WaitAsync(0, cancellationToken))
|
||
{
|
||
_logger.LogInformation("RebuildSnapshot called while a rebuild is already in progress, ignoring.");
|
||
return;
|
||
}
|
||
|
||
try
|
||
{
|
||
// 1️⃣ Flush the old in‑memory snapshot
|
||
_cache.Clear();
|
||
_hashCache.Clear();
|
||
_archiveLookup.Clear();
|
||
_sizeLookup.Clear();
|
||
//_failedAttempts.Clear(); // if you keep per‑user counters
|
||
|
||
// 2️⃣ Re‑build from disk again
|
||
_buildLock.Release();
|
||
await BuildSnapshotAsync(cancellationToken);
|
||
await PersistSnapshotAsync(cancellationToken);
|
||
SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
|
||
}
|
||
finally
|
||
{
|
||
_buildLock.Release();
|
||
}
|
||
}
|
||
|
||
#endregion
|
||
|
||
public ROMSnapshot GetSnapshot()
|
||
{
|
||
if (!File.Exists(_jsonPath)) return new ROMSnapshot();
|
||
|
||
if (_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout))
|
||
{
|
||
try
|
||
{
|
||
var json = File.ReadAllText(_jsonPath);
|
||
var hash = ComputeHash(_jsonPath);
|
||
var romSnapshot = new ROMSnapshot
|
||
{
|
||
Hash = hash,
|
||
Files = JsonSerializer.Deserialize<IReadOnlyList<FileEntry>>(json, _jsonSerializerOptions)!
|
||
};
|
||
return romSnapshot;
|
||
}
|
||
catch (Exception e)
|
||
{
|
||
_logger.LogError(e, "Failed to load snapshot");
|
||
}
|
||
finally
|
||
{
|
||
_snapshotFileSemaphore.Release();
|
||
}
|
||
}
|
||
else
|
||
{
|
||
_logger.LogWarning("Failed to load snapshot due to timeout");
|
||
}
|
||
|
||
return new ROMSnapshot();
|
||
}
|
||
|
||
#region IDisposable
|
||
public void Dispose()
|
||
{
|
||
Stop();
|
||
foreach (var fileSystemWatcher in _watchers)
|
||
{
|
||
fileSystemWatcher.Dispose();
|
||
}
|
||
_cancellation.Dispose();
|
||
}
|
||
#endregion
|
||
|
||
/// <summary>
|
||
/// Represents a single ROM/archive entry in the snapshot cache.
|
||
/// </summary>
|
||
private sealed record SnapshotEntry(string Path, string Hash, long Size, DateTime LastModified, List<NcaMetadataWithHash> NcaMetadataWithHash);
|
||
|
||
// File: TinfoilVibeServer/Services/SnapshotService.cs (inside SnapshotService class)
|
||
|
||
private async Task<string?> ComputeFirstStreamHashAsync(string filePath, CancellationToken cancellationToken = default)
|
||
{
|
||
cancellationToken.ThrowIfCancellationRequested();
|
||
for (var attempt = 0; attempt < _options.MaxRetryCount; attempt++)
|
||
{
|
||
try
|
||
{
|
||
var ext = Path.GetExtension(filePath).ToLowerInvariant();
|
||
|
||
var multiPartBasePathWithExtension = $"{MultiPartRarHelper.GetBaseNameForRarVolume(filePath)}{ext}";
|
||
if (string.CompareOrdinal(multiPartBasePathWithExtension, filePath) != 0)
|
||
{
|
||
filePath = multiPartBasePathWithExtension;
|
||
}
|
||
|
||
if (FileLockHelper.IsFileLocked(filePath))
|
||
{
|
||
throw new IOException("File is locked");
|
||
}
|
||
|
||
// Only treat NSP/XCI/XCZ as “first‑stream” files
|
||
if (ext is not ".nsp" and not ".xci" and not ".xcz")
|
||
{
|
||
// Open the NSP/XCI with LibHac and read the first stream.
|
||
// The first stream is the first entry returned by GetContentInfos().
|
||
try
|
||
{
|
||
await using var reader = new RomArchiveReader(filePath);
|
||
var first = reader.GetEntries().FirstOrDefault();
|
||
if (first == null) return ComputeFullHash(filePath);
|
||
|
||
//using var seekableWrapper = new SeekableBufferedStream(first.Stream, first.Stream.Length, 10*1024*1024, true);
|
||
await using var rewindableWrapper = new RewindableStream(first.Stream, () => first.Stream, 10 * 1024 * 1024, first.Stream.Length);
|
||
var hash = _nspExtractor.ExtractHashFromStream(rewindableWrapper);
|
||
return hash;
|
||
}
|
||
catch
|
||
{
|
||
// ignored
|
||
}
|
||
}
|
||
|
||
await using var fs = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||
var ncaMetadataWithHash = _nspExtractor.ExtractFromStream(fs);
|
||
return ncaMetadataWithHash?.Hash ?? string.Empty;
|
||
}
|
||
catch (IOException ex) when (attempt < _options.MaxRetryCount - 1)
|
||
{
|
||
var delay = (int)((attempt + 1) * _options.DebounceTimeoutMs * _options.RetryMultiplier);
|
||
_logger.LogWarning(ex, "Failed to load {Path}. Attempt {Attempt}, Retrying after {Delay}.",
|
||
filePath, attempt + 1, delay);
|
||
await Task.Delay(delay, cancellationToken);
|
||
}
|
||
catch (IOException) when (attempt >= _options.MaxRetryCount - 1)
|
||
{
|
||
_logger.LogWarning("Load {Path} failed after {retries} attempts", filePath, attempt + 1);
|
||
return null;
|
||
}
|
||
catch (Exception exception)
|
||
{
|
||
_logger.LogError(exception, "Unknown exception: {exception}", exception.Message);
|
||
}
|
||
}
|
||
|
||
return string.Empty;
|
||
}
|
||
|
||
private static string ComputeFullHash(string filePath)
|
||
{
|
||
using var sha256 = SHA256.Create();
|
||
using var stream = File.OpenRead(filePath);
|
||
var hash = sha256.ComputeHash(stream);
|
||
return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
|
||
}
|
||
|
||
public class ROMSnapshot
|
||
{
|
||
public string? Hash { get; init; }
|
||
public IReadOnlyList<FileEntry> Files { get; init; } = new List<FileEntry>();
|
||
}
|
||
|
||
public async Task StartAsync(CancellationToken cancellationToken)
|
||
{
|
||
_logger.LogInformation("Starting snapshot service");
|
||
_ = Task.Run(async () =>
|
||
{
|
||
await ValidateSnapshotAsync(cancellationToken);
|
||
_currentBuildTask = BuildSnapshotAsync(_cancellation.Token);
|
||
await _currentBuildTask.WaitAsync(_cancellation.Token);
|
||
await PersistSnapshotAsync(_cancellation.Token);
|
||
}, cancellationToken); // initial scan
|
||
/*var timer = new Timer(_ => DebounceElapsed(), null, Timeout.Infinite, Timeout.Infinite);*/
|
||
await Task.CompletedTask;
|
||
}
|
||
|
||
public Task StopAsync(CancellationToken cancellationToken)
|
||
{
|
||
Dispose();
|
||
return Task.CompletedTask;
|
||
}
|
||
} |