Build Snapshot from archives
Download from archives Process XCI files in archives
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
using System.Collections.Concurrent;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Caching.Memory;
|
||||
using Microsoft.Extensions.Options;
|
||||
using TinfoilVibeServer.Models;
|
||||
using TinfoilVibeServer.Utilities;
|
||||
@@ -11,55 +12,81 @@ public interface ISnapshotService
|
||||
event EventHandler SnapshotRebuilt; // raised after a rebuild
|
||||
void RebuildSnapshot();
|
||||
SnapshotService.ROMSnapshot GetSnapshot();
|
||||
void BuildSnapshot();
|
||||
|
||||
Task AddToSnapshotAsync(FileEntry entry);
|
||||
Task BuildSnapshotAsync();
|
||||
void GetArchiveName(string titleId);
|
||||
char GetArchivePathSeparator();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Keeps an in‑memory snapshot, watches the filesystem for changes, and
|
||||
/// only re‑processes a file if its hash changed.
|
||||
/// </summary>
|
||||
public sealed class SnapshotService : IDisposable, ISnapshotService
|
||||
public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedService
|
||||
{
|
||||
#region FileSystemWatcher
|
||||
private readonly List<FileSystemWatcher> _watchers = new();
|
||||
#endregion
|
||||
|
||||
private readonly SnapshotOptions _options;
|
||||
private readonly INSPExtractor _nspExtractor;
|
||||
private readonly IArchiveHandler _archiveHandler;
|
||||
private readonly ILogger<SnapshotService> _logger;
|
||||
private readonly string _jsonPath;
|
||||
private readonly string _snapshotPath;
|
||||
private readonly List<FileSystemWatcher> _watchers = new();
|
||||
private readonly ConcurrentDictionary<string, CachedFile> _cache = new();
|
||||
private string? _currentSnapshotHash;
|
||||
private readonly Timer _debounceTimer;
|
||||
private readonly ConcurrentDictionary<string, SnapshotEntry> _cache = new();
|
||||
private readonly ConcurrentDictionary<string, string> _hashCache = new();
|
||||
// Archive full path -> FileEntry.Path
|
||||
private readonly ConcurrentDictionary<string, string> _archiveLookup = new();
|
||||
// hash -> file size
|
||||
private readonly ConcurrentDictionary<string, long> _sizeLookup = new();
|
||||
private readonly IMemoryCache _debouncerCache;
|
||||
public event EventHandler? SnapshotRebuilt;
|
||||
|
||||
public event EventHandler? SnapshotRebuilding;
|
||||
|
||||
private readonly SemaphoreSlim _snapshotFileSemaphore = new(1,1);
|
||||
private const char ArchivePathSeparator = '|';
|
||||
public char GetArchivePathSeparator() => ArchivePathSeparator;
|
||||
|
||||
public SnapshotService(
|
||||
IMemoryCache debouncerCache,
|
||||
IOptionsMonitor<SnapshotOptions> options,
|
||||
INSPExtractor nspExtractor,
|
||||
IArchiveHandler archiveHandler,
|
||||
ILogger<SnapshotService> logger)
|
||||
{
|
||||
_options = options.CurrentValue;
|
||||
_debouncerCache = debouncerCache;
|
||||
_nspExtractor = nspExtractor;
|
||||
_archiveHandler = archiveHandler;
|
||||
_logger = logger;
|
||||
_jsonPath = Path.Combine(AppContext.BaseDirectory, _options.SnapshotFile);
|
||||
FileSystemExtensions.EnsureDirectoryExists(Path.GetDirectoryName(_jsonPath));
|
||||
|
||||
// Debounce timer for persisting snapshot
|
||||
long debounceTime = 200;
|
||||
var entryOptions = new MemoryCacheEntryOptions()
|
||||
.SetSlidingExpiration(TimeSpan.FromSeconds(debounceTime)).RegisterPostEvictionCallback((key, value, reason,
|
||||
state) =>
|
||||
{
|
||||
|
||||
_logger.LogInformation("Should persist the snapshot {Key}, {Reason}", key, reason);
|
||||
}); // <‑‑ sliding!
|
||||
FileSystemExtensions.EnsureDirectoryExists(Path.GetFullPath(Path.GetDirectoryName(_jsonPath)));
|
||||
if (!File.Exists(_jsonPath))
|
||||
{
|
||||
_snapshotFileSemaphore.Wait();
|
||||
File.WriteAllText(_jsonPath, "[]");
|
||||
_snapshotFileSemaphore.Release();
|
||||
}
|
||||
_snapshotPath = Path.Combine(AppContext.BaseDirectory, _options.SnapshotBackupFile);
|
||||
FileSystemExtensions.EnsureDirectoryExists(Path.GetDirectoryName(_snapshotPath));
|
||||
FileSystemExtensions.EnsureDirectoryExists(Path.GetFullPath(Path.GetDirectoryName(_snapshotPath)));
|
||||
// 1️⃣ Register for *property* changes
|
||||
_options.PropertyChanged += (s, e) => OnOptionsChanged(e.PropertyName);
|
||||
|
||||
BuildSnapshot(); // initial scan
|
||||
File.WriteAllText(_snapshotPath, JsonSerializer.Serialize(GetSnapshot()));
|
||||
_debounceTimer = new Timer(_ => DebounceElapsed(), null, Timeout.Infinite, Timeout.Infinite);
|
||||
|
||||
foreach (var path in _options.RootDirectories)
|
||||
{
|
||||
InitializeFileSystemWatcher(path);
|
||||
AddWatchDirectory(path);
|
||||
}
|
||||
}
|
||||
// --------- Private helpers ---------
|
||||
@@ -81,15 +108,18 @@ public sealed class SnapshotService : IDisposable, ISnapshotService
|
||||
|
||||
foreach (var newWatchedDirectory in newWatchedDirectories)
|
||||
{
|
||||
InitializeFileSystemWatcher(newWatchedDirectory);
|
||||
AddWatchDirectory(newWatchedDirectory);
|
||||
|
||||
}
|
||||
|
||||
BuildSnapshot(); // rebuild everything
|
||||
PersistSnapshot();
|
||||
BuildSnapshotAsync(); // rebuild everything
|
||||
PersistSnapshotAsync();
|
||||
}
|
||||
}
|
||||
private void InitializeFileSystemWatcher(string path)
|
||||
|
||||
|
||||
#region FileSystemWatcher
|
||||
private void AddWatchDirectory(string path)
|
||||
{
|
||||
if (!Directory.Exists(path)) return;
|
||||
var watcher = new FileSystemWatcher
|
||||
@@ -104,32 +134,84 @@ public sealed class SnapshotService : IDisposable, ISnapshotService
|
||||
watcher.Deleted += OnChanged;
|
||||
watcher.Renamed += OnRenamed;
|
||||
watcher.EnableRaisingEvents = true;
|
||||
|
||||
_logger.LogInformation("Watching {Path}", path);
|
||||
_watchers.Add(watcher);
|
||||
}
|
||||
|
||||
#region FileSystemWatcher
|
||||
private void RemoveWatchDirectory(string path)
|
||||
{
|
||||
var fileSystemWatchers = _watchers.FirstOrDefault(watcher => watcher.Path == path);
|
||||
if (fileSystemWatchers == null) return;
|
||||
fileSystemWatchers.EnableRaisingEvents = false;
|
||||
fileSystemWatchers.Dispose();
|
||||
_logger.LogInformation("Stopped watching {Path}", path);
|
||||
_watchers.Remove(fileSystemWatchers);
|
||||
}
|
||||
|
||||
private void OnChanged(object? _, FileSystemEventArgs e) => ThrottleSnapshotUpdate(e);
|
||||
private void OnRenamed(object? _, RenamedEventArgs e) => ThrottleSnapshotUpdate(e);
|
||||
|
||||
private void ThrottleSnapshotUpdate(FileSystemEventArgs fileSystemEventArgs)
|
||||
{
|
||||
lock (_lock)
|
||||
{
|
||||
_debounceTimer.Change(_debounceMs, Timeout.Infinite); // reset the timer
|
||||
_logger.LogDebug("File system event {EventType} on {Path} at {Time}", fileSystemEventArgs.ChangeType, fileSystemEventArgs.FullPath, DateTime.Now.ToString("HH:mm:ss"));
|
||||
}
|
||||
/*Task.Run(async () =>
|
||||
{
|
||||
await Task.Delay(250);
|
||||
_logger.LogDebug("File system event {EventType} on {Path}", fileSystemEventArgs.ChangeType, fileSystemEventArgs.FullPath);
|
||||
UpdateSnapshot();
|
||||
});*/
|
||||
SnapshotRebuilding?.Invoke(this, fileSystemEventArgs);
|
||||
using var cacheEntry = _debouncerCache.CreateEntry(fileSystemEventArgs.FullPath)
|
||||
//.SetAbsoluteExpiration(TimeSpan.FromMilliseconds(DebounceMs))
|
||||
.SetValue(fileSystemEventArgs)
|
||||
.SetOptions(new MemoryCacheEntryOptions
|
||||
{
|
||||
PostEvictionCallbacks =
|
||||
{
|
||||
new PostEvictionCallbackRegistration
|
||||
{
|
||||
EvictionCallback =
|
||||
(key, value, reason, state) =>
|
||||
{
|
||||
if (reason != EvictionReason.Expired) return;
|
||||
|
||||
if (value is FileSystemEventArgs args)
|
||||
{
|
||||
if (IsFileLocked(args.FullPath))
|
||||
{
|
||||
_logger.LogInformation("File {FilePath} is locked, skipping snapshot update", args.FullPath);
|
||||
using var rebounce = _debouncerCache.CreateEntry(args.FullPath)
|
||||
.SetAbsoluteExpiration(TimeSpan.FromMilliseconds(DebounceMs))
|
||||
.SetValue(args);
|
||||
}
|
||||
}
|
||||
|
||||
RebuildSnapshot();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
cacheEntry.AbsoluteExpirationRelativeToNow = TimeSpan.FromMilliseconds(DebounceMs);
|
||||
|
||||
_logger.LogDebug("File system event {EventType} on {Path} at {Time}", fileSystemEventArgs.ChangeType,
|
||||
fileSystemEventArgs.FullPath, DateTime.Now.ToString("HH:mm:ss"));
|
||||
}
|
||||
|
||||
private readonly object _lock = new object();
|
||||
private int _debounceMs = 200;
|
||||
private static bool IsFileLocked(string filePath)
|
||||
{
|
||||
FileStream? stream = null;
|
||||
var file = new FileInfo(filePath);
|
||||
|
||||
try
|
||||
{
|
||||
stream = file.Open(FileMode.Open, FileAccess.ReadWrite, FileShare.None);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
stream?.Close();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private const int DebounceMs = 400;
|
||||
private readonly JsonSerializerOptions _jsonSerializerOptions = new() { IncludeFields = true };
|
||||
private int SnapshotFileLockTimeout { get; } = 1000;
|
||||
|
||||
private void DebounceElapsed()
|
||||
{
|
||||
@@ -140,15 +222,66 @@ public sealed class SnapshotService : IDisposable, ISnapshotService
|
||||
|
||||
#region Snapshot logic
|
||||
|
||||
public void BuildSnapshot()
|
||||
public Task AddToSnapshotAsync(FileEntry entry)
|
||||
{
|
||||
// Update lookup tables
|
||||
_cache[entry.Path] = new SnapshotEntry(entry.Path, entry.Hash, entry.Size, entry.Titles);
|
||||
_hashCache[entry.Hash] = entry.Path;
|
||||
_sizeLookup[entry.Hash] = entry.Size;
|
||||
if (entry.Path.Contains(ArchivePathSeparator))
|
||||
{
|
||||
var filename = entry.Path.Split(ArchivePathSeparator)[0];
|
||||
_archiveLookup[filename] = entry.Path;
|
||||
}
|
||||
|
||||
foreach (var ncaMetadataWithHash in entry.Titles)
|
||||
{
|
||||
_hashCache[ncaMetadataWithHash.Hash] = entry.Path;
|
||||
_sizeLookup[ncaMetadataWithHash.Hash] = entry.Size;
|
||||
_logger.LogInformation("Added entry {titleId} to snapshot (hash={hash})", ncaMetadataWithHash.TitleId, ncaMetadataWithHash.Hash);
|
||||
}
|
||||
// Persist snapshot to disk
|
||||
PersistSnapshotAsync();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
/// Builds _cache and _hashCache based on directory configuration
|
||||
public Task BuildSnapshotAsync()
|
||||
{
|
||||
_logger.LogInformation("Building snapshot");
|
||||
var index = LoadSnapshotIndex();
|
||||
var latestModifiedUtcParallel = FileSystemExtensions.GetLatestModifiedUtcParallel(_options.RootDirectories);
|
||||
var fileInfo = new FileInfo(_snapshotPath);
|
||||
bool snapshotVerified = true;
|
||||
if (latestModifiedUtcParallel.HasValue && latestModifiedUtcParallel.Value < fileInfo.LastWriteTimeUtc)
|
||||
{
|
||||
_logger.LogInformation("Snapshot is up to date");
|
||||
return;
|
||||
if (index.Count != 0)
|
||||
{
|
||||
// directory may have been added with older roms, verify that the snapshot is still up to date
|
||||
foreach (var dir in _options.RootDirectories)
|
||||
{
|
||||
// check first entry is in index
|
||||
var entry = BuildSnapshot(dir).FirstOrDefault();
|
||||
if (entry != null)
|
||||
{
|
||||
if (!index.TryGetValue(entry.Path, out var cached))
|
||||
{
|
||||
snapshotVerified = false;
|
||||
_logger.LogInformation("Snapshot does not contain first entry in directory {Directory}", dir);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (snapshotVerified)
|
||||
{
|
||||
_logger.LogInformation("Snapshot is up to date");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation("Snapshot is up to date but index is empty");
|
||||
}
|
||||
}
|
||||
_logger.LogInformation("Rebuilding snapshot (root dirs: {Count})", _options.RootDirectories.Count);
|
||||
var entries = new List<FileEntry>();
|
||||
@@ -156,65 +289,122 @@ public sealed class SnapshotService : IDisposable, ISnapshotService
|
||||
var snapshotChanged = false;
|
||||
foreach (var dir in _options.RootDirectories)
|
||||
{
|
||||
if (!Directory.Exists(dir)) continue;
|
||||
foreach (var file in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories))
|
||||
_ = Task.Run(() =>
|
||||
{
|
||||
var ext = Path.GetExtension(file).ToLowerInvariant();
|
||||
|
||||
if (!(_options.WhitelistExtensions.Contains(ext) || _options.RomExtensions.Contains(ext)))
|
||||
continue;
|
||||
|
||||
if (index.TryGetValue(file, out var value))
|
||||
{
|
||||
entries.Add(value);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 3) extract title if applicable
|
||||
string hash;
|
||||
NcaMetadataWithHash? title = null;
|
||||
if (_options.RomExtensions.Contains(ext))
|
||||
{
|
||||
using var nspStream = File.OpenRead(file);
|
||||
hash = ComputeFirstStreamHash(nspStream);
|
||||
|
||||
// 2) use cached title if unchanged
|
||||
if (index.TryGetValue(file, out var cached) && cached.Hash == hash)
|
||||
{
|
||||
entries.Add(cached);
|
||||
continue;
|
||||
}
|
||||
|
||||
title = _nspExtractor.ExtractFromStream(nspStream);
|
||||
}
|
||||
else
|
||||
{
|
||||
hash = ComputeFirstStreamHash(file);
|
||||
title = _archiveHandler.TryExtractTitleInfo(file);
|
||||
}
|
||||
|
||||
if (title == null)
|
||||
{
|
||||
_logger.LogInformation("Failed to process {File}", file);
|
||||
}
|
||||
// 4) update cache
|
||||
_cache[file] = new CachedFile(file, hash, title);
|
||||
|
||||
// 5) add to snapshot
|
||||
entries.Add(new FileEntry(file, new FileInfo(file).Length, hash, title));
|
||||
_logger.LogInformation("Added {File} to snapshot (hash={Hash}", file, hash);
|
||||
snapshotChanged = true;
|
||||
}
|
||||
_logger.LogInformation("Rebuilding directory {Directory}", dir);
|
||||
var buildSnapshot = BuildSnapshot(dir);
|
||||
var fileEntries = buildSnapshot.ToList();
|
||||
snapshotChanged = snapshotChanged || fileEntries.Count != 0;
|
||||
entries.AddRange(fileEntries.Where(entry => entry != null)!);
|
||||
});
|
||||
}
|
||||
|
||||
// Replace the entire snapshot
|
||||
_currentSnapshotHash = ComputeSnapshotHash(entries);
|
||||
File.WriteAllText(_jsonPath, JsonSerializer.Serialize(entries));
|
||||
ComputeSnapshotHash(entries);
|
||||
if (snapshotChanged)
|
||||
{
|
||||
_logger.LogInformation("Snapshot rebuilt");
|
||||
SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
|
||||
}
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public void GetArchiveName(string titleId)
|
||||
{
|
||||
;
|
||||
}
|
||||
|
||||
// Returns List of FileEntry that do not have a hash in the cache
|
||||
// Each entry that has not been added to the lookup table is added to the cache
|
||||
private IEnumerable<FileEntry?> BuildSnapshot(string dir)
|
||||
{
|
||||
FileEntry entry;
|
||||
if (!Directory.Exists(dir)) yield break;
|
||||
foreach (var file in Directory.EnumerateFiles(dir, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
string hash = string.Empty;
|
||||
var ext = Path.GetExtension(file).ToLowerInvariant();
|
||||
|
||||
if (!(_options.ArchiveExtensions.Contains(ext) || _options.RomExtensions.Contains(ext)))
|
||||
continue;
|
||||
|
||||
if (_cache.ContainsKey(file) || _hashCache.ContainsKey(hash))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// 3) extract title if applicable
|
||||
var titles = new List<(string, long, NcaMetadataWithHash)>();
|
||||
if (_options.RomExtensions.Contains(ext))
|
||||
{
|
||||
using var nspStream = File.OpenRead(file);
|
||||
hash = ComputeFirstStreamHash(nspStream);
|
||||
|
||||
if (_hashCache.ContainsKey(hash))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var nspStreamLength = nspStream.Length;
|
||||
var title = _nspExtractor.ExtractFromStream(nspStream);
|
||||
if (title != null)
|
||||
{
|
||||
var archiveEntry = new FileEntry(file, nspStreamLength, hash, [title]);
|
||||
AddToSnapshotAsync(archiveEntry);
|
||||
titles.Add((title.TitleId, nspStreamLength, title));
|
||||
yield return archiveEntry;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (_options.ArchiveExtensions.Contains(ext))
|
||||
{
|
||||
if (_archiveLookup.ContainsKey(file)) continue;
|
||||
hash = ComputeFirstStreamHash(file);
|
||||
if (_hashCache.ContainsKey(hash))
|
||||
{
|
||||
yield return null;
|
||||
}
|
||||
|
||||
IEnumerable<(string, long, NcaMetadataWithHash)>? titlesEnumerable = null;
|
||||
try
|
||||
{
|
||||
titlesEnumerable = _archiveHandler.TryExtractTitleInfos(file);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "Failed to extract title info from archive {Archive}", file);
|
||||
}
|
||||
|
||||
if (titlesEnumerable == null) continue;
|
||||
|
||||
titles = titlesEnumerable.ToList();
|
||||
foreach (var title in titles)
|
||||
{
|
||||
var archiveEntry = new FileEntry(file + ArchivePathSeparator + title.Item1, title.Item2, title.Item3.Hash, [title.Item3]);
|
||||
AddToSnapshotAsync(archiveEntry);
|
||||
yield return archiveEntry;
|
||||
}
|
||||
/*var fileEntry = new FileEntry(file, new FileInfo(file).Length, hash, titles.Select((tuple, i) => tuple.Item3).ToList());
|
||||
AddToSnapshotAsync(fileEntry);
|
||||
yield return fileEntry;*/
|
||||
}
|
||||
else
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (titles.Count == 0)
|
||||
{
|
||||
_logger.LogInformation("Failed to process {File}", file);
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation("Added {File} to snapshot (hash={Hash})", file, hash);
|
||||
yield return new FileEntry(file, titles.Select((tuple, i) => tuple.Item2).FirstOrDefault(), hash, titles.Select((tuple, i) => tuple.Item3).ToList());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private string ComputeFirstStreamHash(Stream nspStream)
|
||||
@@ -222,20 +412,65 @@ public sealed class SnapshotService : IDisposable, ISnapshotService
|
||||
return _nspExtractor.ExtractHashFromStream(nspStream);
|
||||
}
|
||||
|
||||
private void UpdateSnapshot() => BuildSnapshot();
|
||||
private void UpdateSnapshot() => BuildSnapshotAsync();
|
||||
|
||||
private void PersistSnapshot()
|
||||
IEnumerable<FileEntry> GetEntries()
|
||||
{
|
||||
var snapshot = GetSnapshot();
|
||||
var newHash = ComputeSnapshotHash(snapshot.Files);
|
||||
if (_currentSnapshotHash != newHash)
|
||||
foreach (var snapshotEntry in _cache)
|
||||
{
|
||||
_logger.LogInformation("Snapshot hash changed – persisting new snapshot");
|
||||
_currentSnapshotHash = newHash;
|
||||
File.WriteAllText(_jsonPath, JsonSerializer.Serialize(snapshot.Files));
|
||||
File.WriteAllText(_snapshotPath, JsonSerializer.Serialize(snapshot.Files));
|
||||
_sizeLookup.TryGetValue(snapshotEntry.Value.Hash, out var size);
|
||||
var fileEntry = new FileEntry(snapshotEntry.Key, snapshotEntry.Value.Size, snapshotEntry.Value.Hash, snapshotEntry.Value.NcaMetadataWithHash);
|
||||
yield return fileEntry;
|
||||
}
|
||||
}
|
||||
private Task PersistSnapshotAsync()
|
||||
{
|
||||
if (_debouncerCache.TryGetValue(_jsonPath, out var value))
|
||||
{
|
||||
_logger.LogInformation("Sliding debounce in progress, skipping snapshot persistence");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
var snapshot = GetSnapshot();
|
||||
var entries = GetEntries();
|
||||
var fileEntries = entries.ToList();
|
||||
var newHash = ComputeSnapshotHash(fileEntries);
|
||||
if (snapshot.Hash == newHash) return Task.CompletedTask;
|
||||
|
||||
_logger.LogInformation("Snapshot hash changed – persisting new snapshot");
|
||||
using var debouncedPersistence = _debouncerCache.CreateEntry(_jsonPath);
|
||||
debouncedPersistence.SlidingExpiration = TimeSpan.FromMilliseconds(DebounceMs);
|
||||
debouncedPersistence.Value = fileEntries;
|
||||
debouncedPersistence.PostEvictionCallbacks.Add(new PostEvictionCallbackRegistration
|
||||
{
|
||||
EvictionCallback = (key, entriesCallback, reason, state) =>
|
||||
{
|
||||
if (entriesCallback is IEnumerable<FileEntry> entriesToPersist && key is string filePath)
|
||||
{
|
||||
if (_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout))
|
||||
{
|
||||
if (IsFileLocked(filePath))
|
||||
{
|
||||
_logger.LogInformation("File {FilePath} is locked, skipping snapshot persistence", filePath);
|
||||
}
|
||||
else
|
||||
{
|
||||
File.WriteAllText(filePath,
|
||||
JsonSerializer.Serialize(entriesToPersist, _jsonSerializerOptions));
|
||||
_snapshotFileSemaphore.Release();
|
||||
_logger.LogInformation("Persisted snapshot");
|
||||
SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogInformation("Failed to persist file {FilePath} due to timeout", filePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
|
||||
private static string ComputeHash(string filePath)
|
||||
{
|
||||
@@ -252,38 +487,103 @@ public sealed class SnapshotService : IDisposable, ISnapshotService
|
||||
var hash = sha.ComputeHash(System.Text.Encoding.UTF8.GetBytes(json));
|
||||
return BitConverter.ToString(hash).Replace("-", "").ToLowerInvariant();
|
||||
}
|
||||
/// <summary>
|
||||
/// From filesystem cache, load each entry and build the lookups
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
private Dictionary<string, FileEntry> LoadSnapshotIndex()
|
||||
{
|
||||
if (!File.Exists(_jsonPath)) return new();
|
||||
|
||||
if (!File.Exists(_jsonPath)) return new Dictionary<string, FileEntry>();
|
||||
_snapshotFileSemaphore.Wait();
|
||||
var json = File.ReadAllText(_jsonPath);
|
||||
var entries = JsonSerializer.Deserialize<List<FileEntry>>(json, new JsonSerializerOptions(){IncludeFields = true})!;
|
||||
return entries.ToDictionary(e => e.Path, e => e);
|
||||
_snapshotFileSemaphore.Release();
|
||||
var entries = JsonSerializer.Deserialize<List<FileEntry>>(json, _jsonSerializerOptions)!;
|
||||
try
|
||||
{
|
||||
var fileEntries = new Dictionary<string, FileEntry>();
|
||||
// Reindex the cache
|
||||
foreach (var fileEntry in entries)
|
||||
{
|
||||
if (_hashCache.TryGetValue(fileEntry.Hash, out var value))
|
||||
{
|
||||
_logger.LogWarning("Duplicate hash found in snapshot: {Hash}, {OldPath}, {newPath}", fileEntry.Hash, value, fileEntry.Path);
|
||||
}
|
||||
|
||||
if (_options.RomExtensions.Contains(Path.GetExtension(fileEntry.Path)))
|
||||
{
|
||||
if (fileEntry.Path.Contains(ArchivePathSeparator))
|
||||
{
|
||||
var filename = fileEntry.Path.Split(ArchivePathSeparator)[0];
|
||||
_cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileEntry.Titles);
|
||||
_archiveLookup[filename] = fileEntry.Path;
|
||||
}
|
||||
else
|
||||
{
|
||||
_cache[fileEntry.Path] = new SnapshotEntry(fileEntry.Path, fileEntry.Hash, fileEntry.Size, fileEntry.Titles);
|
||||
fileEntries.TryAdd(fileEntry.Path, fileEntry);
|
||||
_hashCache[fileEntry.Hash] = fileEntry.Path;
|
||||
// ReSharper disable once ConditionIsAlwaysTrueOrFalseAccordingToNullableAPIContract
|
||||
if (fileEntry.Titles == null) continue;
|
||||
foreach (var ncaMetadataWithHash in fileEntry.Titles)
|
||||
{
|
||||
_hashCache[ncaMetadataWithHash.Hash] = fileEntry.Path;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return fileEntries;
|
||||
}
|
||||
catch (ArgumentException e)
|
||||
{
|
||||
_logger.LogError(e, "Failed to load snapshot");
|
||||
return new();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public void RebuildSnapshot()
|
||||
{
|
||||
// Build a fresh snapshot and persist it.
|
||||
BuildSnapshotAsync(); // private method inside the same class
|
||||
PersistSnapshotAsync(); // private method inside the same class
|
||||
SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
|
||||
}
|
||||
#endregion
|
||||
|
||||
public ROMSnapshot GetSnapshot()
|
||||
{
|
||||
if (!File.Exists(_jsonPath)) return new();
|
||||
var json = File.ReadAllText(_jsonPath);
|
||||
var hash = ComputeHash(_jsonPath);
|
||||
var romSnapshot = new ROMSnapshot()
|
||||
if (!File.Exists(_jsonPath)) return new ROMSnapshot();
|
||||
|
||||
if (_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout))
|
||||
{
|
||||
Hash = hash,
|
||||
Files = JsonSerializer.Deserialize<IReadOnlyList<FileEntry>>(json,
|
||||
new JsonSerializerOptions() { IncludeFields = true })!
|
||||
};
|
||||
return romSnapshot;
|
||||
}
|
||||
try
|
||||
{
|
||||
var json = File.ReadAllText(_jsonPath);
|
||||
var hash = ComputeHash(_jsonPath);
|
||||
var romSnapshot = new ROMSnapshot
|
||||
{
|
||||
Hash = hash,
|
||||
Files = JsonSerializer.Deserialize<IReadOnlyList<FileEntry>>(json, _jsonSerializerOptions)!
|
||||
};
|
||||
return romSnapshot;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
_logger.LogError(e, "Failed to load snapshot");
|
||||
}
|
||||
finally
|
||||
{
|
||||
_snapshotFileSemaphore.Release();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogWarning("Failed to load snapshot due to timeout");
|
||||
}
|
||||
|
||||
public void RebuildSnapshot()
|
||||
{
|
||||
// Build a fresh snapshot and persist it.
|
||||
BuildSnapshot(); // private method inside the same class
|
||||
PersistSnapshot(); // private method inside the same class
|
||||
SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
|
||||
return new ROMSnapshot();
|
||||
}
|
||||
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
foreach (var watcher in _watchers)
|
||||
@@ -292,7 +592,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService
|
||||
}
|
||||
}
|
||||
|
||||
private sealed record CachedFile(string Path, string Hash, NcaMetadataWithHash? NcaMetadataWithHash);
|
||||
private sealed record SnapshotEntry(string Path, string Hash, long Size, List<NcaMetadataWithHash> NcaMetadataWithHash);
|
||||
|
||||
// File: TinfoilVibeServer/Services/SnapshotService.cs (inside SnapshotService class)
|
||||
|
||||
@@ -342,4 +642,20 @@ public sealed class SnapshotService : IDisposable, ISnapshotService
|
||||
public string Hash { get; set; }
|
||||
public IReadOnlyList<FileEntry> Files { get; set; } = new List<FileEntry>();
|
||||
}
|
||||
|
||||
public async Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("Starting snapshot service");
|
||||
_ = Task.Run(async () =>
|
||||
{
|
||||
await BuildSnapshotAsync();
|
||||
await PersistSnapshotAsync();
|
||||
}, cancellationToken); // initial scan
|
||||
new Timer(_ => DebounceElapsed(), null, Timeout.Infinite, Timeout.Infinite);
|
||||
}
|
||||
|
||||
public async Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
Dispose();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user