ecenshu/TinfoilVibeServer
1
0
Fork 0
Code Issues Pull Requests Actions 5 Packages Projects Releases Wiki Activity

If filename can extract to a NcaMetadata entry, don't use nspextractor to pull information #3

Merged
ecenshu merged 1 commits from feature/entry_from_filename into main 2025-11-15 06:59:26 +00:00
Conversation 0 Commits 1 Files Changed 8
+134 -114
8 changed files with 134 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files
TinfoilVibeServer.sln.DotSettings.user
+1
View File
@@ -88,6 +88,7 @@
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUnpack_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003F4c375bcb6a2d2378855cad1b1c7cfe7ca1448866f1e8af44226775b5f75df86_003FUnpack_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUnsafeHelpers_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F77078e9a1d254191bb508f54a277fc6e1c2e00_003Fd1_003Fc59f91c2_003FUnsafeHelpers_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AUtility_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003F3ea3ed6216d2412ac7c33016ad940618bcfbcafe1633dc26832be514633b4_003FUtility_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AWaitHandle_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E3_003Fresharper_002Dhost_003FSourcesCache_003Fcab54c7c1f8d9b3da5f7878e39faba52467f16d0899a2c4b10086cb2ef73f2b_003FWaitHandle_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AXciPartition_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FDecompilerCache_003Fdecompiler_003F77078e9a1d254191bb508f54a277fc6e1c2e00_003F04_003F4e8815da_003FXciPartition_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
<s:String x:Key="/Default/CodeInspection/ExcludedFiles/FilesAndFoldersToSkip2/=7020124F_002D9FFC_002D4AC3_002D8F3D_002DAAB8E0240759_002Ff_003AZipArchive_002Ecs_002Fl_003AC_0021_003FUsers_003Fecens_003FAppData_003FRoaming_003FJetBrains_003FRider2025_002E2_003Fresharper_002Dhost_003FSourcesCache_003F39ececcf144e1f9c884152723ed93931cd232485eaf2824bf5beb526f1f321b_003FZipArchive_002Ecs/@EntryIndexedValue">ForceIncluded</s:String>
TinfoilVibeServer/Models/FileEntry.cs
+1 -1
View File
@@ -8,6 +8,6 @@ namespace TinfoilVibeServer.Models;
public sealed record FileEntry(
string Path, // nsp or archive path
long Size, // size of nsp or full archive
string Hash, // SHA‑256 hex of first NCA of first NCP in NSP or archive
string? Hash, // SHA‑256 hex of first NCA of first NCP in NSP or archive
List<NcaMetadataWithHash> Titles // Details of all NSP Roms in the Path
);
TinfoilVibeServer/Models/NcaMetadataWithHash.cs
+26
View File
@@ -0,0 +1,26 @@
using LibHac.Ncm;
namespace TinfoilVibeServer.Models;
/// <summary>
/// DTO returned by the extractor – contains all data the snapshot needs.
/// </summary>
public sealed class NcaMetadataWithHash
{
public string TitleId { get; }
public string ApplicationTitle { get; set; }
public int Version { get; }
public ContentMetaType ContentMetaType { get; set; }
public string? Hash { get; }
public NcaMetadataWithHash(string titleId, string applicationTitle, int version,
ContentMetaType contentMetaType, string? hash = null)
{
TitleId = titleId;
ApplicationTitle = applicationTitle;
Version = version;
ContentMetaType = contentMetaType;
Hash = hash;
}
}
TinfoilVibeServer/Services/NSPExtractor.cs
+2 -25
View File
@@ -1,5 +1,4 @@
using System.ComponentModel.DataAnnotations;
using System.Security.Cryptography;
using System.Security.Cryptography;
using LibHac.Common;
using LibHac.Fs;
using LibHac.Fs.Fsa;
@@ -11,6 +10,7 @@ using LibHac.Ncm;
using LibHac.Tools.Fs;
using LibHac.Tools.Ncm;
using Microsoft.Extensions.Options;
using TinfoilVibeServer.Models;
using Path = System.IO.Path;
namespace TinfoilVibeServer.Services
@@ -322,27 +322,4 @@ namespace TinfoilVibeServer.Services
{
public string? KeyFile { get; set; }
}
/// <summary>
/// DTO returned by the extractor – contains all data the snapshot needs.
/// </summary>
public sealed class NcaMetadataWithHash
{
public string TitleId { get; }
public string ApplicationTitle { get; set; }
public int Version { get; }
public ContentMetaType ContentMetaType { get; set; }
public string Hash { get; }
public NcaMetadataWithHash(string titleId, string applicationTitle, int version,
ContentMetaType contentMetaType, string hash)
{
TitleId = titleId;
ApplicationTitle = applicationTitle;
Version = version;
ContentMetaType = contentMetaType;
Hash = hash;
}
}
}
TinfoilVibeServer/Services/SnapshotService.cs
+72 -87
View File
@@ -79,7 +79,7 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
FileSystemExtensions.EnsureDirectoryExists(Path.GetFullPath(Path.GetDirectoryName(_snapshotPath) ?? throw new InvalidOperationException()));
// 1️⃣ Register for *property* changes
options.OnChange(snapshotOptions =>
options.OnChange((snapshotOptions, arg) =>
{
_options.RootDirectories = snapshotOptions.RootDirectories;
});
@@ -256,61 +256,40 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
var index = LoadSnapshotIndex();
var latestModifiedUtcParallel = FileSystemExtensions.GetLatestModifiedUtcParallel(_options.RootDirectories);
var fileInfo = new FileInfo(_snapshotPath);
bool snapshotVerified = true;
bool snapshotVerified = fileInfo.Exists;
if (latestModifiedUtcParallel.HasValue && latestModifiedUtcParallel.Value < fileInfo.LastWriteTimeUtc)
{
if (index.Count != 0)
{
// directory may have been added with older roms, verify that the snapshot is still up to date
foreach (var dir in _options.RootDirectories)
{
// check first entry is in index
var entry = BuildSnapshot(dir).FirstOrDefault();
if (entry != null)
var firstEntry = BuildSnapshot(dir).FirstOrDefault();
if (firstEntry != null && !index.TryGetValue(firstEntry.Path, out _))
{
if (!index.TryGetValue(entry.Path, out var cached))
{
snapshotVerified = false;
_logger.LogInformation("Snapshot does not contain first entry in directory {Directory}", dir);
}
snapshotVerified = false;
_logger.LogInformation("Snapshot does not contain first entry in directory {Directory}", dir);
}
}
}
}
if (snapshotVerified)
if (!snapshotVerified)
{
_logger.LogInformation("Rebuilding snapshot (root dirs: {Count})", _options.RootDirectories.Count);
var entries = new List<FileEntry>();
foreach (var dir in _options.RootDirectories)
{
foreach (var entry in BuildSnapshot(dir))
{
_logger.LogInformation("Snapshot is up to date");
return Task.CompletedTask;
if (entry != null) entries.Add(entry);
}
}
else
{
_logger.LogInformation("Snapshot is up to date but index is empty");
}
}
_logger.LogInformation("Rebuilding snapshot (root dirs: {Count})", _options.RootDirectories.Count);
var entries = new List<FileEntry>();
var snapshotChanged = false;
foreach (var dir in _options.RootDirectories)
{
_ = Task.Run(() =>
{
_logger.LogInformation("Rebuilding directory {Directory}", dir);
var buildSnapshot = BuildSnapshot(dir);
var fileEntries = buildSnapshot.ToList();
snapshotChanged = snapshotChanged || fileEntries.Count != 0;
entries.AddRange(fileEntries.Where(entry => entry != null)!);
});
var currentHash = ComputeSnapshotHash(entries);
if (entries.Count > 0 || fileInfo.Exists && index.Count == 0)
SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
}
var snapshotEmptied = fileInfo.Exists && index.Count == 0 && _options.RootDirectories.Count == 0;
// Replace the entire snapshot
var currentSnapshotHash = ComputeSnapshotHash(entries);
if (snapshotChanged || snapshotEmptied)
{
_logger.LogInformation("Snapshot rebuilt");
SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
}
PersistSnapshotAsync();
return Task.CompletedTask;
}
@@ -341,6 +320,15 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
var titles = new List<(string, long, NcaMetadataWithHash)>();
if (_options.RomExtensions.Contains(ext))
{
var fileInfo = new FileInfo(file);
var ncaMetadataWithHash = fileInfo.GetNcaMetadataWithHash();
if (ncaMetadataWithHash != null)
{
//var titleInfo = _titleDatabaseService.GetAsync(ncaMetadataWithHash.TitleId).Result;
var fileEntryFromFileName = new FileEntry(file, fileInfo.Length, ncaMetadataWithHash.Hash, [ncaMetadataWithHash]);
AddToSnapshotAsync(fileEntryFromFileName);
yield return fileEntryFromFileName;
}
using var nspStream = File.OpenRead(file);
hash = ComputeFirstStreamHash(nspStream);
@@ -416,74 +404,71 @@ public sealed class SnapshotService : IDisposable, ISnapshotService, IHostedServ
await Task.CompletedTask;
}
private string ComputeFirstStreamHash(Stream nspStream)
{
return _nspExtractor.ExtractHashFromStream(nspStream);
}
private string ComputeFirstStreamHash(Stream nspStream) => _nspExtractor.ExtractHashFromStream(nspStream);
private void UpdateSnapshot() => BuildSnapshotAsync();
IEnumerable<FileEntry> GetEntries()
private IEnumerable<FileEntry> GetEntries()
{
foreach (var snapshotEntry in _cache)
{
_sizeLookup.TryGetValue(snapshotEntry.Value.Hash, out var size);
var fileEntry = new FileEntry(snapshotEntry.Key, snapshotEntry.Value.Size, snapshotEntry.Value.Hash, snapshotEntry.Value.NcaMetadataWithHash);
yield return fileEntry;
}
foreach (var kv in _cache)
yield return new FileEntry(kv.Key, kv.Value.Size, kv.Value.Hash, kv.Value.NcaMetadataWithHash);
}
private Task PersistSnapshotAsync()
{
if (_debouncerCache.TryGetValue(_jsonPath, out var value))
if (_debouncerCache.TryGetValue(_jsonPath, out _))
{
_logger.LogInformation("Sliding debounce in progress, skipping snapshot persistence");
return Task.CompletedTask;
}
}
var entries = GetEntries().ToList();
var newHash = ComputeSnapshotHash(entries);
var snapshot = GetSnapshot();
var entries = GetEntries();
var fileEntries = entries.ToList();
var newHash = ComputeSnapshotHash(fileEntries);
if (snapshot.Hash == newHash) return Task.CompletedTask;
CancellationTokenSource cts = new();
_logger.LogInformation("Snapshot hash changed – persisting new snapshot");
using var debouncedPersistence = _debouncerCache.CreateEntry(_jsonPath);
debouncedPersistence.AddExpirationToken(new CancellationChangeToken(cts.Token));
//debouncedPersistence.AbsoluteExpirationRelativeToNow = TimeSpan.FromMilliseconds(DebounceMs);
debouncedPersistence.Value = fileEntries;
debouncedPersistence.PostEvictionCallbacks.Add(new PostEvictionCallbackRegistration
{
EvictionCallback = (key, entriesCallback, reason, state) =>
var cancellationTokenSource = new CancellationTokenSource();
using var cacheEntry = _debouncerCache.CreateEntry(_jsonPath)
.AddExpirationToken(new CancellationChangeToken(cancellationTokenSource.Token))
.SetValue(entries)
.SetOptions(new MemoryCacheEntryOptions
{
if (entriesCallback is IEnumerable<FileEntry> entriesToPersist && key is string filePath)
PostEvictionCallbacks =
{
if (_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout))
new PostEvictionCallbackRegistration
{
if (IsFileLocked(filePath))
EvictionCallback = (key, value, reason, state) =>
{
_logger.LogInformation("File {FilePath} is locked, skipping snapshot persistence", filePath);
if (!(reason == EvictionReason.Expired || reason == EvictionReason.TokenExpired))
return;
var filePath = (string)key;
if (_snapshotFileSemaphore.Wait(SnapshotFileLockTimeout))
{
try
{
if (IsFileLocked(filePath))
{
_logger.LogInformation("File {FilePath} is locked, skipping snapshot persistence", filePath);
}
else
{
File.WriteAllText(filePath, JsonSerializer.Serialize(value, _jsonSerializerOptions));
SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
}
}
finally
{
_snapshotFileSemaphore.Release();
}
}
}
else
{
File.WriteAllText(filePath,
JsonSerializer.Serialize(entriesToPersist, _jsonSerializerOptions));
_snapshotFileSemaphore.Release();
_logger.LogInformation("Persisted snapshot");
SnapshotRebuilt?.Invoke(this, EventArgs.Empty);
}
}
else
{
_logger.LogInformation("Failed to persist file {FilePath} due to timeout", filePath);
}
}
}
});
cts.CancelAfter(TimeSpan.FromMilliseconds(DebounceMs));
});
cancellationTokenSource.CancelAfter(TimeSpan.FromMilliseconds(DebounceMs));
return Task.CompletedTask;
}
private static string ComputeHash(string filePath)
{
using var sha = SHA256.Create();
TinfoilVibeServer/Models/IdHelper.cs → TinfoilVibeServer/Utilities/IdHelper.cs
+1 -1
View File
@@ -1,7 +1,7 @@
using System.Globalization;
using System.Text.RegularExpressions;
namespace TinfoilVibeServer.Models;
namespace TinfoilVibeServer.Utilities;
public static class IdHelper
TinfoilVibeServer/Utilities/NcaMetadataWithHashHelper.cs
+30
View File
@@ -0,0 +1,30 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using LibHac.Ncm;
using TinfoilVibeServer.Models;
namespace TinfoilVibeServer.Utilities;
public static class NcaMetadataWithHashHelper
{
public static NcaMetadataWithHash? GetNcaMetadataWithHash(this FileInfo fileInfo)
{
var match = Regex.Match(fileInfo.Name, @"^(.+)\[(\w{16})\]\[v(\d{1,7})\]\[(\w+).*\]\.nsp$");
if (!match.Success) return null;
var titleId = match.Groups[2].Value;
var applicationTitle = match.Groups[1].Value.Trim();
var version = int.Parse(match.Groups[3].Value) / 0x10000;
var nspType = match.Groups[4].Value.ToLowerInvariant() switch
{
"base" => ContentMetaType.Application,
"update" => ContentMetaType.Patch,
"dlc" => ContentMetaType.AddOnContent,
_ => ContentMetaType.Patch
};
var bytes = Encoding.UTF8.GetBytes(fileInfo.FullName);
var hash = SHA256.HashData(bytes);
return new NcaMetadataWithHash(titleId, applicationTitle, version, nspType, Convert.ToBase64String(hash));
}
}
compose.yaml
+1
View File
@@ -8,6 +8,7 @@ services:
image: gitea.ecenshu.net/ecenshu/tinfoilvibeserver:latest
container_name: tinfoilvibeserver
restart: unless-stopped
user: "1000:1000"
env_file:
- .env
environment: