From c8140d71783ddf1540367c6219ff50eb0796ef1b Mon Sep 17 00:00:00 2001 From: Michael Green <84688932+michael-j-green@users.noreply.github.com> Date: Sun, 15 Sep 2024 03:35:36 -0700 Subject: [PATCH] Revamp of BIOS handling (#423) Many of the platforms are similar enough to other platforms that they'll share the same BIOS files. The current storage method stores BIOS files per platform, meaning that files can be duplicated multiple times to satisfy the requirements of each platform. This change stores the files as their hash with a .bios extension (example: `85ad74194e87c08904327de1a9443b7a.bios`) in a flat directory structure. This allows BIOS files that are used by multiple platforms to be shared without duplication. --- gaseous-server/Classes/Bios.cs | 44 ++++++++++++--- gaseous-server/Classes/Config.cs | 11 +++- gaseous-server/Classes/ImportGames.cs | 7 +-- .../Controllers/V1.0/BiosController.cs | 53 +++++++++---------- gaseous-server/Program.cs | 3 ++ 5 files changed, 77 insertions(+), 41 deletions(-) diff --git a/gaseous-server/Classes/Bios.cs b/gaseous-server/Classes/Bios.cs index de1aea6..c23c1f0 100644 --- a/gaseous-server/Classes/Bios.cs +++ b/gaseous-server/Classes/Bios.cs @@ -4,12 +4,39 @@ using System.Security.Cryptography; namespace gaseous_server.Classes { - public class Bios - { - public Bios() - { - - } + public class Bios + { + public Bios() + { + + } + + public static void MigrateToNewFolderStructure() + { + // migrate from old BIOS file structure which had each bios file inside a folder named for the platform to the new structure which has each file in a subdirectory named after the MD5 hash + if (Directory.Exists(Config.LibraryConfiguration.LibraryBIOSDirectory)) + { + foreach (Models.PlatformMapping.PlatformMapItem platformMapping in Models.PlatformMapping.PlatformMap) + { + if (platformMapping.Bios != null) + { + foreach (Models.PlatformMapping.PlatformMapItem.EmulatorBiosItem emulatorBiosItem in platformMapping.Bios) + { + string oldBiosPath = Path.Combine(Config.LibraryConfiguration.LibraryBIOSDirectory, platformMapping.IGDBSlug.ToString(), emulatorBiosItem.filename); + string newBiosPath = Path.Combine(Config.LibraryConfiguration.LibraryFirmwareDirectory, emulatorBiosItem.hash + ".bios"); + + if (File.Exists(oldBiosPath)) + { + File.Copy(oldBiosPath, newBiosPath, true); + } + } + } + } + + // remove old BIOS folder structure + Directory.Delete(Config.LibraryConfiguration.LibraryBIOSDirectory, true); + } + } public static Models.PlatformMapping.PlatformMapItem? BiosHashSignatureLookup(string MD5) { @@ -96,10 +123,11 @@ namespace gaseous_server.Classes { get { - return Path.Combine(Config.LibraryConfiguration.LibraryBIOSDirectory, platformslug, base.filename); + return Path.Combine(Config.LibraryConfiguration.LibraryFirmwareDirectory, hash + ".bios"); } } - public bool Available { + public bool Available + { get { bool fileExists = File.Exists(biosPath); diff --git a/gaseous-server/Classes/Config.cs b/gaseous-server/Classes/Config.cs index c82dadb..9b5dacd 100644 --- a/gaseous-server/Classes/Config.cs +++ b/gaseous-server/Classes/Config.cs @@ -575,6 +575,14 @@ namespace gaseous_server.Classes } } + public string LibraryFirmwareDirectory + { + get + { + return Path.Combine(LibraryRootDirectory, "Firmware"); + } + } + public string LibraryUploadDirectory { get @@ -656,7 +664,8 @@ namespace gaseous_server.Classes { if (!Directory.Exists(LibraryRootDirectory)) { Directory.CreateDirectory(LibraryRootDirectory); } if (!Directory.Exists(LibraryImportDirectory)) { Directory.CreateDirectory(LibraryImportDirectory); } - if (!Directory.Exists(LibraryBIOSDirectory)) { Directory.CreateDirectory(LibraryBIOSDirectory); } + // if (!Directory.Exists(LibraryBIOSDirectory)) { Directory.CreateDirectory(LibraryBIOSDirectory); } + if (!Directory.Exists(LibraryFirmwareDirectory)) { Directory.CreateDirectory(LibraryFirmwareDirectory); } if (!Directory.Exists(LibraryUploadDirectory)) { Directory.CreateDirectory(LibraryUploadDirectory); } if (!Directory.Exists(LibraryMetadataDirectory)) { Directory.CreateDirectory(LibraryMetadataDirectory); } if (!Directory.Exists(LibraryTempDirectory)) { Directory.CreateDirectory(LibraryTempDirectory); } diff --git a/gaseous-server/Classes/ImportGames.cs b/gaseous-server/Classes/ImportGames.cs index 155c392..07c5d5e 100644 --- a/gaseous-server/Classes/ImportGames.cs +++ b/gaseous-server/Classes/ImportGames.cs @@ -151,11 +151,8 @@ namespace gaseous_server.Classes { if (biosItem.hash == hash.md5hash) { - string biosPath = biosItem.biosPath.Replace(biosItem.filename, ""); - if (!Directory.Exists(biosPath)) - { - Directory.CreateDirectory(biosPath); - } + string biosPath = Path.Combine(Config.LibraryConfiguration.LibraryFirmwareDirectory, biosItem.hash + ".bios"); + Logging.Log(Logging.LogType.Information, "Import Game", " " + GameFileImportPath + " is a BIOS file - moving to " + biosPath); File.Move(GameFileImportPath, biosItem.biosPath, true); diff --git a/gaseous-server/Controllers/V1.0/BiosController.cs b/gaseous-server/Controllers/V1.0/BiosController.cs index cf777d3..9e26bd9 100644 --- a/gaseous-server/Controllers/V1.0/BiosController.cs +++ b/gaseous-server/Controllers/V1.0/BiosController.cs @@ -10,6 +10,8 @@ using Asp.Versioning; using Authentication; using Microsoft.AspNetCore.Identity; using gaseous_server.Models; +using IGDB.Models; +using gaseous_server.Classes.Metadata; namespace gaseous_server.Controllers { @@ -62,25 +64,15 @@ namespace gaseous_server.Controllers { try { + Platform platform = Platforms.GetPlatform(PlatformId); + PlatformMapping.PlatformMapItem platformMap = PlatformMapping.GetPlatformMap(PlatformId); + + List biosHashes = new List(); + if (GameId == -1 || filtered == false) { - IGDB.Models.Platform platform = Classes.Metadata.Platforms.GetPlatform(PlatformId); - - string biosPath = Path.Combine(Config.LibraryConfiguration.LibraryBIOSDirectory, platform.Slug); - - string tempFile = Path.GetTempFileName(); - - using (FileStream zipFile = System.IO.File.Create(tempFile)) - using (var zipArchive = new ZipArchive(zipFile, ZipArchiveMode.Create)) - { - foreach (string file in Directory.GetFiles(biosPath)) - { - zipArchive.CreateEntryFromFile(file, Path.GetFileName(file)); - } - } - - var stream = new FileStream(tempFile, FileMode.Open); - return File(stream, "application/zip", platform.Slug + ".zip"); + // get all bios files for selected platform + biosHashes.AddRange(platformMap.EnabledBIOSHashes); } else { @@ -90,24 +82,31 @@ namespace gaseous_server.Controllers PlatformMapping platformMapping = new PlatformMapping(); PlatformMapping.PlatformMapItem userPlatformMap = platformMapping.GetUserPlatformMap(user.Id, PlatformId, GameId); - // build zip file - string tempFile = Path.GetTempFileName(); + biosHashes.AddRange(userPlatformMap.EnabledBIOSHashes); + } - using (FileStream zipFile = System.IO.File.Create(tempFile)) - using (var zipArchive = new ZipArchive(zipFile, ZipArchiveMode.Create)) + // build zip file + string tempFile = Path.GetTempFileName(); + + using (FileStream zipFile = System.IO.File.Create(tempFile)) + using (var zipArchive = new ZipArchive(zipFile, ZipArchiveMode.Create)) + { + foreach (string hash in biosHashes) { - foreach (Bios.BiosItem bios in GetBios(PlatformId, true)) + // get the bios data for the hash + foreach (PlatformMapping.PlatformMapItem.EmulatorBiosItem bios in platformMap.Bios) { - if (userPlatformMap.EnabledBIOSHashes.Contains(bios.hash)) + if (bios.hash == hash) { - zipArchive.CreateEntryFromFile(bios.biosPath, bios.filename); + // add the bios file to the zip + zipArchive.CreateEntryFromFile(Path.Combine(Config.LibraryConfiguration.LibraryFirmwareDirectory, hash + ".bios"), bios.filename); } } } - - var stream = new FileStream(tempFile, FileMode.Open); - return File(stream, "application/zip", userPlatformMap.IGDBSlug + ".zip"); } + + var stream = new FileStream(tempFile, FileMode.Open); + return File(stream, "application/zip", platform.Slug + ".zip"); } catch { diff --git a/gaseous-server/Program.cs b/gaseous-server/Program.cs index 23b63b1..3bb21fa 100644 --- a/gaseous-server/Program.cs +++ b/gaseous-server/Program.cs @@ -338,6 +338,9 @@ gaseous_server.Classes.Metadata.Platforms.AssignAllPlatformsToGameIdZero(); // extract platform map if not present PlatformMapping.ExtractPlatformMap(); +// migrate old firmware directory structure to new style +Bios.MigrateToNewFolderStructure(); + // add background tasks ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem( ProcessQueue.QueueItemType.SignatureIngestor)