From f16b2aabbf42bec733b3ebb88d9d1248614f66e7 Mon Sep 17 00:00:00 2001 From: Michael Green <84688932+michael-j-green@users.noreply.github.com> Date: Wed, 3 May 2023 23:36:07 +1000 Subject: [PATCH] fix: updated import code and added organise library command --- gaseous-server/Classes/ImportGames.cs | 299 ++++++++++++++---- gaseous-server/Classes/Metadata/Games.cs | 10 +- .../Classes/Metadata/PlatformLogos.cs | 40 ++- .../Classes/Metadata/PlatformVersions.cs | 27 +- gaseous-server/Classes/Roms.cs | 64 ++++ gaseous-server/Program.cs | 3 + 6 files changed, 360 insertions(+), 83 deletions(-) create mode 100644 gaseous-server/Classes/Roms.cs diff --git a/gaseous-server/Classes/ImportGames.cs b/gaseous-server/Classes/ImportGames.cs index c4c4aed..257f27a 100644 --- a/gaseous-server/Classes/ImportGames.cs +++ b/gaseous-server/Classes/ImportGames.cs @@ -2,6 +2,7 @@ using System.Data; using System.Threading.Tasks; using gaseous_tools; +using Org.BouncyCastle.Utilities.IO.Pem; namespace gaseous_server.Classes { @@ -16,8 +17,7 @@ namespace gaseous_server.Classes // import files first foreach (string importContent in importContents_Files) { - ImportGame importGame = new ImportGame(); - importGame.ImportGameFile(importContent); + ImportGame.ImportGameFile(importContent); } } else @@ -34,9 +34,13 @@ namespace gaseous_server.Classes { private Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); - public void ImportGameFile(string GameFileImportPath, bool IsDirectory = false) + public static void ImportGameFile(string GameFileImportPath, bool IsDirectory = false, bool ForceImport = false) { - if (String.Equals(Path.GetFileName(GameFileImportPath),".DS_STORE", StringComparison.OrdinalIgnoreCase)) + Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); + string sql = ""; + Dictionary dbDict = new Dictionary(); + + if (String.Equals(Path.GetFileName(GameFileImportPath),".DS_STORE", StringComparison.OrdinalIgnoreCase)) { Logging.Log(Logging.LogType.Information, "Import Game", "Skipping item " + GameFileImportPath); } @@ -46,75 +50,248 @@ namespace gaseous_server.Classes if (IsDirectory == false) { FileInfo fi = new FileInfo(GameFileImportPath); + Common.hashObject hash = new Common.hashObject(GameFileImportPath); - // process as a single file - // check 1: do we have a signature for it? - Common.hashObject hash = new Common.hashObject(GameFileImportPath); - gaseous_server.Controllers.SignaturesController sc = new Controllers.SignaturesController(); - List signatures = sc.GetSignature(hash.md5hash); - if (signatures.Count == 0) + // check to make sure we don't already have this file imported + sql = "SELECT COUNT(Id) AS count FROM games_roms WHERE md5=@md5 AND sha1=@sha1"; + dbDict.Add("md5", hash.md5hash); + dbDict.Add("sha1", hash.sha1hash); + DataTable importDB = db.ExecuteCMD(sql, dbDict); + if ((Int64)importDB.Rows[0]["count"] > 0) { - // no md5 signature found - try sha1 - signatures = sc.GetSignature("", hash.sha1hash); - } - - Models.Signatures_Games discoveredSignature = new Models.Signatures_Games(); - if (signatures.Count == 1) - { - // only 1 signature found! - discoveredSignature = signatures.ElementAt(0); - gaseous_server.Models.PlatformMapping.GetIGDBPlatformMapping(ref discoveredSignature, fi, false); - } - else if (signatures.Count > 1) - { - // more than one signature found - find one with highest score - foreach (Models.Signatures_Games Sig in signatures) - { - if (Sig.Score > discoveredSignature.Score) - { - discoveredSignature = Sig; - gaseous_server.Models.PlatformMapping.GetIGDBPlatformMapping(ref discoveredSignature, fi, false); - } - } + Logging.Log(Logging.LogType.Information, "Import Game", " " + GameFileImportPath + " already in database - skipping"); } else { - // no signature match found - try alternate methods - Models.Signatures_Games.GameItem gi = new Models.Signatures_Games.GameItem(); - Models.Signatures_Games.RomItem ri = new Models.Signatures_Games.RomItem(); + Logging.Log(Logging.LogType.Information, "Import Game", " " + GameFileImportPath + " not in database - processing"); - discoveredSignature.Game = gi; - discoveredSignature.Rom = ri; + // process as a single file + // check 1: do we have a signature for it? + gaseous_server.Controllers.SignaturesController sc = new Controllers.SignaturesController(); + List signatures = sc.GetSignature(hash.md5hash); + if (signatures.Count == 0) + { + // no md5 signature found - try sha1 + signatures = sc.GetSignature("", hash.sha1hash); + } - // game title is the file name without the extension or path - gi.Name = Path.GetFileNameWithoutExtension(GameFileImportPath); + Models.Signatures_Games discoveredSignature = new Models.Signatures_Games(); + if (signatures.Count == 1) + { + // only 1 signature found! + discoveredSignature = signatures.ElementAt(0); + gaseous_server.Models.PlatformMapping.GetIGDBPlatformMapping(ref discoveredSignature, fi, false); + } + else if (signatures.Count > 1) + { + // more than one signature found - find one with highest score + foreach (Models.Signatures_Games Sig in signatures) + { + if (Sig.Score > discoveredSignature.Score) + { + discoveredSignature = Sig; + gaseous_server.Models.PlatformMapping.GetIGDBPlatformMapping(ref discoveredSignature, fi, false); + } + } + } + else + { + // no signature match found - try alternate methods + Models.Signatures_Games.GameItem gi = new Models.Signatures_Games.GameItem(); + Models.Signatures_Games.RomItem ri = new Models.Signatures_Games.RomItem(); - // guess platform - gaseous_server.Models.PlatformMapping.GetIGDBPlatformMapping(ref discoveredSignature, fi, true); + discoveredSignature.Game = gi; + discoveredSignature.Rom = ri; - // get rom data - ri.Name = Path.GetFileName(GameFileImportPath); - ri.Md5 = hash.md5hash; - ri.Sha1 = hash.sha1hash; - } + // game title is the file name without the extension or path + gi.Name = Path.GetFileNameWithoutExtension(GameFileImportPath); - Console.WriteLine("Importing " + discoveredSignature.Game.Name + " (" + discoveredSignature.Game.Year + ") " + discoveredSignature.Game.System); - // get discovered platform - IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(discoveredSignature.Flags.IGDBPlatformId); - // search discovered game - IGDB.Models.Game[] games = Metadata.Games.SearchForGame(discoveredSignature.Game.Name, discoveredSignature.Flags.IGDBPlatformId, Metadata.Games.SearchType.where); - if (games.Length == 0) - { - games = Metadata.Games.SearchForGame(discoveredSignature.Game.Name, discoveredSignature.Flags.IGDBPlatformId, Metadata.Games.SearchType.search); - } - if (games.Length > 0) - { - IGDB.Models.Game determinedGame = Metadata.Games.GetGame((long)games[0].Id); - Console.WriteLine(" IGDB game: " + determinedGame.Name); - } + // guess platform + gaseous_server.Models.PlatformMapping.GetIGDBPlatformMapping(ref discoveredSignature, fi, true); + + // get rom data + ri.Name = Path.GetFileName(GameFileImportPath); + ri.Md5 = hash.md5hash; + ri.Sha1 = hash.sha1hash; + ri.Size = fi.Length; + } + + Logging.Log(Logging.LogType.Information, "Import Game", " Determined import file as: " + discoveredSignature.Game.Name + " (" + discoveredSignature.Game.Year + ") " + discoveredSignature.Game.System); + // get discovered platform + IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(discoveredSignature.Flags.IGDBPlatformId); + if (determinedPlatform == null) + { + determinedPlatform = new IGDB.Models.Platform(); + } + + // search discovered game - case insensitive exact match first + IGDB.Models.Game determinedGame = new IGDB.Models.Game(); + + foreach (Metadata.Games.SearchType searchType in Enum.GetValues(typeof(Metadata.Games.SearchType))) + { + Logging.Log(Logging.LogType.Information, "Import Game", " Search type: " + searchType.ToString()); + IGDB.Models.Game[] games = Metadata.Games.SearchForGame(discoveredSignature.Game.Name, discoveredSignature.Flags.IGDBPlatformId, searchType); + if (games.Length == 1) + { + // exact match! + determinedGame = Metadata.Games.GetGame((long)games[0].Id); + Logging.Log(Logging.LogType.Information, "Import Game", " IGDB game: " + determinedGame.Name); + break; + } + } + if (determinedGame == null) + { + determinedGame = new IGDB.Models.Game(); + } + + string destSlug = ""; + if (determinedGame.Id == null) + { + Logging.Log(Logging.LogType.Information, "Import Game", " Unable to determine game"); + } + + // add to database + sql = "INSERT INTO games_roms (platformid, gameid, name, size, crc, md5, sha1, developmentstatus, flags, romtype, romtypemedia, medialabel, path) VALUES (@platformid, @gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @path); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; + dbDict.Add("platformid", Common.ReturnValueIfNull(determinedPlatform.Id, 0)); + dbDict.Add("gameid", Common.ReturnValueIfNull(determinedGame.Id, 0)); + dbDict.Add("name", Common.ReturnValueIfNull(discoveredSignature.Rom.Name, "")); + dbDict.Add("size", Common.ReturnValueIfNull(discoveredSignature.Rom.Size, 0)); + dbDict.Add("crc", Common.ReturnValueIfNull(discoveredSignature.Rom.Crc, "")); + dbDict.Add("developmentstatus", Common.ReturnValueIfNull(discoveredSignature.Rom.DevelopmentStatus, "")); + + if (discoveredSignature.Rom.flags != null) + { + if (discoveredSignature.Rom.flags.Count > 0) + { + dbDict.Add("flags", Newtonsoft.Json.JsonConvert.SerializeObject(discoveredSignature.Rom.flags)); + } + else + { + dbDict.Add("flags", "[ ]"); + } + } + else + { + dbDict.Add("flags", "[ ]"); + } + dbDict.Add("romtype", (int)discoveredSignature.Rom.RomType); + dbDict.Add("romtypemedia", Common.ReturnValueIfNull(discoveredSignature.Rom.RomTypeMedia, "")); + dbDict.Add("medialabel", Common.ReturnValueIfNull(discoveredSignature.Rom.MediaLabel, "")); + dbDict.Add("path", GameFileImportPath); + + DataTable romInsert = db.ExecuteCMD(sql, dbDict); + long romId = (long)romInsert.Rows[0][0]; + + // move to destination + MoveGameFile(romId); + } } } } - } + + public static string ComputeROMPath(long RomId) + { + Classes.Roms.RomItem rom = Classes.Roms.GetRom(RomId); + + // get metadata + IGDB.Models.Platform platform = gaseous_server.Classes.Metadata.Platforms.GetPlatform(rom.PlatformId); + IGDB.Models.Game game = gaseous_server.Classes.Metadata.Games.GetGame(rom.GameId); + + // build path + string platformSlug = "Unknown Platform"; + if (platform != null) + { + platformSlug = platform.Slug; + } + string gameSlug = "Unknown Title"; + if (game != null) + { + gameSlug = game.Slug; + } + string DestinationPath = Path.Combine(Config.LibraryConfiguration.LibraryDataDirectory, gameSlug, platformSlug); + if (!Directory.Exists(DestinationPath)) + { + Directory.CreateDirectory(DestinationPath); + } + + string DestinationPathName = Path.Combine(DestinationPath, rom.Name); + + return DestinationPathName; + } + + public static void MoveGameFile(long RomId) + { + Classes.Roms.RomItem rom = Classes.Roms.GetRom(RomId); + string romPath = rom.Path; + + if (File.Exists(romPath)) + { + string DestinationPath = ComputeROMPath(RomId); + + if (romPath == DestinationPath) + { + Logging.Log(Logging.LogType.Debug, "Move Game ROM", "Destination path is the same as the current path - aborting"); + } + else + { + Logging.Log(Logging.LogType.Information, "Move Game ROM", "Moving " + romPath + " to " + DestinationPath); + if (File.Exists(DestinationPath)) + { + Logging.Log(Logging.LogType.Information, "Move Game ROM", "A file with the same name exists at the destination - aborting"); + } + else + { + File.Move(romPath, DestinationPath); + + // update the db + Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); + string sql = "UPDATE games_roms SET path=@path WHERE id=@id"; + Dictionary dbDict = new Dictionary(); + dbDict.Add("id", RomId); + dbDict.Add("path", DestinationPath); + db.ExecuteCMD(sql, dbDict); + + } + } + } + else + { + Logging.Log(Logging.LogType.Warning, "Move Game ROM", "File " + romPath + " appears to be missing!"); + } + } + + public static void OrganiseLibrary() + { + // move rom files to their new location + Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); + string sql = "SELECT * FROM games_roms"; + DataTable romDT = db.ExecuteCMD(sql); + + if (romDT.Rows.Count > 0) + { + foreach (DataRow dr in romDT.Rows) + { + long RomId = (long)dr["id"]; + MoveGameFile(RomId); + } + } + + // clean up empty directories + processDirectory(Config.LibraryConfiguration.LibraryDataDirectory); + } + + private static void processDirectory(string startLocation) + { + foreach (var directory in Directory.GetDirectories(startLocation)) + { + processDirectory(directory); + if (Directory.GetFiles(directory).Length == 0 && + Directory.GetDirectories(directory).Length == 0) + { + Directory.Delete(directory, false); + } + } + } + } } diff --git a/gaseous-server/Classes/Metadata/Games.cs b/gaseous-server/Classes/Metadata/Games.cs index 27dc20d..a23f3e2 100644 --- a/gaseous-server/Classes/Metadata/Games.cs +++ b/gaseous-server/Classes/Metadata/Games.cs @@ -146,9 +146,12 @@ namespace gaseous_server.Classes.Metadata searchBody += "search \"" + SearchString + "\"; "; searchBody += "where platforms = (" + PlatformId + ");"; break; - case SearchType.where: + case SearchType.wherefuzzy: searchBody += "where platforms = (" + PlatformId + ") & name ~ *\"" + SearchString + "\"*;"; break; + case SearchType.where: + searchBody += "where platforms = (" + PlatformId + ") & name ~ \"" + SearchString + "\";"; + break; } @@ -160,8 +163,9 @@ namespace gaseous_server.Classes.Metadata public enum SearchType { - where, - search + where = 0, + wherefuzzy = 1, + search = 2 } } } \ No newline at end of file diff --git a/gaseous-server/Classes/Metadata/PlatformLogos.cs b/gaseous-server/Classes/Metadata/PlatformLogos.cs index f873dad..8bf419c 100644 --- a/gaseous-server/Classes/Metadata/PlatformLogos.cs +++ b/gaseous-server/Classes/Metadata/PlatformLogos.cs @@ -73,13 +73,19 @@ namespace gaseous_server.Classes.Metadata { case Storage.CacheStatus.NotPresent: returnValue = await GetObjectFromServer(WhereClause, LogoPath); - Storage.NewCacheValue(returnValue); - forceImageDownload = true; + if (returnValue != null) + { + Storage.NewCacheValue(returnValue); + forceImageDownload = true; + } break; case Storage.CacheStatus.Expired: returnValue = await GetObjectFromServer(WhereClause, LogoPath); - Storage.NewCacheValue(returnValue, true); - forceImageDownload = true; + if (returnValue != null) + { + Storage.NewCacheValue(returnValue, true); + forceImageDownload = true; + } break; case Storage.CacheStatus.Current: returnValue = Storage.GetCacheValue(returnValue, "id", (long)searchValue); @@ -88,10 +94,13 @@ namespace gaseous_server.Classes.Metadata throw new Exception("How did you get here?"); } - if ((!File.Exists(Path.Combine(LogoPath, "Logo.jpg"))) || forceImageDownload == true) + if (returnValue != null) { - GetImageFromServer(returnValue.Url, LogoPath, LogoSize.t_thumb); - GetImageFromServer(returnValue.Url, LogoPath, LogoSize.t_logo_med); + if ((!File.Exists(Path.Combine(LogoPath, "Logo.jpg"))) || forceImageDownload == true) + { + GetImageFromServer(returnValue.Url, LogoPath, LogoSize.t_thumb); + GetImageFromServer(returnValue.Url, LogoPath, LogoSize.t_logo_med); + } } return returnValue; @@ -103,16 +112,23 @@ namespace gaseous_server.Classes.Metadata slug } - private static async Task GetObjectFromServer(string WhereClause, string LogoPath) + private static async Task GetObjectFromServer(string WhereClause, string LogoPath) { // get PlatformLogo metadata var results = await igdb.QueryAsync(IGDBClient.Endpoints.PlatformLogos, query: fieldList + " " + WhereClause + ";"); - var result = results.First(); + if (results.Length > 0) + { + var result = results.First(); - GetImageFromServer(result.Url, LogoPath, LogoSize.t_thumb); - GetImageFromServer(result.Url, LogoPath, LogoSize.t_logo_med); + GetImageFromServer(result.Url, LogoPath, LogoSize.t_thumb); + GetImageFromServer(result.Url, LogoPath, LogoSize.t_logo_med); - return result; + return result; + } + else + { + return null; + } } private static void GetImageFromServer(string Url, string LogoPath, LogoSize logoSize) diff --git a/gaseous-server/Classes/Metadata/PlatformVersions.cs b/gaseous-server/Classes/Metadata/PlatformVersions.cs index a15f6a0..39b98ff 100644 --- a/gaseous-server/Classes/Metadata/PlatformVersions.cs +++ b/gaseous-server/Classes/Metadata/PlatformVersions.cs @@ -71,13 +71,19 @@ namespace gaseous_server.Classes.Metadata { case Storage.CacheStatus.NotPresent: returnValue = await GetObjectFromServer(WhereClause); - Storage.NewCacheValue(returnValue); - UpdateSubClasses(ParentPlatform, returnValue); + if (returnValue != null) + { + Storage.NewCacheValue(returnValue); + UpdateSubClasses(ParentPlatform, returnValue); + } return returnValue; case Storage.CacheStatus.Expired: returnValue = await GetObjectFromServer(WhereClause); - Storage.NewCacheValue(returnValue, true); - UpdateSubClasses(ParentPlatform, returnValue); + if (returnValue != null) + { + Storage.NewCacheValue(returnValue, true); + UpdateSubClasses(ParentPlatform, returnValue); + } return returnValue; case Storage.CacheStatus.Current: return Storage.GetCacheValue(returnValue, "id", (long)searchValue); @@ -100,13 +106,20 @@ namespace gaseous_server.Classes.Metadata slug } - private static async Task GetObjectFromServer(string WhereClause) + private static async Task GetObjectFromServer(string WhereClause) { // get PlatformVersion metadata var results = await igdb.QueryAsync(IGDBClient.Endpoints.PlatformVersions, query: fieldList + " " + WhereClause + ";"); - var result = results.First(); + if (results.Length > 0) + { + var result = results.First(); - return result; + return result; + } + else + { + return null; + } } } } diff --git a/gaseous-server/Classes/Roms.cs b/gaseous-server/Classes/Roms.cs new file mode 100644 index 0000000..20f8a17 --- /dev/null +++ b/gaseous-server/Classes/Roms.cs @@ -0,0 +1,64 @@ +using System; +using System.Data; +using gaseous_tools; + +namespace gaseous_server.Classes +{ + public class Roms + { + public static RomItem GetRom(long RomId) + { + Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); + string sql = "SELECT * FROM games_roms WHERE id = @id"; + Dictionary dbDict = new Dictionary(); + dbDict.Add("id", RomId); + DataTable romDT = db.ExecuteCMD(sql, dbDict); + + if (romDT.Rows.Count > 0) + { + DataRow romDR = romDT.Rows[0]; + RomItem romItem = new RomItem + { + Id = (long)romDR["id"], + PlatformId = (long)romDR["platformid"], + GameId = (long)romDR["gameid"], + Name = (string)romDR["name"], + Size = (long)romDR["size"], + CRC = (string)romDR["crc"], + MD5 = (string)romDR["md5"], + SHA1 = (string)romDR["sha1"], + DevelopmentStatus = (string)romDR["developmentstatus"], + Flags = Newtonsoft.Json.JsonConvert.DeserializeObject((string)romDR["flags"]), + RomType = (int)romDR["romtype"], + RomTypeMedia = (string)romDR["romtypemedia"], + MediaLabel = (string)romDR["medialabel"], + Path = (string)romDR["path"] + }; + return romItem; + } + else + { + throw new Exception("Unknown ROM Id"); + } + } + + public class RomItem + { + public long Id { get; set; } + public long PlatformId { get; set; } + public long GameId { get; set; } + public string? Name { get; set; } + public long Size { get; set; } + public string? CRC { get; set; } + public string? MD5 { get; set; } + public string? SHA1 { get; set; } + public string? DevelopmentStatus { get; set; } + public string[]? Flags { get; set; } + public int RomType { get; set; } + public string? RomTypeMedia { get; set; } + public string? MediaLabel { get; set; } + public string? Path { get; set; } + } + } +} + diff --git a/gaseous-server/Program.cs b/gaseous-server/Program.cs index dd471da..92a3abb 100644 --- a/gaseous-server/Program.cs +++ b/gaseous-server/Program.cs @@ -53,6 +53,9 @@ app.MapControllers(); // setup library directories Config.LibraryConfiguration.InitLibrary(); +// organise library +gaseous_server.Classes.ImportGame.OrganiseLibrary(); + // add background tasks ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.SignatureIngestor, 60)); ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.TitleIngestor, 1));