diff --git a/gaseous-server/Classes/ImportGames.cs b/gaseous-server/Classes/ImportGames.cs index 5eae101..108951d 100644 --- a/gaseous-server/Classes/ImportGames.cs +++ b/gaseous-server/Classes/ImportGames.cs @@ -5,6 +5,7 @@ using System.Security.Policy; using System.Text.RegularExpressions; using System.Threading.Tasks; using gaseous_tools; +using IGDB.Models; using MySqlX.XDevAPI; using Org.BouncyCastle.Utilities.IO.Pem; using static gaseous_server.Classes.Metadata.Games; @@ -43,11 +44,7 @@ namespace gaseous_server.Classes string sql = ""; Dictionary dbDict = new Dictionary(); - string[] SkippableFiles = { - ".DS_STORE", - "desktop.ini" - }; - if (SkippableFiles.Contains(Path.GetFileName(GameFileImportPath), StringComparer.OrdinalIgnoreCase)) + if (Common.SkippableFiles.Contains(Path.GetFileName(GameFileImportPath), StringComparer.OrdinalIgnoreCase)) { Logging.Log(Logging.LogType.Debug, "Import Game", "Skipping item " + GameFileImportPath); } @@ -144,6 +141,17 @@ namespace gaseous_server.Classes if (zDiscoveredSignature.Score > discoveredSignature.Score) { + if ( + zDiscoveredSignature.Rom.SignatureSource == gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.MAMEArcade || + zDiscoveredSignature.Rom.SignatureSource == gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.MAMEMess + ) + { + zDiscoveredSignature.Rom.Name = zDiscoveredSignature.Game.Description + ".zip"; + } + zDiscoveredSignature.Rom.Crc = discoveredSignature.Rom.Crc; + zDiscoveredSignature.Rom.Md5 = discoveredSignature.Rom.Md5; + zDiscoveredSignature.Rom.Sha1 = discoveredSignature.Rom.Sha1; + zDiscoveredSignature.Rom.Size = discoveredSignature.Rom.Size; discoveredSignature = zDiscoveredSignature; break; @@ -238,7 +246,7 @@ namespace gaseous_server.Classes ri.Md5 = hash.md5hash; ri.Sha1 = hash.sha1hash; ri.Size = fi.Length; - ri.SignatureSource = Models.Signatures_Games.RomItem.SignatureSourceType.None; + ri.SignatureSource = gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None; } } @@ -275,6 +283,17 @@ namespace gaseous_server.Classes else if (games.Length > 0) { Logging.Log(Logging.LogType.Information, "Import Game", " " + games.Length + " search results found"); + + // quite likely we've found sequels and alternate types + foreach (Game game in games) { + if (game.Name == SearchCandidate) { + // found game title matches the search candidate + determinedGame = Metadata.Games.GetGame((long)games[0].Id, false, false, false); + Logging.Log(Logging.LogType.Information, "Import Game", "Found exact match!"); + GameFound = true; + break; + } + } } else { @@ -340,11 +359,17 @@ namespace gaseous_server.Classes GameName = Regex.Replace(GameName, @"v(\d+\.)?(\d+\.)?(\*|\d+)$", "").Trim(); GameName = Regex.Replace(GameName, @"Rev (\d+\.)?(\d+\.)?(\*|\d+)$", "").Trim(); + // assumption: no games have () in their titles so we'll remove them + int idx = GameName.IndexOf('('); + if (idx >= 0) { + GameName = GameName.Substring(0, idx); + } + List SearchCandidates = new List(); - SearchCandidates.Add(GameName); + SearchCandidates.Add(GameName.Trim()); if (GameName.Contains(" - ")) { - SearchCandidates.Add(GameName.Replace(" - ", ": ")); + SearchCandidates.Add(GameName.Replace(" - ", ": ").Trim()); SearchCandidates.Add(GameName.Substring(0, GameName.IndexOf(" - ")).Trim()); } if (GameName.Contains(": ")) @@ -367,10 +392,10 @@ namespace gaseous_server.Classes if (UpdateId == 0) { - sql = "INSERT INTO Games_Roms (PlatformId, GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel, Path, MetadataSource) VALUES (@platformid, @gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @path, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; + sql = "INSERT INTO Games_Roms (PlatformId, GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Attributes, RomType, RomTypeMedia, MediaLabel, Path, MetadataSource, MetadataGameName, MetadataVersion) VALUES (@platformid, @gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @Attributes, @romtype, @romtypemedia, @medialabel, @path, @metadatasource, @metadatagamename, @metadataversion); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; } else { - sql = "UPDATE Games_Roms SET PlatformId=platformid, GameId=@gameid, Name=@name, Size=@size, DevelopmentStatus=@developmentstatus, Flags=@flags, RomType=@romtype, RomTypeMedia=@romtypemedia, MediaLabel=@medialabel, MetadataSource=@metadatasource WHERE Id=@id;"; + sql = "UPDATE Games_Roms SET PlatformId=platformid, GameId=@gameid, Name=@name, Size=@size, DevelopmentStatus=@developmentstatus, Attributes=@Attributes, RomType=@romtype, RomTypeMedia=@romtypemedia, MediaLabel=@medialabel, MetadataSource=@metadatasource, MetadataGameName=@metadatagamename, MetadataVersion=@metadataversion WHERE Id=@id;"; dbDict.Add("id", UpdateId); } dbDict.Add("platformid", Common.ReturnValueIfNull(determinedPlatform.Id, 0)); @@ -382,21 +407,23 @@ namespace gaseous_server.Classes dbDict.Add("crc", Common.ReturnValueIfNull(discoveredSignature.Rom.Crc, "")); dbDict.Add("developmentstatus", Common.ReturnValueIfNull(discoveredSignature.Rom.DevelopmentStatus, "")); dbDict.Add("metadatasource", discoveredSignature.Rom.SignatureSource); + dbDict.Add("metadatagamename", discoveredSignature.Game.Name); + dbDict.Add("metadataversion", 2); - if (discoveredSignature.Rom.flags != null) + if (discoveredSignature.Rom.Attributes != null) { - if (discoveredSignature.Rom.flags.Count > 0) + if (discoveredSignature.Rom.Attributes.Count > 0) { - dbDict.Add("flags", Newtonsoft.Json.JsonConvert.SerializeObject(discoveredSignature.Rom.flags)); + dbDict.Add("attributes", Newtonsoft.Json.JsonConvert.SerializeObject(discoveredSignature.Rom.Attributes)); } else { - dbDict.Add("flags", "[ ]"); + dbDict.Add("attributes", "[ ]"); } } else { - dbDict.Add("flags", "[ ]"); + dbDict.Add("attributes", "[ ]"); } dbDict.Add("romtype", (int)discoveredSignature.Rom.RomType); dbDict.Add("romtypemedia", Common.ReturnValueIfNull(discoveredSignature.Rom.RomTypeMedia, "")); @@ -572,43 +599,46 @@ namespace gaseous_server.Classes string[] LibraryFiles = Directory.GetFiles(Config.LibraryConfiguration.LibraryDataDirectory, "*.*", SearchOption.AllDirectories); foreach (string LibraryFile in LibraryFiles) { - Common.hashObject LibraryFileHash = new Common.hashObject(LibraryFile); - - // check if file is in database - bool romFound = false; - for (var i = 0; i < dtRoms.Rows.Count; i++) + if (!Common.SkippableFiles.Contains(Path.GetFileName(LibraryFile), StringComparer.OrdinalIgnoreCase)) { - long romId = (long)dtRoms.Rows[i]["Id"]; - string romPath = (string)dtRoms.Rows[i]["Path"]; - string romMd5 = (string)dtRoms.Rows[i]["MD5"]; + Common.hashObject LibraryFileHash = new Common.hashObject(LibraryFile); - if ((LibraryFile == romPath) || (LibraryFileHash.md5hash == romMd5)) + // check if file is in database + bool romFound = false; + for (var i = 0; i < dtRoms.Rows.Count; i++) { - romFound = true; - break; - } - } + long romId = (long)dtRoms.Rows[i]["Id"]; + string romPath = (string)dtRoms.Rows[i]["Path"]; + string romMd5 = (string)dtRoms.Rows[i]["MD5"]; - if (romFound == false) - { - // file is not in database - process it - Common.hashObject hash = new Common.hashObject(LibraryFile); - FileInfo fi = new FileInfo(LibraryFile); - - Models.Signatures_Games sig = GetFileSignature(hash, fi, LibraryFile); - - Logging.Log(Logging.LogType.Information, "Library Scan", " Orphaned file found in library: " + LibraryFile); - - // get discovered platform - IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(sig.Flags.IGDBPlatformId); - if (determinedPlatform == null) - { - determinedPlatform = new IGDB.Models.Platform(); + if ((LibraryFile == romPath) || (LibraryFileHash.md5hash == romMd5)) + { + romFound = true; + break; + } } - IGDB.Models.Game determinedGame = SearchForGame(sig.Game.Name, sig.Flags.IGDBPlatformId); + if (romFound == false) + { + // file is not in database - process it + Common.hashObject hash = new Common.hashObject(LibraryFile); + FileInfo fi = new FileInfo(LibraryFile); - StoreROM(hash, determinedGame, determinedPlatform, sig, LibraryFile); + Models.Signatures_Games sig = GetFileSignature(hash, fi, LibraryFile); + + Logging.Log(Logging.LogType.Information, "Library Scan", " Orphaned file found in library: " + LibraryFile); + + // get discovered platform + IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(sig.Flags.IGDBPlatformId); + if (determinedPlatform == null) + { + determinedPlatform = new IGDB.Models.Platform(); + } + + IGDB.Models.Game determinedGame = SearchForGame(sig.Game.Name, sig.Flags.IGDBPlatformId); + + StoreROM(hash, determinedGame, determinedPlatform, sig, LibraryFile); + } } } @@ -623,24 +653,27 @@ namespace gaseous_server.Classes { long romId = (long)dtRoms.Rows[i]["Id"]; string romPath = (string)dtRoms.Rows[i]["Path"]; - Classes.Roms.GameRomItem.SourceType romMetadataSource = (Classes.Roms.GameRomItem.SourceType)(int)dtRoms.Rows[i]["MetadataSource"]; + gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType romMetadataSource = (gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType)(int)dtRoms.Rows[i]["MetadataSource"]; Logging.Log(Logging.LogType.Information, "Library Scan", " Processing ROM at path " + romPath); if (File.Exists(romPath)) { // file exists, so lets check to make sure the signature was matched, and update if a signature can be found - if (romMetadataSource == Roms.GameRomItem.SourceType.None) + if ( + romMetadataSource == gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None || + (int)dtRoms.Rows[i]["MetadataVersion"] == 1 + ) { Common.hashObject hash = new Common.hashObject { - md5hash = "", - sha1hash = "" + md5hash = (string)dtRoms.Rows[i]["MD5"], + sha1hash = (string)dtRoms.Rows[i]["SHA1"] }; FileInfo fi = new FileInfo(romPath); Models.Signatures_Games sig = GetFileSignature(hash, fi, romPath); - if (sig.Rom.SignatureSource != Models.Signatures_Games.RomItem.SignatureSourceType.None) + if (sig.Rom.SignatureSource != gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None) { Logging.Log(Logging.LogType.Information, "Library Scan", " Update signature found for " + romPath); diff --git a/gaseous-server/Classes/Roms.cs b/gaseous-server/Classes/Roms.cs index 89dc647..143a987 100644 --- a/gaseous-server/Classes/Roms.cs +++ b/gaseous-server/Classes/Roms.cs @@ -95,7 +95,7 @@ namespace gaseous_server.Classes private static GameRomItem BuildRom(DataRow romDR) { - GameRomItem romItem = new GameRomItem + GameRomItem romItem = new GameRomItem { Id = (long)romDR["id"], PlatformId = (long)romDR["platformid"], @@ -107,12 +107,13 @@ namespace gaseous_server.Classes MD5 = (string)romDR["md5"], SHA1 = (string)romDR["sha1"], DevelopmentStatus = (string)romDR["developmentstatus"], - Flags = Newtonsoft.Json.JsonConvert.DeserializeObject((string)romDR["flags"]), + Attributes = Newtonsoft.Json.JsonConvert.DeserializeObject>>((string)Common.ReturnValueIfNull(romDR["attributes"], "[ ]")), RomType = (int)romDR["romtype"], RomTypeMedia = (string)romDR["romtypemedia"], MediaLabel = (string)romDR["medialabel"], Path = (string)romDR["path"], - Source = (GameRomItem.SourceType)(Int32)romDR["metadatasource"] + Source = (gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType)(Int32)romDR["metadatasource"], + SignatureSourceGameTitle = (string)Common.ReturnValueIfNull(romDR["MetadataGameName"], "") }; // check for a web emulator and update the romItem @@ -145,17 +146,13 @@ namespace gaseous_server.Classes public string? SHA1 { get; set; } public string? DevelopmentStatus { get; set; } public string[]? Flags { get; set; } + public List>? Attributes { get; set;} public int RomType { get; set; } public string? RomTypeMedia { get; set; } public string? MediaLabel { get; set; } public string? Path { get; set; } - public SourceType Source { get; set; } - - public enum SourceType - { - None = 0, - TOSEC = 1 - } + public gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType Source { get; set; } + public string? SignatureSourceGameTitle { get; set;} } } } diff --git a/gaseous-server/Classes/SignatureIngestors/TOSEC.cs b/gaseous-server/Classes/SignatureIngestors/TOSEC.cs deleted file mode 100644 index 54c4319..0000000 --- a/gaseous-server/Classes/SignatureIngestors/TOSEC.cs +++ /dev/null @@ -1,234 +0,0 @@ -using System; -using System.IO; -using MySql.Data.MySqlClient; -using gaseous_signature_parser.models.RomSignatureObject; -using gaseous_signature_parser.classes.parsers; -using gaseous_tools; -using MySqlX.XDevAPI; - -namespace gaseous_server.SignatureIngestors.TOSEC -{ - public class TOSECIngestor - { - public void Import(string SearchPath) - { - // connect to database - Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); - - // process provided files - Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing from " + SearchPath); - if (Directory.Exists(Config.LibraryConfiguration.LibrarySignatureImportDirectory_TOSEC)) - { - string[] tosecPathContents = Directory.GetFiles(SearchPath); - Array.Sort(tosecPathContents); - - string sql = ""; - Dictionary dbDict = new Dictionary(); - System.Data.DataTable sigDB; - - for (UInt16 i = 0; i < tosecPathContents.Length; ++i) - { - string tosecXMLFile = tosecPathContents[i]; - - // check tosec file md5 - Common.hashObject hashObject = new Common.hashObject(tosecXMLFile); - sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5"; - dbDict = new Dictionary(); - dbDict.Add("sourcemd5", hashObject.md5hash); - sigDB = db.ExecuteCMD(sql, dbDict); - - if (sigDB.Rows.Count == 0) - { - try - { - Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing file: " + tosecXMLFile); - - // start parsing file - TosecParser tosecParser = new TosecParser(); - RomSignatureObject tosecObject = tosecParser.Parse(tosecXMLFile); - - // store in database - - // store source object - bool processGames = false; - if (tosecObject.SourceMd5 != null) - { - sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5"; - dbDict = new Dictionary(); - dbDict.Add("name", Common.ReturnValueIfNull(tosecObject.Name, "")); - dbDict.Add("description", Common.ReturnValueIfNull(tosecObject.Description, "")); - dbDict.Add("category", Common.ReturnValueIfNull(tosecObject.Category, "")); - dbDict.Add("version", Common.ReturnValueIfNull(tosecObject.Version, "")); - dbDict.Add("author", Common.ReturnValueIfNull(tosecObject.Author, "")); - dbDict.Add("email", Common.ReturnValueIfNull(tosecObject.Email, "")); - dbDict.Add("homepage", Common.ReturnValueIfNull(tosecObject.Homepage, "")); - dbDict.Add("uri", Common.ReturnValueIfNull(tosecObject.Url, "")); - dbDict.Add("sourcetype", Common.ReturnValueIfNull(tosecObject.SourceType, "")); - dbDict.Add("sourcemd5", tosecObject.SourceMd5); - dbDict.Add("sourcesha1", tosecObject.SourceSHA1); - - sigDB = db.ExecuteCMD(sql, dbDict); - if (sigDB.Rows.Count == 0) - { - // entry not present, insert it - sql = "INSERT INTO Signatures_Sources (Name, Description, Category, Version, Author, Email, Homepage, Url, SourceType, SourceMD5, SourceSHA1) VALUES (@name, @description, @category, @version, @author, @email, @homepage, @uri, @sourcetype, @sourcemd5, @sourcesha1)"; - - db.ExecuteCMD(sql, dbDict); - - processGames = true; - } - - if (processGames == true) - { - for (int x = 0; x < tosecObject.Games.Count; ++x) - { - RomSignatureObject.Game gameObject = tosecObject.Games[x]; - - // set up game dictionary - dbDict = new Dictionary(); - dbDict.Add("name", Common.ReturnValueIfNull(gameObject.Name, "")); - dbDict.Add("description", Common.ReturnValueIfNull(gameObject.Description, "")); - dbDict.Add("year", Common.ReturnValueIfNull(gameObject.Year, "")); - dbDict.Add("publisher", Common.ReturnValueIfNull(gameObject.Publisher, "")); - dbDict.Add("demo", (int)gameObject.Demo); - dbDict.Add("system", Common.ReturnValueIfNull(gameObject.System, "")); - dbDict.Add("platform", Common.ReturnValueIfNull(gameObject.System, "")); - dbDict.Add("systemvariant", Common.ReturnValueIfNull(gameObject.SystemVariant, "")); - dbDict.Add("video", Common.ReturnValueIfNull(gameObject.Video, "")); - dbDict.Add("country", Common.ReturnValueIfNull(gameObject.Country, "")); - dbDict.Add("language", Common.ReturnValueIfNull(gameObject.Language, "")); - dbDict.Add("copyright", Common.ReturnValueIfNull(gameObject.Copyright, "")); - - // store platform - int gameSystem = 0; - if (gameObject.System != null) - { - sql = "SELECT Id FROM Signatures_Platforms WHERE Platform=@platform"; - - sigDB = db.ExecuteCMD(sql, dbDict); - if (sigDB.Rows.Count == 0) - { - // entry not present, insert it - sql = "INSERT INTO Signatures_Platforms (Platform) VALUES (@platform); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; - sigDB = db.ExecuteCMD(sql, dbDict); - - gameSystem = Convert.ToInt32(sigDB.Rows[0][0]); - } - else - { - gameSystem = (int)sigDB.Rows[0][0]; - } - } - dbDict.Add("systemid", gameSystem); - - // store publisher - int gamePublisher = 0; - if (gameObject.Publisher != null) - { - sql = "SELECT * FROM Signatures_Publishers WHERE Publisher=@publisher"; - - sigDB = db.ExecuteCMD(sql, dbDict); - if (sigDB.Rows.Count == 0) - { - // entry not present, insert it - sql = "INSERT INTO Signatures_Publishers (Publisher) VALUES (@publisher); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; - sigDB = db.ExecuteCMD(sql, dbDict); - gamePublisher = Convert.ToInt32(sigDB.Rows[0][0]); - } - else - { - gamePublisher = (int)sigDB.Rows[0][0]; - } - } - dbDict.Add("publisherid", gamePublisher); - - // store game - int gameId = 0; - sql = "SELECT * FROM Signatures_Games WHERE Name=@name AND Year=@year AND Publisherid=@publisher AND Systemid=@systemid AND Country=@country AND Language=@language"; - - sigDB = db.ExecuteCMD(sql, dbDict); - if (sigDB.Rows.Count == 0) - { - // entry not present, insert it - sql = "INSERT INTO Signatures_Games " + - "(Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " + - "(@name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; - sigDB = db.ExecuteCMD(sql, dbDict); - - gameId = Convert.ToInt32(sigDB.Rows[0][0]); - } - else - { - gameId = (int)sigDB.Rows[0][0]; - } - - // store rom - foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms) - { - if (romObject.Md5 != null) - { - int romId = 0; - sql = "SELECT * FROM Signatures_Roms WHERE GameId=@gameid AND MD5=@md5"; - dbDict = new Dictionary(); - dbDict.Add("gameid", gameId); - dbDict.Add("name", Common.ReturnValueIfNull(romObject.Name, "")); - dbDict.Add("size", Common.ReturnValueIfNull(romObject.Size, "")); - dbDict.Add("crc", Common.ReturnValueIfNull(romObject.Crc, "")); - dbDict.Add("md5", romObject.Md5); - dbDict.Add("sha1", Common.ReturnValueIfNull(romObject.Sha1, "")); - dbDict.Add("developmentstatus", Common.ReturnValueIfNull(romObject.DevelopmentStatus, "")); - - if (romObject.flags != null) - { - if (romObject.flags.Count > 0) - { - dbDict.Add("flags", Newtonsoft.Json.JsonConvert.SerializeObject(romObject.flags)); - } - else - { - dbDict.Add("flags", "[ ]"); - } - } - else - { - dbDict.Add("flags", "[ ]"); - } - dbDict.Add("romtype", (int)romObject.RomType); - dbDict.Add("romtypemedia", Common.ReturnValueIfNull(romObject.RomTypeMedia, "")); - dbDict.Add("medialabel", Common.ReturnValueIfNull(romObject.MediaLabel, "")); - dbDict.Add("metadatasource", Classes.Roms.GameRomItem.SourceType.TOSEC); - - sigDB = db.ExecuteCMD(sql, dbDict); - if (sigDB.Rows.Count == 0) - { - // entry not present, insert it - sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel, MetadataSource) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; - sigDB = db.ExecuteCMD(sql, dbDict); - - - romId = Convert.ToInt32(sigDB.Rows[0][0]); - } - else - { - romId = (int)sigDB.Rows[0][0]; - } - } - } - } - } - } - } - catch (Exception ex) - { - Logging.Log(Logging.LogType.Warning, "Signature Ingestor - TOSEC", "Invalid import file: " + tosecXMLFile, ex); - } - } - else - { - Logging.Log(Logging.LogType.Debug, "Signature Ingestor - TOSEC", "Rejecting already imported file: " + tosecXMLFile); - } - } - } - } - } -} \ No newline at end of file diff --git a/gaseous-server/Classes/SignatureIngestors/XML.cs b/gaseous-server/Classes/SignatureIngestors/XML.cs new file mode 100644 index 0000000..e30d83c --- /dev/null +++ b/gaseous-server/Classes/SignatureIngestors/XML.cs @@ -0,0 +1,344 @@ +using System; +using System.IO; +using MySql.Data.MySqlClient; +using gaseous_signature_parser.models.RomSignatureObject; +using gaseous_tools; +using MySqlX.XDevAPI; +using System.Data; + +namespace gaseous_server.SignatureIngestors.XML +{ + public class XMLIngestor + { + public void Import(string SearchPath, gaseous_signature_parser.parser.SignatureParser XMLType) + { + // connect to database + Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); + + // process provided files + Logging.Log(Logging.LogType.Information, "Signature Ingestor - XML", "Importing from " + SearchPath); + if (!Directory.Exists(SearchPath)) + { + Directory.CreateDirectory(SearchPath); + } + + string[] PathContents = Directory.GetFiles(SearchPath); + Array.Sort(PathContents); + + string sql = ""; + Dictionary dbDict = new Dictionary(); + System.Data.DataTable sigDB; + + for (UInt16 i = 0; i < PathContents.Length; ++i) + { + string XMLFile = PathContents[i]; + + // check xml file md5 + Common.hashObject hashObject = new Common.hashObject(XMLFile); + sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5"; + dbDict = new Dictionary(); + dbDict.Add("sourcemd5", hashObject.md5hash); + sigDB = db.ExecuteCMD(sql, dbDict); + + if (sigDB.Rows.Count == 0) + { + try + { + Logging.Log(Logging.LogType.Information, "Signature Ingestor - XML", "Importing file: " + XMLFile); + + // start parsing file + gaseous_signature_parser.parser Parser = new gaseous_signature_parser.parser(); + RomSignatureObject Object = Parser.ParseSignatureDAT(XMLFile, XMLType); + + // store in database + string[] flipNameAndDescription = { + "MAMEArcade", + "MAMEMess" + }; + + // store source object + bool processGames = false; + if (Object.SourceMd5 != null) + { + sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5"; + dbDict = new Dictionary(); + dbDict.Add("name", Common.ReturnValueIfNull(Object.Name, "")); + dbDict.Add("description", Common.ReturnValueIfNull(Object.Description, "")); + dbDict.Add("category", Common.ReturnValueIfNull(Object.Category, "")); + dbDict.Add("version", Common.ReturnValueIfNull(Object.Version, "")); + dbDict.Add("author", Common.ReturnValueIfNull(Object.Author, "")); + dbDict.Add("email", Common.ReturnValueIfNull(Object.Email, "")); + dbDict.Add("homepage", Common.ReturnValueIfNull(Object.Homepage, "")); + dbDict.Add("uri", Common.ReturnValueIfNull(Object.Url, "")); + dbDict.Add("sourcetype", Common.ReturnValueIfNull(Object.SourceType, "")); + dbDict.Add("sourcemd5", Object.SourceMd5); + dbDict.Add("sourcesha1", Object.SourceSHA1); + + sigDB = db.ExecuteCMD(sql, dbDict); + if (sigDB.Rows.Count == 0) + { + // entry not present, insert it + sql = "INSERT INTO Signatures_Sources (Name, Description, Category, Version, Author, Email, Homepage, Url, SourceType, SourceMD5, SourceSHA1) VALUES (@name, @description, @category, @version, @author, @email, @homepage, @uri, @sourcetype, @sourcemd5, @sourcesha1)"; + + db.ExecuteCMD(sql, dbDict); + + processGames = true; + } + + if (processGames == true) + { + for (int x = 0; x < Object.Games.Count; ++x) + { + RomSignatureObject.Game gameObject = Object.Games[x]; + + // set up game dictionary + dbDict = new Dictionary(); + if (flipNameAndDescription.Contains(Object.SourceType)) + { + dbDict.Add("name", Common.ReturnValueIfNull(gameObject.Description, "")); + dbDict.Add("description", Common.ReturnValueIfNull(gameObject.Name, "")); + } + else + { + dbDict.Add("name", Common.ReturnValueIfNull(gameObject.Name, "")); + dbDict.Add("description", Common.ReturnValueIfNull(gameObject.Description, "")); + } + dbDict.Add("year", Common.ReturnValueIfNull(gameObject.Year, "")); + dbDict.Add("publisher", Common.ReturnValueIfNull(gameObject.Publisher, "")); + dbDict.Add("demo", (int)gameObject.Demo); + dbDict.Add("system", Common.ReturnValueIfNull(gameObject.System, "")); + dbDict.Add("platform", Common.ReturnValueIfNull(gameObject.System, "")); + dbDict.Add("systemvariant", Common.ReturnValueIfNull(gameObject.SystemVariant, "")); + dbDict.Add("video", Common.ReturnValueIfNull(gameObject.Video, "")); + dbDict.Add("country", Common.ReturnValueIfNull(gameObject.Country, "")); + dbDict.Add("language", Common.ReturnValueIfNull(gameObject.Language, "")); + dbDict.Add("copyright", Common.ReturnValueIfNull(gameObject.Copyright, "")); + + // store platform + int gameSystem = 0; + if (gameObject.System != null) + { + sql = "SELECT Id FROM Signatures_Platforms WHERE Platform=@platform"; + + sigDB = db.ExecuteCMD(sql, dbDict); + if (sigDB.Rows.Count == 0) + { + // entry not present, insert it + sql = "INSERT INTO Signatures_Platforms (Platform) VALUES (@platform); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; + sigDB = db.ExecuteCMD(sql, dbDict); + + gameSystem = Convert.ToInt32(sigDB.Rows[0][0]); + } + else + { + gameSystem = (int)sigDB.Rows[0][0]; + } + } + dbDict.Add("systemid", gameSystem); + + // store publisher + int gamePublisher = 0; + if (gameObject.Publisher != null) + { + sql = "SELECT * FROM Signatures_Publishers WHERE Publisher=@publisher"; + + sigDB = db.ExecuteCMD(sql, dbDict); + if (sigDB.Rows.Count == 0) + { + // entry not present, insert it + sql = "INSERT INTO Signatures_Publishers (Publisher) VALUES (@publisher); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; + sigDB = db.ExecuteCMD(sql, dbDict); + gamePublisher = Convert.ToInt32(sigDB.Rows[0][0]); + } + else + { + gamePublisher = (int)sigDB.Rows[0][0]; + } + } + dbDict.Add("publisherid", gamePublisher); + + // store game + int gameId = 0; + sql = "SELECT * FROM Signatures_Games WHERE Name=@name AND Year=@year AND Publisherid=@publisher AND Systemid=@systemid AND Country=@country AND Language=@language"; + + sigDB = db.ExecuteCMD(sql, dbDict); + if (sigDB.Rows.Count == 0) + { + // entry not present, insert it + sql = "INSERT INTO Signatures_Games " + + "(Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " + + "(@name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; + sigDB = db.ExecuteCMD(sql, dbDict); + + gameId = Convert.ToInt32(sigDB.Rows[0][0]); + } + else + { + gameId = (int)sigDB.Rows[0][0]; + } + + // store rom + foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms) + { + if (romObject.Md5 != null || romObject.Sha1 != null) + { + int romId = 0; + sql = "SELECT * FROM Signatures_Roms WHERE GameId=@gameid AND MD5=@md5"; + dbDict = new Dictionary(); + dbDict.Add("gameid", gameId); + dbDict.Add("name", Common.ReturnValueIfNull(romObject.Name, "")); + dbDict.Add("size", Common.ReturnValueIfNull(romObject.Size, "")); + dbDict.Add("crc", Common.ReturnValueIfNull(romObject.Crc, "")); + dbDict.Add("md5", Common.ReturnValueIfNull(romObject.Md5, "")); + dbDict.Add("sha1", Common.ReturnValueIfNull(romObject.Sha1, "")); + dbDict.Add("developmentstatus", Common.ReturnValueIfNull(romObject.DevelopmentStatus, "")); + + if (romObject.Attributes != null) + { + if (romObject.Attributes.Count > 0) + { + dbDict.Add("attributes", Newtonsoft.Json.JsonConvert.SerializeObject(romObject.Attributes)); + } + else + { + dbDict.Add("attributes", "[ ]"); + } + } + else + { + dbDict.Add("attributes", "[ ]"); + } + dbDict.Add("romtype", (int)romObject.RomType); + dbDict.Add("romtypemedia", Common.ReturnValueIfNull(romObject.RomTypeMedia, "")); + dbDict.Add("medialabel", Common.ReturnValueIfNull(romObject.MediaLabel, "")); + dbDict.Add("metadatasource", romObject.SignatureSource); + + sigDB = db.ExecuteCMD(sql, dbDict); + if (sigDB.Rows.Count == 0) + { + // entry not present, insert it + sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Attributes, RomType, RomTypeMedia, MediaLabel, MetadataSource) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @attributes, @romtype, @romtypemedia, @medialabel, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; + sigDB = db.ExecuteCMD(sql, dbDict); + + + romId = Convert.ToInt32(sigDB.Rows[0][0]); + } + else + { + romId = (int)sigDB.Rows[0][0]; + } + } + } + } + } + } + } + catch (Exception ex) + { + Logging.Log(Logging.LogType.Warning, "Signature Ingestor - XML", "Invalid import file: " + XMLFile, ex); + } + } + else + { + Logging.Log(Logging.LogType.Debug, "Signature Ingestor - XML", "Rejecting already imported file: " + XMLFile); + } + } + } + + public void MigrateMetadatVersion() { + Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); + string sql = ""; + Dictionary dbDict = new Dictionary(); + + // update signature roms to v2 + sql = "SELECT Id, Flags, Attributes, IngestorVersion FROM Signatures_Roms WHERE IngestorVersion = 1"; + DataTable data = db.ExecuteCMD(sql); + if (data.Rows.Count > 0) + { + Logging.Log(Logging.LogType.Information, "Signature Ingestor - Database Update", "Updating " + data.Rows.Count + " database entries"); + int Counter = 0; + int LastCounterCheck = 0; + foreach (DataRow row in data.Rows) + { + List Flags = Newtonsoft.Json.JsonConvert.DeserializeObject>((string)Common.ReturnValueIfNull(row["flags"], "[]")); + List> Attributes = new List>(); + foreach (string Flag in Flags) + { + if (Flag.StartsWith("a")) + { + Attributes.Add( + new KeyValuePair( + "a", + Flag + ) + ); + } + else + { + string[] FlagCompare = Flag.Split(' '); + switch (FlagCompare[0].Trim().ToLower()) + { + case "cr": + // cracked + case "f": + // fixed + case "h": + // hacked + case "m": + // modified + case "p": + // pirated + case "t": + // trained + case "tr": + // translated + case "o": + // overdump + case "u": + // underdump + case "v": + // virus + case "b": + // bad dump + case "a": + // alternate + case "!": + // known verified dump + // ------------------- + string shavedToken = Flag.Substring(FlagCompare[0].Trim().Length).Trim(); + Attributes.Add(new KeyValuePair( + FlagCompare[0].Trim().ToLower(), + shavedToken + )); + break; + } + } + } + + string AttributesJson; + if (Attributes.Count > 0) + { + AttributesJson = Newtonsoft.Json.JsonConvert.SerializeObject(Attributes); + } + else + { + AttributesJson = "[]"; + } + + string updateSQL = "UPDATE Signatures_Roms SET Attributes=@attributes, IngestorVersion=2 WHERE Id=@id"; + dbDict = new Dictionary(); + dbDict.Add("attributes", AttributesJson); + dbDict.Add("id", (int)row["Id"]); + db.ExecuteCMD(updateSQL, dbDict); + + if ((Counter - LastCounterCheck) > 10) + { + LastCounterCheck = Counter; + Logging.Log(Logging.LogType.Information, "Signature Ingestor - Database Update", "Updating " + Counter + " / " + data.Rows.Count + " database entries"); + } + Counter += 1; + } + } + } + } +} \ No newline at end of file diff --git a/gaseous-server/Controllers/SignaturesController.cs b/gaseous-server/Controllers/SignaturesController.cs index f59173a..89b720a 100644 --- a/gaseous-server/Controllers/SignaturesController.cs +++ b/gaseous-server/Controllers/SignaturesController.cs @@ -55,7 +55,7 @@ namespace gaseous_server.Controllers private List _GetSignature(string sqlWhere, string searchString) { Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); - string sql = "SELECT view_Signatures_Games.*, Signatures_Roms.Id AS romid, Signatures_Roms.Name AS romname, Signatures_Roms.Size, Signatures_Roms.CRC, Signatures_Roms.MD5, Signatures_Roms.SHA1, Signatures_Roms.DevelopmentStatus, Signatures_Roms.Flags, Signatures_Roms.RomType, Signatures_Roms.RomTypeMedia, Signatures_Roms.MediaLabel, Signatures_Roms.MetadataSource FROM Signatures_Roms INNER JOIN view_Signatures_Games ON Signatures_Roms.GameId = view_Signatures_Games.Id WHERE " + sqlWhere; + string sql = "SELECT view_Signatures_Games.*, Signatures_Roms.Id AS romid, Signatures_Roms.Name AS romname, Signatures_Roms.Size, Signatures_Roms.CRC, Signatures_Roms.MD5, Signatures_Roms.SHA1, Signatures_Roms.DevelopmentStatus, Signatures_Roms.Attributes, Signatures_Roms.RomType, Signatures_Roms.RomTypeMedia, Signatures_Roms.MediaLabel, Signatures_Roms.MetadataSource FROM Signatures_Roms INNER JOIN view_Signatures_Games ON Signatures_Roms.GameId = view_Signatures_Games.Id WHERE " + sqlWhere; Dictionary dbDict = new Dictionary(); dbDict.Add("searchString", searchString); @@ -91,11 +91,11 @@ namespace gaseous_server.Controllers Md5 = (string)sigDbRow["MD5"], Sha1 = (string)sigDbRow["SHA1"], DevelopmentStatus = (string)sigDbRow["DevelopmentStatus"], - flags = Newtonsoft.Json.JsonConvert.DeserializeObject>((string)sigDbRow["Flags"]), + Attributes = Newtonsoft.Json.JsonConvert.DeserializeObject>>((string)Common.ReturnValueIfNull(sigDbRow["Attributes"], "[]")), RomType = (Models.Signatures_Games.RomItem.RomTypes)(int)sigDbRow["RomType"], RomTypeMedia = (string)sigDbRow["RomTypeMedia"], MediaLabel = (string)sigDbRow["MediaLabel"], - SignatureSource = (Models.Signatures_Games.RomItem.SignatureSourceType)(Int32)sigDbRow["MetadataSource"] + SignatureSource = (gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType)(Int32)sigDbRow["MetadataSource"] } }; GamesList.Add(gameItem); diff --git a/gaseous-server/Models/PlatformMapping.cs b/gaseous-server/Models/PlatformMapping.cs index 5c7d221..10e4c3f 100644 --- a/gaseous-server/Models/PlatformMapping.cs +++ b/gaseous-server/Models/PlatformMapping.cs @@ -56,7 +56,10 @@ namespace gaseous_server.Models { foreach (Models.PlatformMapping.PlatformMapItem PlatformMapping in Models.PlatformMapping.PlatformMap) { - if (PlatformMapping.AlternateNames.Contains(Signature.Game.System, StringComparer.OrdinalIgnoreCase)) + if ( + PlatformMapping.IGDBName == Signature.Game.System || + PlatformMapping.AlternateNames.Contains(Signature.Game.System, StringComparer.OrdinalIgnoreCase) + ) { if (SetSystemName == true) { diff --git a/gaseous-server/Models/Signatures_Games.cs b/gaseous-server/Models/Signatures_Games.cs index 33e83ef..87ff858 100644 --- a/gaseous-server/Models/Signatures_Games.cs +++ b/gaseous-server/Models/Signatures_Games.cs @@ -126,19 +126,13 @@ namespace gaseous_server.Models public string? DevelopmentStatus { get; set; } - public List flags { get; set; } = new List(); + public List> Attributes { get; set; } = new List>(); public RomTypes RomType { get; set; } public string? RomTypeMedia { get; set; } public string? MediaLabel { get; set; } - public SignatureSourceType SignatureSource { get; set; } - - public enum SignatureSourceType - { - None = 0, - TOSEC = 1 - } + public gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType SignatureSource { get; set; } public enum RomTypes { @@ -197,6 +191,7 @@ namespace gaseous_server.Models case "crc": case "developmentstatus": case "flags": + case "attributes": case "romtypemedia": case "medialabel": if (prop.PropertyType == typeof(string) || prop.PropertyType == typeof(Int64) || prop.PropertyType == typeof(List)) diff --git a/gaseous-server/ProcessQueue.cs b/gaseous-server/ProcessQueue.cs index faf92ce..b5164f9 100644 --- a/gaseous-server/ProcessQueue.cs +++ b/gaseous-server/ProcessQueue.cs @@ -76,8 +76,17 @@ namespace gaseous_server { case QueueItemType.SignatureIngestor: Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Signature Ingestor"); - SignatureIngestors.TOSEC.TOSECIngestor tIngest = new SignatureIngestors.TOSEC.TOSECIngestor(); - tIngest.Import(Config.LibraryConfiguration.LibrarySignatureImportDirectory_TOSEC); + SignatureIngestors.XML.XMLIngestor tIngest = new SignatureIngestors.XML.XMLIngestor(); + + Logging.Log(Logging.LogType.Information, "Signature Import", "Processing TOSEC files"); + tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "TOSEC"), gaseous_signature_parser.parser.SignatureParser.TOSEC); + + Logging.Log(Logging.LogType.Information, "Signature Import", "Processing MAME Arcade files"); + tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME Arcade"), gaseous_signature_parser.parser.SignatureParser.MAMEArcade); + + Logging.Log(Logging.LogType.Information, "Signature Import", "Processing MAME MESS files"); + tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME MESS"), gaseous_signature_parser.parser.SignatureParser.MAMEMess); + break; case QueueItemType.TitleIngestor: diff --git a/gaseous-server/Program.cs b/gaseous-server/Program.cs index d3603f2..88d0924 100644 --- a/gaseous-server/Program.cs +++ b/gaseous-server/Program.cs @@ -1,6 +1,7 @@ using System.Reflection; using System.Text.Json.Serialization; using gaseous_server; +using gaseous_server.SignatureIngestors.XML; using gaseous_tools; using Microsoft.AspNetCore.Http.Features; using Microsoft.AspNetCore.Mvc; @@ -146,6 +147,10 @@ gaseous_server.Classes.Metadata.Platforms.GetPlatform(0); // organise library //gaseous_server.Classes.ImportGame.OrganiseLibrary(); +// Migrate signature data if needed +XMLIngestor ingestor = new XMLIngestor(); +ingestor.MigrateMetadatVersion(); + // add background tasks ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.SignatureIngestor, 60)); ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem( diff --git a/gaseous-server/Support/PlatformMap.json b/gaseous-server/Support/PlatformMap.json index 66574dd..abf1661 100644 --- a/gaseous-server/Support/PlatformMap.json +++ b/gaseous-server/Support/PlatformMap.json @@ -205,7 +205,9 @@ { "IGDBId": 52, "IGDBName": "Arcade", - "AlternateNames": [], + "AlternateNames": [ + "Arcade" + ], "KnownFileExtensions": [], "WebEmulator": { "Type": "EmulatorJS", diff --git a/gaseous-server/gaseous-server.csproj b/gaseous-server/gaseous-server.csproj index b252951..3c2ad6d 100644 --- a/gaseous-server/gaseous-server.csproj +++ b/gaseous-server/gaseous-server.csproj @@ -18,7 +18,7 @@ bin\Release\net7.0\gaseous-server.xml - + diff --git a/gaseous-server/wwwroot/pages/dialogs/rominfo.html b/gaseous-server/wwwroot/pages/dialogs/rominfo.html index 78fb8f7..3224387 100644 --- a/gaseous-server/wwwroot/pages/dialogs/rominfo.html +++ b/gaseous-server/wwwroot/pages/dialogs/rominfo.html @@ -1,5 +1,6 @@ 
General
+
Attributes
Title Match
@@ -39,8 +40,8 @@ - Signature Source Flags - + Signature Game Title + @@ -48,6 +49,10 @@ + +