From 6f954f8d4abe7e3befac5dfcda868b75d505fe2d Mon Sep 17 00:00:00 2001 From: Michael Green <84688932+michael-j-green@users.noreply.github.com> Date: Wed, 5 Jul 2023 22:41:22 +1000 Subject: [PATCH] fix: skip dud files on signature import --- .../Classes/SignatureIngestors/TOSEC.cs | 500 +++++++++--------- 1 file changed, 258 insertions(+), 242 deletions(-) diff --git a/gaseous-server/Classes/SignatureIngestors/TOSEC.cs b/gaseous-server/Classes/SignatureIngestors/TOSEC.cs index d4db16c..95c0627 100644 --- a/gaseous-server/Classes/SignatureIngestors/TOSEC.cs +++ b/gaseous-server/Classes/SignatureIngestors/TOSEC.cs @@ -36,300 +36,316 @@ namespace gaseous_server.SignatureIngestors.TOSEC { string tosecXMLFile = tosecPathContents[i]; - // check tosec file md5 - Common.hashObject hashObject = new Common.hashObject(tosecXMLFile); - sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5"; - dbDict = new Dictionary(); - dbDict.Add("sourcemd5", hashObject.md5hash); - sigDB = db.ExecuteCMD(sql, dbDict); + string[] SkippableFiles = { + ".DS_STORE", + "desktop.ini" + }; - if (sigDB.Rows.Count == 0) + if (!SkippableFiles.Contains(Path.GetFileName(tosecXMLFile), StringComparer.OrdinalIgnoreCase)) { - Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing file: " + tosecXMLFile); - // start parsing file - TosecParser tosecParser = new TosecParser(); - RomSignatureObject tosecObject = tosecParser.Parse(tosecXMLFile); + // check tosec file md5 + Common.hashObject hashObject = new Common.hashObject(tosecXMLFile); + sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5"; + dbDict = new Dictionary(); + dbDict.Add("sourcemd5", hashObject.md5hash); + sigDB = db.ExecuteCMD(sql, dbDict); - // store in database - - // store source object - bool processGames = false; - if (tosecObject.SourceMd5 != null) + if (sigDB.Rows.Count == 0) { - sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5"; - dbDict = new Dictionary(); - dbDict.Add("name", Common.ReturnValueIfNull(tosecObject.Name, "")); - dbDict.Add("description", Common.ReturnValueIfNull(tosecObject.Description, "")); - dbDict.Add("category", Common.ReturnValueIfNull(tosecObject.Category, "")); - dbDict.Add("version", Common.ReturnValueIfNull(tosecObject.Version, "")); - dbDict.Add("author", Common.ReturnValueIfNull(tosecObject.Author, "")); - dbDict.Add("email", Common.ReturnValueIfNull(tosecObject.Email, "")); - dbDict.Add("homepage", Common.ReturnValueIfNull(tosecObject.Homepage, "")); - dbDict.Add("uri", Common.ReturnValueIfNull(tosecObject.Url, "")); - dbDict.Add("sourcetype", Common.ReturnValueIfNull(tosecObject.SourceType, "")); - dbDict.Add("sourcemd5", tosecObject.SourceMd5); - dbDict.Add("sourcesha1", tosecObject.SourceSHA1); - - sigDB = db.ExecuteCMD(sql, dbDict); - if (sigDB.Rows.Count == 0) + try { - // entry not present, insert it - sql = "INSERT INTO Signatures_Sources (Name, Description, Category, Version, Author, Email, Homepage, Url, SourceType, SourceMD5, SourceSHA1) VALUES (@name, @description, @category, @version, @author, @email, @homepage, @uri, @sourcetype, @sourcemd5, @sourcesha1)"; + Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing file: " + tosecXMLFile); - db.ExecuteCMD(sql, dbDict); + // start parsing file + TosecParser tosecParser = new TosecParser(); + RomSignatureObject tosecObject = tosecParser.Parse(tosecXMLFile); - processGames = true; - } + // store in database - if (processGames == true) - { - for (int x = 0; x < tosecObject.Games.Count; ++x) + // store source object + bool processGames = false; + if (tosecObject.SourceMd5 != null) { - RomSignatureObject.Game gameObject = tosecObject.Games[x]; - - // set up game dictionary + sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5"; dbDict = new Dictionary(); - dbDict.Add("name", Common.ReturnValueIfNull(gameObject.Name, "")); - dbDict.Add("description", Common.ReturnValueIfNull(gameObject.Description, "")); - dbDict.Add("year", Common.ReturnValueIfNull(gameObject.Year, "")); - dbDict.Add("publisher", Common.ReturnValueIfNull(gameObject.Publisher, "")); - dbDict.Add("demo", (int)gameObject.Demo); - dbDict.Add("system", Common.ReturnValueIfNull(gameObject.System, "")); - dbDict.Add("platform", Common.ReturnValueIfNull(gameObject.System, "")); - dbDict.Add("systemvariant", Common.ReturnValueIfNull(gameObject.SystemVariant, "")); - dbDict.Add("video", Common.ReturnValueIfNull(gameObject.Video, "")); - dbDict.Add("country", Common.ReturnValueIfNull(gameObject.Country, "")); - dbDict.Add("language", Common.ReturnValueIfNull(gameObject.Language, "")); - dbDict.Add("copyright", Common.ReturnValueIfNull(gameObject.Copyright, "")); + dbDict.Add("name", Common.ReturnValueIfNull(tosecObject.Name, "")); + dbDict.Add("description", Common.ReturnValueIfNull(tosecObject.Description, "")); + dbDict.Add("category", Common.ReturnValueIfNull(tosecObject.Category, "")); + dbDict.Add("version", Common.ReturnValueIfNull(tosecObject.Version, "")); + dbDict.Add("author", Common.ReturnValueIfNull(tosecObject.Author, "")); + dbDict.Add("email", Common.ReturnValueIfNull(tosecObject.Email, "")); + dbDict.Add("homepage", Common.ReturnValueIfNull(tosecObject.Homepage, "")); + dbDict.Add("uri", Common.ReturnValueIfNull(tosecObject.Url, "")); + dbDict.Add("sourcetype", Common.ReturnValueIfNull(tosecObject.SourceType, "")); + dbDict.Add("sourcemd5", tosecObject.SourceMd5); + dbDict.Add("sourcesha1", tosecObject.SourceSHA1); - // store platform - int gameSystem = 0; - if (gameObject.System != null) + sigDB = db.ExecuteCMD(sql, dbDict); + if (sigDB.Rows.Count == 0) { - bool foundPlatform = false; - string platformName = (string)dbDict["platform"]; - foreach (Dictionary ImportedPlatform in ImportedPlatforms) - { - string importedPlatformName = (string)ImportedPlatform["Name"]; - if (importedPlatformName == platformName) - { - foundPlatform = true; - gameSystem = (int)ImportedPlatform["Id"]; - break; - } - } + // entry not present, insert it + sql = "INSERT INTO Signatures_Sources (Name, Description, Category, Version, Author, Email, Homepage, Url, SourceType, SourceMD5, SourceSHA1) VALUES (@name, @description, @category, @version, @author, @email, @homepage, @uri, @sourcetype, @sourcemd5, @sourcesha1)"; - if (foundPlatform == false) - { - sql = "SELECT Id FROM Signatures_Platforms WHERE Platform=@platform"; + db.ExecuteCMD(sql, dbDict); - sigDB = db.ExecuteCMD(sql, dbDict); - if (sigDB.Rows.Count == 0) - { - // entry not present, insert it - sql = "INSERT INTO Signatures_Platforms (Platform) VALUES (@platform); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; - sigDB = db.ExecuteCMD(sql, dbDict); - - gameSystem = Convert.ToInt32(sigDB.Rows[0][0]); - } - else - { - gameSystem = (int)sigDB.Rows[0][0]; - } - - Dictionary platformDetails = new Dictionary(); - platformDetails.Add("Id", gameSystem); - platformDetails.Add("Name", dbDict["platform"]); - ImportedPlatforms.Add(platformDetails); - } - } - dbDict.Add("systemid", gameSystem); - - // store publisher - int gamePublisher = 0; - if (gameObject.Publisher != null) - { - bool foundPublisher = false; - string publisherName = (string)dbDict["publisher"]; - foreach (Dictionary ImportedPublisher in ImportedPublishers) - { - string importedPublisherName = (string)ImportedPublisher["Name"]; - if (importedPublisherName == publisherName) - { - foundPublisher = true; - gamePublisher = (int)ImportedPublisher["Id"]; - break; - } - } - - if (foundPublisher == false) - { - sql = "SELECT * FROM Signatures_Publishers WHERE Publisher=@publisher"; - - sigDB = db.ExecuteCMD(sql, dbDict); - if (sigDB.Rows.Count == 0) - { - // entry not present, insert it - sql = "INSERT INTO Signatures_Publishers (Publisher) VALUES (@publisher); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; - sigDB = db.ExecuteCMD(sql, dbDict); - gamePublisher = Convert.ToInt32(sigDB.Rows[0][0]); - } - else - { - gamePublisher = (int)sigDB.Rows[0][0]; - } - - Dictionary publisherDetails = new Dictionary(); - publisherDetails.Add("Id", gamePublisher); - publisherDetails.Add("Name", dbDict["publisher"]); - ImportedPublishers.Add(publisherDetails); - } - } - dbDict.Add("publisherid", gamePublisher); - - // store game - int gameId = 0; - bool foundGame = false; - foreach (Dictionary ImportedGame in ImportedGames) - { - if (((string)ImportedGame["Name"] == (string)dbDict["name"]) && - ((string)ImportedGame["Year"] == (string)dbDict["year"]) && - ((int)ImportedGame["PublisherId"] == (int)dbDict["publisherid"]) && - ((int)ImportedGame["SystemId"] == (int)dbDict["systemid"]) && - ((string)ImportedGame["Country"] == (string)dbDict["country"]) && - ((string)ImportedGame["Language"] == (string)dbDict["language"])) - { - foundGame = true; - gameId = (int)ImportedGame["Id"]; - break; - } + processGames = true; } - if (foundGame == false) + if (processGames == true) { - sql = "SELECT * FROM Signatures_Games WHERE Name=@name AND Year=@year AND Publisherid=@publisher AND Systemid=@systemid AND Country=@country AND Language=@language"; - - sigDB = db.ExecuteCMD(sql, dbDict); - if (sigDB.Rows.Count == 0) + for (int x = 0; x < tosecObject.Games.Count; ++x) { - // entry not present, insert it - sql = "INSERT INTO Signatures_Games " + - "(Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " + - "(@name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; - sigDB = db.ExecuteCMD(sql, dbDict); + RomSignatureObject.Game gameObject = tosecObject.Games[x]; - gameId = Convert.ToInt32(sigDB.Rows[0][0]); - } - else - { - gameId = (int)sigDB.Rows[0][0]; - } - - Dictionary gameDetails = new Dictionary(); - gameDetails.Add("Id", gameId); - gameDetails.Add("Name", dbDict["name"]); - gameDetails.Add("Year", dbDict["year"]); - gameDetails.Add("PublisherId", dbDict["publisherid"]); - gameDetails.Add("SystemId", dbDict["systemid"]); - gameDetails.Add("Country", dbDict["country"]); - gameDetails.Add("Language", dbDict["language"]); - ImportedGames.Add(gameDetails); - } - - // store rom - int transactionCounterMax = 1000; - List sQLTransactionItems = new List(); - foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms) - { - if (romObject.Md5 != null) - { - int romId = 0; - sql = "SELECT * FROM Signatures_Roms WHERE GameId=@gameid AND MD5=@md5"; + // set up game dictionary dbDict = new Dictionary(); - dbDict.Add("gameid", gameId); - dbDict.Add("name", Common.ReturnValueIfNull(romObject.Name, "")); - dbDict.Add("size", Common.ReturnValueIfNull(romObject.Size, "")); - dbDict.Add("crc", Common.ReturnValueIfNull(romObject.Crc, "")); - dbDict.Add("md5", romObject.Md5); - dbDict.Add("sha1", Common.ReturnValueIfNull(romObject.Sha1, "")); - dbDict.Add("developmentstatus", Common.ReturnValueIfNull(romObject.DevelopmentStatus, "")); + dbDict.Add("name", Common.ReturnValueIfNull(gameObject.Name, "")); + dbDict.Add("description", Common.ReturnValueIfNull(gameObject.Description, "")); + dbDict.Add("year", Common.ReturnValueIfNull(gameObject.Year, "")); + dbDict.Add("publisher", Common.ReturnValueIfNull(gameObject.Publisher, "")); + dbDict.Add("demo", (int)gameObject.Demo); + dbDict.Add("system", Common.ReturnValueIfNull(gameObject.System, "")); + dbDict.Add("platform", Common.ReturnValueIfNull(gameObject.System, "")); + dbDict.Add("systemvariant", Common.ReturnValueIfNull(gameObject.SystemVariant, "")); + dbDict.Add("video", Common.ReturnValueIfNull(gameObject.Video, "")); + dbDict.Add("country", Common.ReturnValueIfNull(gameObject.Country, "")); + dbDict.Add("language", Common.ReturnValueIfNull(gameObject.Language, "")); + dbDict.Add("copyright", Common.ReturnValueIfNull(gameObject.Copyright, "")); - if (romObject.flags != null) + // store platform + int gameSystem = 0; + if (gameObject.System != null) { - if (romObject.flags.Count > 0) + bool foundPlatform = false; + string platformName = (string)dbDict["platform"]; + foreach (Dictionary ImportedPlatform in ImportedPlatforms) { - dbDict.Add("flags", Newtonsoft.Json.JsonConvert.SerializeObject(romObject.flags)); + string importedPlatformName = (string)ImportedPlatform["Name"]; + if (importedPlatformName == platformName) + { + foundPlatform = true; + gameSystem = (int)ImportedPlatform["Id"]; + break; + } } - else + + if (foundPlatform == false) { - dbDict.Add("flags", "[ ]"); + sql = "SELECT Id FROM Signatures_Platforms WHERE Platform=@platform"; + + sigDB = db.ExecuteCMD(sql, dbDict); + if (sigDB.Rows.Count == 0) + { + // entry not present, insert it + sql = "INSERT INTO Signatures_Platforms (Platform) VALUES (@platform); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; + sigDB = db.ExecuteCMD(sql, dbDict); + + gameSystem = Convert.ToInt32(sigDB.Rows[0][0]); + } + else + { + gameSystem = (int)sigDB.Rows[0][0]; + } + + Dictionary platformDetails = new Dictionary(); + platformDetails.Add("Id", gameSystem); + platformDetails.Add("Name", dbDict["platform"]); + ImportedPlatforms.Add(platformDetails); } } - else - { - dbDict.Add("flags", "[ ]"); - } - dbDict.Add("romtype", (int)romObject.RomType); - dbDict.Add("romtypemedia", Common.ReturnValueIfNull(romObject.RomTypeMedia, "")); - dbDict.Add("medialabel", Common.ReturnValueIfNull(romObject.MediaLabel, "")); - dbDict.Add("metadatasource", Classes.Roms.GameRomItem.SourceType.TOSEC); + dbDict.Add("systemid", gameSystem); - bool ImportedRomFound = false; - foreach (Dictionary ImportedRom in ImportedRoms) + // store publisher + int gamePublisher = 0; + if (gameObject.Publisher != null) { - if (((int)ImportedRom["gameid"] == gameId) && ((string)ImportedRom["md5"] == romObject.Md5)) + bool foundPublisher = false; + string publisherName = (string)dbDict["publisher"]; + foreach (Dictionary ImportedPublisher in ImportedPublishers) { - ImportedRomFound = true; + string importedPublisherName = (string)ImportedPublisher["Name"]; + if (importedPublisherName == publisherName) + { + foundPublisher = true; + gamePublisher = (int)ImportedPublisher["Id"]; + break; + } + } + + if (foundPublisher == false) + { + sql = "SELECT * FROM Signatures_Publishers WHERE Publisher=@publisher"; + + sigDB = db.ExecuteCMD(sql, dbDict); + if (sigDB.Rows.Count == 0) + { + // entry not present, insert it + sql = "INSERT INTO Signatures_Publishers (Publisher) VALUES (@publisher); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; + sigDB = db.ExecuteCMD(sql, dbDict); + gamePublisher = Convert.ToInt32(sigDB.Rows[0][0]); + } + else + { + gamePublisher = (int)sigDB.Rows[0][0]; + } + + Dictionary publisherDetails = new Dictionary(); + publisherDetails.Add("Id", gamePublisher); + publisherDetails.Add("Name", dbDict["publisher"]); + ImportedPublishers.Add(publisherDetails); + } + } + dbDict.Add("publisherid", gamePublisher); + + // store game + int gameId = 0; + bool foundGame = false; + foreach (Dictionary ImportedGame in ImportedGames) + { + if (((string)ImportedGame["Name"] == (string)dbDict["name"]) && + ((string)ImportedGame["Year"] == (string)dbDict["year"]) && + ((int)ImportedGame["PublisherId"] == (int)dbDict["publisherid"]) && + ((int)ImportedGame["SystemId"] == (int)dbDict["systemid"]) && + ((string)ImportedGame["Country"] == (string)dbDict["country"]) && + ((string)ImportedGame["Language"] == (string)dbDict["language"])) + { + foundGame = true; + gameId = (int)ImportedGame["Id"]; break; } } - if (ImportedRomFound == false) + if (foundGame == false) { + sql = "SELECT * FROM Signatures_Games WHERE Name=@name AND Year=@year AND Publisherid=@publisher AND Systemid=@systemid AND Country=@country AND Language=@language"; + sigDB = db.ExecuteCMD(sql, dbDict); if (sigDB.Rows.Count == 0) { - if (sQLTransactionItems.Count == transactionCounterMax) - { - db.ExecuteTransactionCMD(sQLTransactionItems); - } - // entry not present, insert it - sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel, MetadataSource) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; + sql = "INSERT INTO Signatures_Games " + + "(Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " + + "(@name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; + sigDB = db.ExecuteCMD(sql, dbDict); - SQLTransactionItem sQLTransactionItem = new SQLTransactionItem( - SQLCommand: sql, - Parameters: dbDict - ); - - sQLTransactionItems.Add(sQLTransactionItem); + gameId = Convert.ToInt32(sigDB.Rows[0][0]); } else { - romId = (int)sigDB.Rows[0][0]; + gameId = (int)sigDB.Rows[0][0]; } - Dictionary romDetails = new Dictionary(); - romDetails.Add("gameid", dbDict["gameid"]); - romDetails.Add("md5", dbDict["md5"]); - romDetails.Add("name", dbDict["name"]); - ImportedRoms.Add(romDetails); + Dictionary gameDetails = new Dictionary(); + gameDetails.Add("Id", gameId); + gameDetails.Add("Name", dbDict["name"]); + gameDetails.Add("Year", dbDict["year"]); + gameDetails.Add("PublisherId", dbDict["publisherid"]); + gameDetails.Add("SystemId", dbDict["systemid"]); + gameDetails.Add("Country", dbDict["country"]); + gameDetails.Add("Language", dbDict["language"]); + ImportedGames.Add(gameDetails); + } + + // store rom + int transactionCounterMax = 1000; + List sQLTransactionItems = new List(); + foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms) + { + if (romObject.Md5 != null) + { + int romId = 0; + sql = "SELECT * FROM Signatures_Roms WHERE GameId=@gameid AND MD5=@md5"; + dbDict = new Dictionary(); + dbDict.Add("gameid", gameId); + dbDict.Add("name", Common.ReturnValueIfNull(romObject.Name, "")); + dbDict.Add("size", Common.ReturnValueIfNull(romObject.Size, "")); + dbDict.Add("crc", Common.ReturnValueIfNull(romObject.Crc, "")); + dbDict.Add("md5", romObject.Md5); + dbDict.Add("sha1", Common.ReturnValueIfNull(romObject.Sha1, "")); + dbDict.Add("developmentstatus", Common.ReturnValueIfNull(romObject.DevelopmentStatus, "")); + + if (romObject.flags != null) + { + if (romObject.flags.Count > 0) + { + dbDict.Add("flags", Newtonsoft.Json.JsonConvert.SerializeObject(romObject.flags)); + } + else + { + dbDict.Add("flags", "[ ]"); + } + } + else + { + dbDict.Add("flags", "[ ]"); + } + dbDict.Add("romtype", (int)romObject.RomType); + dbDict.Add("romtypemedia", Common.ReturnValueIfNull(romObject.RomTypeMedia, "")); + dbDict.Add("medialabel", Common.ReturnValueIfNull(romObject.MediaLabel, "")); + dbDict.Add("metadatasource", Classes.Roms.GameRomItem.SourceType.TOSEC); + + bool ImportedRomFound = false; + foreach (Dictionary ImportedRom in ImportedRoms) + { + if (((int)ImportedRom["gameid"] == gameId) && ((string)ImportedRom["md5"] == romObject.Md5)) + { + ImportedRomFound = true; + break; + } + } + + if (ImportedRomFound == false) + { + sigDB = db.ExecuteCMD(sql, dbDict); + if (sigDB.Rows.Count == 0) + { + if (sQLTransactionItems.Count == transactionCounterMax) + { + db.ExecuteTransactionCMD(sQLTransactionItems); + } + + // entry not present, insert it + sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel, MetadataSource) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);"; + + SQLTransactionItem sQLTransactionItem = new SQLTransactionItem( + SQLCommand: sql, + Parameters: dbDict + ); + + sQLTransactionItems.Add(sQLTransactionItem); + } + else + { + romId = (int)sigDB.Rows[0][0]; + } + + Dictionary romDetails = new Dictionary(); + romDetails.Add("gameid", dbDict["gameid"]); + romDetails.Add("md5", dbDict["md5"]); + romDetails.Add("name", dbDict["name"]); + ImportedRoms.Add(romDetails); + } + } + } + + if (sQLTransactionItems.Count > 0) + { + db.ExecuteTransactionCMD(sQLTransactionItems); } } } - - if (sQLTransactionItems.Count > 0) - { - db.ExecuteTransactionCMD(sQLTransactionItems); - } } } + catch (Exception ex) + { + Logging.Log(Logging.LogType.Warning, "Signature Ingestor - TOSEC", "Failed to import file " + tosecXMLFile, ex); + } + } + else + { + Logging.Log(Logging.LogType.Debug, "Signature Ingestor - TOSEC", "Rejecting already imported file: " + tosecXMLFile); } - } - else - { - Logging.Log(Logging.LogType.Debug, "Signature Ingestor - TOSEC", "Rejecting already imported file: " + tosecXMLFile); } } }