fix: hopefully resolve the signature import performance issues
This commit is contained in:
Michael Green
@@ -5,21 +5,15 @@ using gaseous_romsignatureobject;
|
||||
using gaseous_signature_parser.parsers;
|
||||
using gaseous_tools;
|
||||
using MySqlX.XDevAPI;
|
||||
using static gaseous_tools.Database;
|
||||
using System.Data;
|
||||
|
||||
namespace gaseous_server.SignatureIngestors.TOSEC
|
||||
{
|
||||
public class TOSECIngestor
|
||||
{
|
||||
private Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
|
||||
|
||||
public void Import(string SearchPath)
|
||||
{
|
||||
List<Dictionary<string, object>> ImportedPlatforms = new List<Dictionary<string, object>>();
|
||||
List<Dictionary<string, object>> ImportedPublishers = new List<Dictionary<string, object>>();
|
||||
List<Dictionary<string, object>> ImportedGames = new List<Dictionary<string, object>>();
|
||||
List<Dictionary<string, object>> ImportedRoms = new List<Dictionary<string, object>>();
|
||||
// connect to database
|
||||
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
|
||||
|
||||
// process provided files
|
||||
Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing from " + SearchPath);
|
||||
@@ -36,14 +30,6 @@ namespace gaseous_server.SignatureIngestors.TOSEC
|
||||
{
|
||||
string tosecXMLFile = tosecPathContents[i];
|
||||
|
||||
string[] SkippableFiles = {
|
||||
".DS_STORE",
|
||||
"desktop.ini"
|
||||
};
|
||||
|
||||
if (!SkippableFiles.Contains<string>(Path.GetFileName(tosecXMLFile), StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
|
||||
// check tosec file md5
|
||||
Common.hashObject hashObject = new Common.hashObject(tosecXMLFile);
|
||||
sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
|
||||
@@ -94,7 +80,6 @@ namespace gaseous_server.SignatureIngestors.TOSEC
|
||||
|
||||
if (processGames == true)
|
||||
{
|
||||
Console.WriteLine("Games to process: " + tosecObject.Games.Count);
|
||||
for (int x = 0; x < tosecObject.Games.Count; ++x)
|
||||
{
|
||||
RomSignatureObject.Game gameObject = tosecObject.Games[x];
|
||||
@@ -117,21 +102,6 @@ namespace gaseous_server.SignatureIngestors.TOSEC
|
||||
// store platform
|
||||
int gameSystem = 0;
|
||||
if (gameObject.System != null)
|
||||
{
|
||||
bool foundPlatform = false;
|
||||
string platformName = (string)dbDict["platform"];
|
||||
foreach (Dictionary<string, object> ImportedPlatform in ImportedPlatforms)
|
||||
{
|
||||
string importedPlatformName = (string)ImportedPlatform["Name"];
|
||||
if (importedPlatformName == platformName)
|
||||
{
|
||||
foundPlatform = true;
|
||||
gameSystem = (int)ImportedPlatform["Id"];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (foundPlatform == false)
|
||||
{
|
||||
sql = "SELECT Id FROM Signatures_Platforms WHERE Platform=@platform";
|
||||
|
||||
@@ -148,33 +118,12 @@ namespace gaseous_server.SignatureIngestors.TOSEC
|
||||
{
|
||||
gameSystem = (int)sigDB.Rows[0][0];
|
||||
}
|
||||
|
||||
Dictionary<string, object> platformDetails = new Dictionary<string, object>();
|
||||
platformDetails.Add("Id", gameSystem);
|
||||
platformDetails.Add("Name", dbDict["platform"]);
|
||||
ImportedPlatforms.Add(platformDetails);
|
||||
}
|
||||
}
|
||||
dbDict.Add("systemid", gameSystem);
|
||||
|
||||
// store publisher
|
||||
int gamePublisher = 0;
|
||||
if (gameObject.Publisher != null)
|
||||
{
|
||||
bool foundPublisher = false;
|
||||
string publisherName = (string)dbDict["publisher"];
|
||||
foreach (Dictionary<string, object> ImportedPublisher in ImportedPublishers)
|
||||
{
|
||||
string importedPublisherName = (string)ImportedPublisher["Name"];
|
||||
if (importedPublisherName == publisherName)
|
||||
{
|
||||
foundPublisher = true;
|
||||
gamePublisher = (int)ImportedPublisher["Id"];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (foundPublisher == false)
|
||||
{
|
||||
sql = "SELECT * FROM Signatures_Publishers WHERE Publisher=@publisher";
|
||||
|
||||
@@ -190,97 +139,30 @@ namespace gaseous_server.SignatureIngestors.TOSEC
|
||||
{
|
||||
gamePublisher = (int)sigDB.Rows[0][0];
|
||||
}
|
||||
|
||||
Dictionary<string, object> publisherDetails = new Dictionary<string, object>();
|
||||
publisherDetails.Add("Id", gamePublisher);
|
||||
publisherDetails.Add("Name", dbDict["publisher"]);
|
||||
ImportedPublishers.Add(publisherDetails);
|
||||
}
|
||||
}
|
||||
dbDict.Add("publisherid", gamePublisher);
|
||||
|
||||
// store game
|
||||
ulong gameId = 0;
|
||||
bool foundGame = false;
|
||||
foreach (Dictionary<string, object> ImportedGame in ImportedGames)
|
||||
{
|
||||
if (((string)ImportedGame["Name"] == (string)dbDict["name"]) &&
|
||||
((string)ImportedGame["Year"] == (string)dbDict["year"]) &&
|
||||
((int)ImportedGame["PublisherId"] == (int)dbDict["publisherid"]) &&
|
||||
((int)ImportedGame["SystemId"] == (int)dbDict["systemid"]) &&
|
||||
((string)ImportedGame["Country"] == (string)dbDict["country"]) &&
|
||||
((string)ImportedGame["Language"] == (string)dbDict["language"]))
|
||||
{
|
||||
foundGame = true;
|
||||
gameId = (ulong)ImportedGame["Id"];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (foundGame == false)
|
||||
{
|
||||
int gameId = 0;
|
||||
sql = "SELECT * FROM Signatures_Games WHERE Name=@name AND Year=@year AND Publisherid=@publisher AND Systemid=@systemid AND Country=@country AND Language=@language";
|
||||
|
||||
sigDB = db.ExecuteCMD(sql, dbDict);
|
||||
if (sigDB.Rows.Count == 0)
|
||||
{
|
||||
// entry not present, insert it
|
||||
|
||||
sql = "SELECT Id FROM Signatures_Games ORDER BY Id DESC LIMIT 0,1;";
|
||||
DataTable gameQuery = db.ExecuteCMD(sql);
|
||||
if (gameQuery.Rows.Count > 0)
|
||||
{
|
||||
gameId = ((ulong)(int)gameQuery.Rows[0][0]) + 1;
|
||||
}
|
||||
else
|
||||
{
|
||||
gameId = 1;
|
||||
}
|
||||
|
||||
if (dbDict.ContainsKey("id"))
|
||||
{
|
||||
dbDict["id"] = gameId;
|
||||
}
|
||||
else
|
||||
{
|
||||
dbDict.Add("id", gameId);
|
||||
}
|
||||
|
||||
//sql = "INSERT INTO Signatures_Games " +
|
||||
// "(Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " +
|
||||
// "(@name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
|
||||
sql = "INSERT INTO Signatures_Games " +
|
||||
"(Id, Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " +
|
||||
"(@id, @name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright);";
|
||||
//sigDB = db.ExecuteCMD(sql, dbDict);
|
||||
"(Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " +
|
||||
"(@name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
|
||||
sigDB = db.ExecuteCMD(sql, dbDict);
|
||||
|
||||
//gameId = Convert.ToInt32(sigDB.Rows[0][0]);
|
||||
|
||||
SaveTransaction(new SQLTransactionItem
|
||||
{
|
||||
SQLCommand = sql,
|
||||
Parameters = dbDict
|
||||
});
|
||||
gameId = Convert.ToInt32(sigDB.Rows[0][0]);
|
||||
}
|
||||
else
|
||||
{
|
||||
gameId = (ulong)sigDB.Rows[0][0];
|
||||
}
|
||||
|
||||
Dictionary<string, object> gameDetails = new Dictionary<string, object>();
|
||||
gameDetails.Add("Id", gameId);
|
||||
gameDetails.Add("Name", dbDict["name"]);
|
||||
gameDetails.Add("Year", dbDict["year"]);
|
||||
gameDetails.Add("PublisherId", dbDict["publisherid"]);
|
||||
gameDetails.Add("SystemId", dbDict["systemid"]);
|
||||
gameDetails.Add("Country", dbDict["country"]);
|
||||
gameDetails.Add("Language", dbDict["language"]);
|
||||
ImportedGames.Add(gameDetails);
|
||||
gameId = (int)sigDB.Rows[0][0];
|
||||
}
|
||||
|
||||
// store rom
|
||||
//int transactionCounterMax = 1000;
|
||||
//List<SQLTransactionItem> sQLTransactionItems = new List<SQLTransactionItem>();
|
||||
foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms)
|
||||
{
|
||||
if (romObject.Md5 != null)
|
||||
@@ -316,68 +198,29 @@ namespace gaseous_server.SignatureIngestors.TOSEC
|
||||
dbDict.Add("medialabel", Common.ReturnValueIfNull(romObject.MediaLabel, ""));
|
||||
dbDict.Add("metadatasource", Classes.Roms.GameRomItem.SourceType.TOSEC);
|
||||
|
||||
bool ImportedRomFound = false;
|
||||
foreach (Dictionary<string, object> ImportedRom in ImportedRoms)
|
||||
{
|
||||
if (((ulong)ImportedRom["gameid"] == gameId) && ((string)ImportedRom["md5"] == romObject.Md5))
|
||||
{
|
||||
ImportedRomFound = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (ImportedRomFound == false)
|
||||
{
|
||||
sigDB = db.ExecuteCMD(sql, dbDict);
|
||||
if (sigDB.Rows.Count == 0)
|
||||
{
|
||||
//if (sQLTransactionItems.Count == transactionCounterMax)
|
||||
//{
|
||||
// db.ExecuteTransactionCMD(sQLTransactionItems);
|
||||
//}
|
||||
|
||||
// entry not present, insert it
|
||||
sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel, MetadataSource) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
|
||||
sigDB = db.ExecuteCMD(sql, dbDict);
|
||||
|
||||
//SQLTransactionItem sQLTransactionItem = new SQLTransactionItem(
|
||||
// SQLCommand: sql,
|
||||
// Parameters: dbDict
|
||||
// );
|
||||
|
||||
//sQLTransactionItems.Add(sQLTransactionItem);
|
||||
|
||||
SaveTransaction(new SQLTransactionItem
|
||||
{
|
||||
SQLCommand = sql,
|
||||
Parameters = dbDict
|
||||
});
|
||||
romId = Convert.ToInt32(sigDB.Rows[0][0]);
|
||||
}
|
||||
else
|
||||
{
|
||||
romId = (int)sigDB.Rows[0][0];
|
||||
}
|
||||
|
||||
Dictionary<string, object> romDetails = new Dictionary<string, object>();
|
||||
romDetails.Add("gameid", dbDict["gameid"]);
|
||||
romDetails.Add("md5", dbDict["md5"]);
|
||||
romDetails.Add("name", dbDict["name"]);
|
||||
ImportedRoms.Add(romDetails);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (sQLTransactionItems.Count > 0)
|
||||
{
|
||||
//db.ExecuteTransactionCMD(sQLTransactionItems);
|
||||
SaveTransaction();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Logging.Log(Logging.LogType.Warning, "Signature Ingestor - TOSEC", "Failed to import file " + tosecXMLFile, ex);
|
||||
Logging.Log(Logging.LogType.Warning, "Signature Ingestor - TOSEC", "Invalid import file: " + tosecXMLFile, ex);
|
||||
}
|
||||
}
|
||||
else
|
||||
@@ -388,26 +231,4 @@ namespace gaseous_server.SignatureIngestors.TOSEC
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<SQLTransactionItem> sQLTransactionItems = new List<SQLTransactionItem>();
|
||||
|
||||
private void SaveTransaction()
|
||||
{
|
||||
if (sQLTransactionItems.Count > 0)
|
||||
{
|
||||
db.ExecuteTransactionCMD(sQLTransactionItems);
|
||||
sQLTransactionItems.Clear();
|
||||
}
|
||||
}
|
||||
|
||||
private void SaveTransaction(SQLTransactionItem sQLTransactionItem)
|
||||
{
|
||||
sQLTransactionItems.Add(sQLTransactionItem);
|
||||
if (sQLTransactionItems.Count > 1000)
|
||||
{
|
||||
db.ExecuteTransactionCMD(sQLTransactionItems);
|
||||
sQLTransactionItems.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -577,9 +577,7 @@ CREATE TABLE `Signatures_Games` (
|
||||
UNIQUE KEY `Id_UNIQUE` (`Id`),
|
||||
KEY `publisher_Idx` (`PublisherId`),
|
||||
KEY `system_Idx` (`SystemId`),
|
||||
KEY `ingest_Idx` (`Name`,`Year`,`PublisherId`,`SystemId`,`Country`,`Language`) USING BTREE,
|
||||
CONSTRAINT `Publisher` FOREIGN KEY (`PublisherId`) REFERENCES `Signatures_Publishers` (`Id`) ON DELETE CASCADE ON UPDATE CASCADE,
|
||||
CONSTRAINT `System` FOREIGN KEY (`SystemId`) REFERENCES `Signatures_Platforms` (`Id`) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
KEY `ingest_Idx` (`Name`,`Year`,`PublisherId`,`SystemId`,`Country`,`Language`) USING BTREE
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
/*!40101 SET character_set_client = @saved_cs_client */;
|
||||
|
||||
@@ -641,8 +639,7 @@ CREATE TABLE `Signatures_Roms` (
|
||||
KEY `GameId_Idx` (`GameId`),
|
||||
KEY `md5_Idx` (`MD5`) USING BTREE,
|
||||
KEY `sha1_Idx` (`SHA1`) USING BTREE,
|
||||
KEY `flags_Idx` ((cast(`Flags` as char(255) array))),
|
||||
CONSTRAINT `GameId` FOREIGN KEY (`GameId`) REFERENCES `Signatures_Games` (`Id`) ON DELETE CASCADE ON UPDATE CASCADE
|
||||
KEY `flags_Idx` ((cast(`Flags` as char(255) array)))
|
||||
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||
/*!40101 SET character_set_client = @saved_cs_client */;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user