fix: hopefully resolve the signature import performance issues
This commit is contained in:
Michael Green
@@ -5,21 +5,15 @@ using gaseous_romsignatureobject;
|
|||||||
using gaseous_signature_parser.parsers;
|
using gaseous_signature_parser.parsers;
|
||||||
using gaseous_tools;
|
using gaseous_tools;
|
||||||
using MySqlX.XDevAPI;
|
using MySqlX.XDevAPI;
|
||||||
using static gaseous_tools.Database;
|
|
||||||
using System.Data;
|
|
||||||
|
|
||||||
namespace gaseous_server.SignatureIngestors.TOSEC
|
namespace gaseous_server.SignatureIngestors.TOSEC
|
||||||
{
|
{
|
||||||
public class TOSECIngestor
|
public class TOSECIngestor
|
||||||
{
|
{
|
||||||
private Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
|
|
||||||
|
|
||||||
public void Import(string SearchPath)
|
public void Import(string SearchPath)
|
||||||
{
|
{
|
||||||
List<Dictionary<string, object>> ImportedPlatforms = new List<Dictionary<string, object>>();
|
// connect to database
|
||||||
List<Dictionary<string, object>> ImportedPublishers = new List<Dictionary<string, object>>();
|
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
|
||||||
List<Dictionary<string, object>> ImportedGames = new List<Dictionary<string, object>>();
|
|
||||||
List<Dictionary<string, object>> ImportedRoms = new List<Dictionary<string, object>>();
|
|
||||||
|
|
||||||
// process provided files
|
// process provided files
|
||||||
Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing from " + SearchPath);
|
Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing from " + SearchPath);
|
||||||
@@ -27,7 +21,7 @@ namespace gaseous_server.SignatureIngestors.TOSEC
|
|||||||
{
|
{
|
||||||
string[] tosecPathContents = Directory.GetFiles(SearchPath);
|
string[] tosecPathContents = Directory.GetFiles(SearchPath);
|
||||||
Array.Sort(tosecPathContents);
|
Array.Sort(tosecPathContents);
|
||||||
|
|
||||||
string sql = "";
|
string sql = "";
|
||||||
Dictionary<string, object> dbDict = new Dictionary<string, object>();
|
Dictionary<string, object> dbDict = new Dictionary<string, object>();
|
||||||
System.Data.DataTable sigDB;
|
System.Data.DataTable sigDB;
|
||||||
@@ -36,378 +30,205 @@ namespace gaseous_server.SignatureIngestors.TOSEC
|
|||||||
{
|
{
|
||||||
string tosecXMLFile = tosecPathContents[i];
|
string tosecXMLFile = tosecPathContents[i];
|
||||||
|
|
||||||
string[] SkippableFiles = {
|
// check tosec file md5
|
||||||
".DS_STORE",
|
Common.hashObject hashObject = new Common.hashObject(tosecXMLFile);
|
||||||
"desktop.ini"
|
sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
|
||||||
};
|
dbDict = new Dictionary<string, object>();
|
||||||
|
dbDict.Add("sourcemd5", hashObject.md5hash);
|
||||||
|
sigDB = db.ExecuteCMD(sql, dbDict);
|
||||||
|
|
||||||
if (!SkippableFiles.Contains<string>(Path.GetFileName(tosecXMLFile), StringComparer.OrdinalIgnoreCase))
|
if (sigDB.Rows.Count == 0)
|
||||||
{
|
{
|
||||||
|
try
|
||||||
// check tosec file md5
|
|
||||||
Common.hashObject hashObject = new Common.hashObject(tosecXMLFile);
|
|
||||||
sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
|
|
||||||
dbDict = new Dictionary<string, object>();
|
|
||||||
dbDict.Add("sourcemd5", hashObject.md5hash);
|
|
||||||
sigDB = db.ExecuteCMD(sql, dbDict);
|
|
||||||
|
|
||||||
if (sigDB.Rows.Count == 0)
|
|
||||||
{
|
{
|
||||||
try
|
Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing file: " + tosecXMLFile);
|
||||||
|
|
||||||
|
// start parsing file
|
||||||
|
TosecParser tosecParser = new TosecParser();
|
||||||
|
RomSignatureObject tosecObject = tosecParser.Parse(tosecXMLFile);
|
||||||
|
|
||||||
|
// store in database
|
||||||
|
|
||||||
|
// store source object
|
||||||
|
bool processGames = false;
|
||||||
|
if (tosecObject.SourceMd5 != null)
|
||||||
{
|
{
|
||||||
Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing file: " + tosecXMLFile);
|
sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
|
||||||
|
dbDict = new Dictionary<string, object>();
|
||||||
|
dbDict.Add("name", Common.ReturnValueIfNull(tosecObject.Name, ""));
|
||||||
|
dbDict.Add("description", Common.ReturnValueIfNull(tosecObject.Description, ""));
|
||||||
|
dbDict.Add("category", Common.ReturnValueIfNull(tosecObject.Category, ""));
|
||||||
|
dbDict.Add("version", Common.ReturnValueIfNull(tosecObject.Version, ""));
|
||||||
|
dbDict.Add("author", Common.ReturnValueIfNull(tosecObject.Author, ""));
|
||||||
|
dbDict.Add("email", Common.ReturnValueIfNull(tosecObject.Email, ""));
|
||||||
|
dbDict.Add("homepage", Common.ReturnValueIfNull(tosecObject.Homepage, ""));
|
||||||
|
dbDict.Add("uri", Common.ReturnValueIfNull(tosecObject.Url, ""));
|
||||||
|
dbDict.Add("sourcetype", Common.ReturnValueIfNull(tosecObject.SourceType, ""));
|
||||||
|
dbDict.Add("sourcemd5", tosecObject.SourceMd5);
|
||||||
|
dbDict.Add("sourcesha1", tosecObject.SourceSHA1);
|
||||||
|
|
||||||
// start parsing file
|
sigDB = db.ExecuteCMD(sql, dbDict);
|
||||||
TosecParser tosecParser = new TosecParser();
|
if (sigDB.Rows.Count == 0)
|
||||||
RomSignatureObject tosecObject = tosecParser.Parse(tosecXMLFile);
|
|
||||||
|
|
||||||
// store in database
|
|
||||||
|
|
||||||
// store source object
|
|
||||||
bool processGames = false;
|
|
||||||
if (tosecObject.SourceMd5 != null)
|
|
||||||
{
|
{
|
||||||
sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
|
// entry not present, insert it
|
||||||
dbDict = new Dictionary<string, object>();
|
sql = "INSERT INTO Signatures_Sources (Name, Description, Category, Version, Author, Email, Homepage, Url, SourceType, SourceMD5, SourceSHA1) VALUES (@name, @description, @category, @version, @author, @email, @homepage, @uri, @sourcetype, @sourcemd5, @sourcesha1)";
|
||||||
dbDict.Add("name", Common.ReturnValueIfNull(tosecObject.Name, ""));
|
|
||||||
dbDict.Add("description", Common.ReturnValueIfNull(tosecObject.Description, ""));
|
|
||||||
dbDict.Add("category", Common.ReturnValueIfNull(tosecObject.Category, ""));
|
|
||||||
dbDict.Add("version", Common.ReturnValueIfNull(tosecObject.Version, ""));
|
|
||||||
dbDict.Add("author", Common.ReturnValueIfNull(tosecObject.Author, ""));
|
|
||||||
dbDict.Add("email", Common.ReturnValueIfNull(tosecObject.Email, ""));
|
|
||||||
dbDict.Add("homepage", Common.ReturnValueIfNull(tosecObject.Homepage, ""));
|
|
||||||
dbDict.Add("uri", Common.ReturnValueIfNull(tosecObject.Url, ""));
|
|
||||||
dbDict.Add("sourcetype", Common.ReturnValueIfNull(tosecObject.SourceType, ""));
|
|
||||||
dbDict.Add("sourcemd5", tosecObject.SourceMd5);
|
|
||||||
dbDict.Add("sourcesha1", tosecObject.SourceSHA1);
|
|
||||||
|
|
||||||
sigDB = db.ExecuteCMD(sql, dbDict);
|
db.ExecuteCMD(sql, dbDict);
|
||||||
if (sigDB.Rows.Count == 0)
|
|
||||||
|
processGames = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (processGames == true)
|
||||||
|
{
|
||||||
|
for (int x = 0; x < tosecObject.Games.Count; ++x)
|
||||||
{
|
{
|
||||||
// entry not present, insert it
|
RomSignatureObject.Game gameObject = tosecObject.Games[x];
|
||||||
sql = "INSERT INTO Signatures_Sources (Name, Description, Category, Version, Author, Email, Homepage, Url, SourceType, SourceMD5, SourceSHA1) VALUES (@name, @description, @category, @version, @author, @email, @homepage, @uri, @sourcetype, @sourcemd5, @sourcesha1)";
|
|
||||||
|
|
||||||
db.ExecuteCMD(sql, dbDict);
|
// set up game dictionary
|
||||||
|
dbDict = new Dictionary<string, object>();
|
||||||
|
dbDict.Add("name", Common.ReturnValueIfNull(gameObject.Name, ""));
|
||||||
|
dbDict.Add("description", Common.ReturnValueIfNull(gameObject.Description, ""));
|
||||||
|
dbDict.Add("year", Common.ReturnValueIfNull(gameObject.Year, ""));
|
||||||
|
dbDict.Add("publisher", Common.ReturnValueIfNull(gameObject.Publisher, ""));
|
||||||
|
dbDict.Add("demo", (int)gameObject.Demo);
|
||||||
|
dbDict.Add("system", Common.ReturnValueIfNull(gameObject.System, ""));
|
||||||
|
dbDict.Add("platform", Common.ReturnValueIfNull(gameObject.System, ""));
|
||||||
|
dbDict.Add("systemvariant", Common.ReturnValueIfNull(gameObject.SystemVariant, ""));
|
||||||
|
dbDict.Add("video", Common.ReturnValueIfNull(gameObject.Video, ""));
|
||||||
|
dbDict.Add("country", Common.ReturnValueIfNull(gameObject.Country, ""));
|
||||||
|
dbDict.Add("language", Common.ReturnValueIfNull(gameObject.Language, ""));
|
||||||
|
dbDict.Add("copyright", Common.ReturnValueIfNull(gameObject.Copyright, ""));
|
||||||
|
|
||||||
processGames = true;
|
// store platform
|
||||||
}
|
int gameSystem = 0;
|
||||||
|
if (gameObject.System != null)
|
||||||
if (processGames == true)
|
|
||||||
{
|
|
||||||
Console.WriteLine("Games to process: " + tosecObject.Games.Count);
|
|
||||||
for (int x = 0; x < tosecObject.Games.Count; ++x)
|
|
||||||
{
|
{
|
||||||
RomSignatureObject.Game gameObject = tosecObject.Games[x];
|
sql = "SELECT Id FROM Signatures_Platforms WHERE Platform=@platform";
|
||||||
|
|
||||||
// set up game dictionary
|
sigDB = db.ExecuteCMD(sql, dbDict);
|
||||||
dbDict = new Dictionary<string, object>();
|
if (sigDB.Rows.Count == 0)
|
||||||
dbDict.Add("name", Common.ReturnValueIfNull(gameObject.Name, ""));
|
|
||||||
dbDict.Add("description", Common.ReturnValueIfNull(gameObject.Description, ""));
|
|
||||||
dbDict.Add("year", Common.ReturnValueIfNull(gameObject.Year, ""));
|
|
||||||
dbDict.Add("publisher", Common.ReturnValueIfNull(gameObject.Publisher, ""));
|
|
||||||
dbDict.Add("demo", (int)gameObject.Demo);
|
|
||||||
dbDict.Add("system", Common.ReturnValueIfNull(gameObject.System, ""));
|
|
||||||
dbDict.Add("platform", Common.ReturnValueIfNull(gameObject.System, ""));
|
|
||||||
dbDict.Add("systemvariant", Common.ReturnValueIfNull(gameObject.SystemVariant, ""));
|
|
||||||
dbDict.Add("video", Common.ReturnValueIfNull(gameObject.Video, ""));
|
|
||||||
dbDict.Add("country", Common.ReturnValueIfNull(gameObject.Country, ""));
|
|
||||||
dbDict.Add("language", Common.ReturnValueIfNull(gameObject.Language, ""));
|
|
||||||
dbDict.Add("copyright", Common.ReturnValueIfNull(gameObject.Copyright, ""));
|
|
||||||
|
|
||||||
// store platform
|
|
||||||
int gameSystem = 0;
|
|
||||||
if (gameObject.System != null)
|
|
||||||
{
|
{
|
||||||
bool foundPlatform = false;
|
// entry not present, insert it
|
||||||
string platformName = (string)dbDict["platform"];
|
sql = "INSERT INTO Signatures_Platforms (Platform) VALUES (@platform); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
|
||||||
foreach (Dictionary<string, object> ImportedPlatform in ImportedPlatforms)
|
|
||||||
{
|
|
||||||
string importedPlatformName = (string)ImportedPlatform["Name"];
|
|
||||||
if (importedPlatformName == platformName)
|
|
||||||
{
|
|
||||||
foundPlatform = true;
|
|
||||||
gameSystem = (int)ImportedPlatform["Id"];
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (foundPlatform == false)
|
|
||||||
{
|
|
||||||
sql = "SELECT Id FROM Signatures_Platforms WHERE Platform=@platform";
|
|
||||||
|
|
||||||
sigDB = db.ExecuteCMD(sql, dbDict);
|
|
||||||
if (sigDB.Rows.Count == 0)
|
|
||||||
{
|
|
||||||
// entry not present, insert it
|
|
||||||
sql = "INSERT INTO Signatures_Platforms (Platform) VALUES (@platform); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
|
|
||||||
sigDB = db.ExecuteCMD(sql, dbDict);
|
|
||||||
|
|
||||||
gameSystem = Convert.ToInt32(sigDB.Rows[0][0]);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
gameSystem = (int)sigDB.Rows[0][0];
|
|
||||||
}
|
|
||||||
|
|
||||||
Dictionary<string, object> platformDetails = new Dictionary<string, object>();
|
|
||||||
platformDetails.Add("Id", gameSystem);
|
|
||||||
platformDetails.Add("Name", dbDict["platform"]);
|
|
||||||
ImportedPlatforms.Add(platformDetails);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
dbDict.Add("systemid", gameSystem);
|
|
||||||
|
|
||||||
// store publisher
|
|
||||||
int gamePublisher = 0;
|
|
||||||
if (gameObject.Publisher != null)
|
|
||||||
{
|
|
||||||
bool foundPublisher = false;
|
|
||||||
string publisherName = (string)dbDict["publisher"];
|
|
||||||
foreach (Dictionary<string, object> ImportedPublisher in ImportedPublishers)
|
|
||||||
{
|
|
||||||
string importedPublisherName = (string)ImportedPublisher["Name"];
|
|
||||||
if (importedPublisherName == publisherName)
|
|
||||||
{
|
|
||||||
foundPublisher = true;
|
|
||||||
gamePublisher = (int)ImportedPublisher["Id"];
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (foundPublisher == false)
|
|
||||||
{
|
|
||||||
sql = "SELECT * FROM Signatures_Publishers WHERE Publisher=@publisher";
|
|
||||||
|
|
||||||
sigDB = db.ExecuteCMD(sql, dbDict);
|
|
||||||
if (sigDB.Rows.Count == 0)
|
|
||||||
{
|
|
||||||
// entry not present, insert it
|
|
||||||
sql = "INSERT INTO Signatures_Publishers (Publisher) VALUES (@publisher); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
|
|
||||||
sigDB = db.ExecuteCMD(sql, dbDict);
|
|
||||||
gamePublisher = Convert.ToInt32(sigDB.Rows[0][0]);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
gamePublisher = (int)sigDB.Rows[0][0];
|
|
||||||
}
|
|
||||||
|
|
||||||
Dictionary<string, object> publisherDetails = new Dictionary<string, object>();
|
|
||||||
publisherDetails.Add("Id", gamePublisher);
|
|
||||||
publisherDetails.Add("Name", dbDict["publisher"]);
|
|
||||||
ImportedPublishers.Add(publisherDetails);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
dbDict.Add("publisherid", gamePublisher);
|
|
||||||
|
|
||||||
// store game
|
|
||||||
ulong gameId = 0;
|
|
||||||
bool foundGame = false;
|
|
||||||
foreach (Dictionary<string, object> ImportedGame in ImportedGames)
|
|
||||||
{
|
|
||||||
if (((string)ImportedGame["Name"] == (string)dbDict["name"]) &&
|
|
||||||
((string)ImportedGame["Year"] == (string)dbDict["year"]) &&
|
|
||||||
((int)ImportedGame["PublisherId"] == (int)dbDict["publisherid"]) &&
|
|
||||||
((int)ImportedGame["SystemId"] == (int)dbDict["systemid"]) &&
|
|
||||||
((string)ImportedGame["Country"] == (string)dbDict["country"]) &&
|
|
||||||
((string)ImportedGame["Language"] == (string)dbDict["language"]))
|
|
||||||
{
|
|
||||||
foundGame = true;
|
|
||||||
gameId = (ulong)ImportedGame["Id"];
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (foundGame == false)
|
|
||||||
{
|
|
||||||
sql = "SELECT * FROM Signatures_Games WHERE Name=@name AND Year=@year AND Publisherid=@publisher AND Systemid=@systemid AND Country=@country AND Language=@language";
|
|
||||||
|
|
||||||
sigDB = db.ExecuteCMD(sql, dbDict);
|
sigDB = db.ExecuteCMD(sql, dbDict);
|
||||||
if (sigDB.Rows.Count == 0)
|
|
||||||
{
|
|
||||||
// entry not present, insert it
|
|
||||||
|
|
||||||
sql = "SELECT Id FROM Signatures_Games ORDER BY Id DESC LIMIT 0,1;";
|
gameSystem = Convert.ToInt32(sigDB.Rows[0][0]);
|
||||||
DataTable gameQuery = db.ExecuteCMD(sql);
|
|
||||||
if (gameQuery.Rows.Count > 0)
|
|
||||||
{
|
|
||||||
gameId = ((ulong)(int)gameQuery.Rows[0][0]) + 1;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
gameId = 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (dbDict.ContainsKey("id"))
|
|
||||||
{
|
|
||||||
dbDict["id"] = gameId;
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
dbDict.Add("id", gameId);
|
|
||||||
}
|
|
||||||
|
|
||||||
//sql = "INSERT INTO Signatures_Games " +
|
|
||||||
// "(Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " +
|
|
||||||
// "(@name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
|
|
||||||
sql = "INSERT INTO Signatures_Games " +
|
|
||||||
"(Id, Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " +
|
|
||||||
"(@id, @name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright);";
|
|
||||||
//sigDB = db.ExecuteCMD(sql, dbDict);
|
|
||||||
|
|
||||||
//gameId = Convert.ToInt32(sigDB.Rows[0][0]);
|
|
||||||
|
|
||||||
SaveTransaction(new SQLTransactionItem
|
|
||||||
{
|
|
||||||
SQLCommand = sql,
|
|
||||||
Parameters = dbDict
|
|
||||||
});
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
gameId = (ulong)sigDB.Rows[0][0];
|
|
||||||
}
|
|
||||||
|
|
||||||
Dictionary<string, object> gameDetails = new Dictionary<string, object>();
|
|
||||||
gameDetails.Add("Id", gameId);
|
|
||||||
gameDetails.Add("Name", dbDict["name"]);
|
|
||||||
gameDetails.Add("Year", dbDict["year"]);
|
|
||||||
gameDetails.Add("PublisherId", dbDict["publisherid"]);
|
|
||||||
gameDetails.Add("SystemId", dbDict["systemid"]);
|
|
||||||
gameDetails.Add("Country", dbDict["country"]);
|
|
||||||
gameDetails.Add("Language", dbDict["language"]);
|
|
||||||
ImportedGames.Add(gameDetails);
|
|
||||||
}
|
}
|
||||||
|
else
|
||||||
// store rom
|
|
||||||
//int transactionCounterMax = 1000;
|
|
||||||
//List<SQLTransactionItem> sQLTransactionItems = new List<SQLTransactionItem>();
|
|
||||||
foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms)
|
|
||||||
{
|
{
|
||||||
if (romObject.Md5 != null)
|
gameSystem = (int)sigDB.Rows[0][0];
|
||||||
{
|
}
|
||||||
int romId = 0;
|
}
|
||||||
sql = "SELECT * FROM Signatures_Roms WHERE GameId=@gameid AND MD5=@md5";
|
dbDict.Add("systemid", gameSystem);
|
||||||
dbDict = new Dictionary<string, object>();
|
|
||||||
dbDict.Add("gameid", gameId);
|
|
||||||
dbDict.Add("name", Common.ReturnValueIfNull(romObject.Name, ""));
|
|
||||||
dbDict.Add("size", Common.ReturnValueIfNull(romObject.Size, ""));
|
|
||||||
dbDict.Add("crc", Common.ReturnValueIfNull(romObject.Crc, ""));
|
|
||||||
dbDict.Add("md5", romObject.Md5);
|
|
||||||
dbDict.Add("sha1", Common.ReturnValueIfNull(romObject.Sha1, ""));
|
|
||||||
dbDict.Add("developmentstatus", Common.ReturnValueIfNull(romObject.DevelopmentStatus, ""));
|
|
||||||
|
|
||||||
if (romObject.flags != null)
|
// store publisher
|
||||||
|
int gamePublisher = 0;
|
||||||
|
if (gameObject.Publisher != null)
|
||||||
|
{
|
||||||
|
sql = "SELECT * FROM Signatures_Publishers WHERE Publisher=@publisher";
|
||||||
|
|
||||||
|
sigDB = db.ExecuteCMD(sql, dbDict);
|
||||||
|
if (sigDB.Rows.Count == 0)
|
||||||
|
{
|
||||||
|
// entry not present, insert it
|
||||||
|
sql = "INSERT INTO Signatures_Publishers (Publisher) VALUES (@publisher); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
|
||||||
|
sigDB = db.ExecuteCMD(sql, dbDict);
|
||||||
|
gamePublisher = Convert.ToInt32(sigDB.Rows[0][0]);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
gamePublisher = (int)sigDB.Rows[0][0];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
dbDict.Add("publisherid", gamePublisher);
|
||||||
|
|
||||||
|
// store game
|
||||||
|
int gameId = 0;
|
||||||
|
sql = "SELECT * FROM Signatures_Games WHERE Name=@name AND Year=@year AND Publisherid=@publisher AND Systemid=@systemid AND Country=@country AND Language=@language";
|
||||||
|
|
||||||
|
sigDB = db.ExecuteCMD(sql, dbDict);
|
||||||
|
if (sigDB.Rows.Count == 0)
|
||||||
|
{
|
||||||
|
// entry not present, insert it
|
||||||
|
sql = "INSERT INTO Signatures_Games " +
|
||||||
|
"(Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " +
|
||||||
|
"(@name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
|
||||||
|
sigDB = db.ExecuteCMD(sql, dbDict);
|
||||||
|
|
||||||
|
gameId = Convert.ToInt32(sigDB.Rows[0][0]);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
gameId = (int)sigDB.Rows[0][0];
|
||||||
|
}
|
||||||
|
|
||||||
|
// store rom
|
||||||
|
foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms)
|
||||||
|
{
|
||||||
|
if (romObject.Md5 != null)
|
||||||
|
{
|
||||||
|
int romId = 0;
|
||||||
|
sql = "SELECT * FROM Signatures_Roms WHERE GameId=@gameid AND MD5=@md5";
|
||||||
|
dbDict = new Dictionary<string, object>();
|
||||||
|
dbDict.Add("gameid", gameId);
|
||||||
|
dbDict.Add("name", Common.ReturnValueIfNull(romObject.Name, ""));
|
||||||
|
dbDict.Add("size", Common.ReturnValueIfNull(romObject.Size, ""));
|
||||||
|
dbDict.Add("crc", Common.ReturnValueIfNull(romObject.Crc, ""));
|
||||||
|
dbDict.Add("md5", romObject.Md5);
|
||||||
|
dbDict.Add("sha1", Common.ReturnValueIfNull(romObject.Sha1, ""));
|
||||||
|
dbDict.Add("developmentstatus", Common.ReturnValueIfNull(romObject.DevelopmentStatus, ""));
|
||||||
|
|
||||||
|
if (romObject.flags != null)
|
||||||
|
{
|
||||||
|
if (romObject.flags.Count > 0)
|
||||||
{
|
{
|
||||||
if (romObject.flags.Count > 0)
|
dbDict.Add("flags", Newtonsoft.Json.JsonConvert.SerializeObject(romObject.flags));
|
||||||
{
|
|
||||||
dbDict.Add("flags", Newtonsoft.Json.JsonConvert.SerializeObject(romObject.flags));
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
dbDict.Add("flags", "[ ]");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
dbDict.Add("flags", "[ ]");
|
dbDict.Add("flags", "[ ]");
|
||||||
}
|
}
|
||||||
dbDict.Add("romtype", (int)romObject.RomType);
|
|
||||||
dbDict.Add("romtypemedia", Common.ReturnValueIfNull(romObject.RomTypeMedia, ""));
|
|
||||||
dbDict.Add("medialabel", Common.ReturnValueIfNull(romObject.MediaLabel, ""));
|
|
||||||
dbDict.Add("metadatasource", Classes.Roms.GameRomItem.SourceType.TOSEC);
|
|
||||||
|
|
||||||
bool ImportedRomFound = false;
|
|
||||||
foreach (Dictionary<string, object> ImportedRom in ImportedRoms)
|
|
||||||
{
|
|
||||||
if (((ulong)ImportedRom["gameid"] == gameId) && ((string)ImportedRom["md5"] == romObject.Md5))
|
|
||||||
{
|
|
||||||
ImportedRomFound = true;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if (ImportedRomFound == false)
|
|
||||||
{
|
|
||||||
sigDB = db.ExecuteCMD(sql, dbDict);
|
|
||||||
if (sigDB.Rows.Count == 0)
|
|
||||||
{
|
|
||||||
//if (sQLTransactionItems.Count == transactionCounterMax)
|
|
||||||
//{
|
|
||||||
// db.ExecuteTransactionCMD(sQLTransactionItems);
|
|
||||||
//}
|
|
||||||
|
|
||||||
// entry not present, insert it
|
|
||||||
sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel, MetadataSource) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
|
|
||||||
|
|
||||||
//SQLTransactionItem sQLTransactionItem = new SQLTransactionItem(
|
|
||||||
// SQLCommand: sql,
|
|
||||||
// Parameters: dbDict
|
|
||||||
// );
|
|
||||||
|
|
||||||
//sQLTransactionItems.Add(sQLTransactionItem);
|
|
||||||
|
|
||||||
SaveTransaction(new SQLTransactionItem
|
|
||||||
{
|
|
||||||
SQLCommand = sql,
|
|
||||||
Parameters = dbDict
|
|
||||||
});
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
romId = (int)sigDB.Rows[0][0];
|
|
||||||
}
|
|
||||||
|
|
||||||
Dictionary<string, object> romDetails = new Dictionary<string, object>();
|
|
||||||
romDetails.Add("gameid", dbDict["gameid"]);
|
|
||||||
romDetails.Add("md5", dbDict["md5"]);
|
|
||||||
romDetails.Add("name", dbDict["name"]);
|
|
||||||
ImportedRoms.Add(romDetails);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
else
|
||||||
|
{
|
||||||
|
dbDict.Add("flags", "[ ]");
|
||||||
|
}
|
||||||
|
dbDict.Add("romtype", (int)romObject.RomType);
|
||||||
|
dbDict.Add("romtypemedia", Common.ReturnValueIfNull(romObject.RomTypeMedia, ""));
|
||||||
|
dbDict.Add("medialabel", Common.ReturnValueIfNull(romObject.MediaLabel, ""));
|
||||||
|
dbDict.Add("metadatasource", Classes.Roms.GameRomItem.SourceType.TOSEC);
|
||||||
|
|
||||||
if (sQLTransactionItems.Count > 0)
|
sigDB = db.ExecuteCMD(sql, dbDict);
|
||||||
{
|
if (sigDB.Rows.Count == 0)
|
||||||
//db.ExecuteTransactionCMD(sQLTransactionItems);
|
{
|
||||||
SaveTransaction();
|
// entry not present, insert it
|
||||||
|
sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel, MetadataSource) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
|
||||||
|
sigDB = db.ExecuteCMD(sql, dbDict);
|
||||||
|
|
||||||
|
|
||||||
|
romId = Convert.ToInt32(sigDB.Rows[0][0]);
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
romId = (int)sigDB.Rows[0][0];
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
catch (Exception ex)
|
|
||||||
{
|
|
||||||
Logging.Log(Logging.LogType.Warning, "Signature Ingestor - TOSEC", "Failed to import file " + tosecXMLFile, ex);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
else
|
catch (Exception ex)
|
||||||
{
|
{
|
||||||
Logging.Log(Logging.LogType.Debug, "Signature Ingestor - TOSEC", "Rejecting already imported file: " + tosecXMLFile);
|
Logging.Log(Logging.LogType.Warning, "Signature Ingestor - TOSEC", "Invalid import file: " + tosecXMLFile, ex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
Logging.Log(Logging.LogType.Debug, "Signature Ingestor - TOSEC", "Rejecting already imported file: " + tosecXMLFile);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private List<SQLTransactionItem> sQLTransactionItems = new List<SQLTransactionItem>();
|
|
||||||
|
|
||||||
private void SaveTransaction()
|
|
||||||
{
|
|
||||||
if (sQLTransactionItems.Count > 0)
|
|
||||||
{
|
|
||||||
db.ExecuteTransactionCMD(sQLTransactionItems);
|
|
||||||
sQLTransactionItems.Clear();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private void SaveTransaction(SQLTransactionItem sQLTransactionItem)
|
|
||||||
{
|
|
||||||
sQLTransactionItems.Add(sQLTransactionItem);
|
|
||||||
if (sQLTransactionItems.Count > 1000)
|
|
||||||
{
|
|
||||||
db.ExecuteTransactionCMD(sQLTransactionItems);
|
|
||||||
sQLTransactionItems.Clear();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -577,9 +577,7 @@ CREATE TABLE `Signatures_Games` (
|
|||||||
UNIQUE KEY `Id_UNIQUE` (`Id`),
|
UNIQUE KEY `Id_UNIQUE` (`Id`),
|
||||||
KEY `publisher_Idx` (`PublisherId`),
|
KEY `publisher_Idx` (`PublisherId`),
|
||||||
KEY `system_Idx` (`SystemId`),
|
KEY `system_Idx` (`SystemId`),
|
||||||
KEY `ingest_Idx` (`Name`,`Year`,`PublisherId`,`SystemId`,`Country`,`Language`) USING BTREE,
|
KEY `ingest_Idx` (`Name`,`Year`,`PublisherId`,`SystemId`,`Country`,`Language`) USING BTREE
|
||||||
CONSTRAINT `Publisher` FOREIGN KEY (`PublisherId`) REFERENCES `Signatures_Publishers` (`Id`) ON DELETE CASCADE ON UPDATE CASCADE,
|
|
||||||
CONSTRAINT `System` FOREIGN KEY (`SystemId`) REFERENCES `Signatures_Platforms` (`Id`) ON DELETE CASCADE ON UPDATE CASCADE
|
|
||||||
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||||
/*!40101 SET character_set_client = @saved_cs_client */;
|
/*!40101 SET character_set_client = @saved_cs_client */;
|
||||||
|
|
||||||
@@ -641,8 +639,7 @@ CREATE TABLE `Signatures_Roms` (
|
|||||||
KEY `GameId_Idx` (`GameId`),
|
KEY `GameId_Idx` (`GameId`),
|
||||||
KEY `md5_Idx` (`MD5`) USING BTREE,
|
KEY `md5_Idx` (`MD5`) USING BTREE,
|
||||||
KEY `sha1_Idx` (`SHA1`) USING BTREE,
|
KEY `sha1_Idx` (`SHA1`) USING BTREE,
|
||||||
KEY `flags_Idx` ((cast(`Flags` as char(255) array))),
|
KEY `flags_Idx` ((cast(`Flags` as char(255) array)))
|
||||||
CONSTRAINT `GameId` FOREIGN KEY (`GameId`) REFERENCES `Signatures_Games` (`Id`) ON DELETE CASCADE ON UPDATE CASCADE
|
|
||||||
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
|
||||||
/*!40101 SET character_set_client = @saved_cs_client */;
|
/*!40101 SET character_set_client = @saved_cs_client */;
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user