sendnrw/gaseous-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: tosec dat import is now batched using mysql transactions

Browse Source
This commit is contained in:
Michael Green
2023-07-04 22:54:28 +10:00
parent 99062da10b
commit 97e8ffa386
2 changed files with 131 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
gaseous-server/Classes/SignatureIngestors/TOSEC.cs
View File

@@ -5,6 +5,7 @@ using gaseous_romsignatureobject;
using gaseous_signature_parser.parsers;
using gaseous_tools;
using MySqlX.XDevAPI;
using static gaseous_tools.Database;
namespace gaseous_server.SignatureIngestors.TOSEC
{
@@ -18,6 +19,7 @@ namespace gaseous_server.SignatureIngestors.TOSEC
List<Dictionary<string, object>> ImportedPlatforms = new List<Dictionary<string, object>>();
List<Dictionary<string, object>> ImportedPublishers = new List<Dictionary<string, object>>();
List<Dictionary<string, object>> ImportedGames = new List<Dictionary<string, object>>();
List<Dictionary<string, object>> ImportedRoms = new List<Dictionary<string, object>>();
// process provided files
Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing from " + SearchPath);
@@ -236,6 +238,8 @@ namespace gaseous_server.SignatureIngestors.TOSEC
}
// store rom
int transactionCounterMax = 1000;
List<SQLTransactionItem> sQLTransactionItems = new List<SQLTransactionItem>();
foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms)
{
if (romObject.Md5 != null)
@@ -271,22 +275,54 @@ namespace gaseous_server.SignatureIngestors.TOSEC
dbDict.Add("medialabel", Common.ReturnValueIfNull(romObject.MediaLabel, ""));
dbDict.Add("metadatasource", Classes.Roms.GameRomItem.SourceType.TOSEC);
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
bool ImportedRomFound = false;
foreach (Dictionary<string, object> ImportedRom in ImportedRoms)
{
// entry not present, insert it
sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel, MetadataSource) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
sigDB = db.ExecuteCMD(sql, dbDict);
romId = Convert.ToInt32(sigDB.Rows[0][0]);
if (((int)ImportedRom["gameid"] == gameId) && ((string)ImportedRom["md5"] == romObject.Md5))
{
ImportedRomFound = true;
break;
}
}
else
if (ImportedRomFound == false)
{
romId = (int)sigDB.Rows[0][0];
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
if (sQLTransactionItems.Count == transactionCounterMax)
{
db.ExecuteTransactionCMD(sQLTransactionItems);
}
// entry not present, insert it
sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel, MetadataSource) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
SQLTransactionItem sQLTransactionItem = new SQLTransactionItem(
SQLCommand: sql,
Parameters: dbDict
);
sQLTransactionItems.Add(sQLTransactionItem);
}
else
{
romId = (int)sigDB.Rows[0][0];
}
Dictionary<string, object> romDetails = new Dictionary<string, object>();
romDetails.Add("gameid", dbDict["gameid"]);
romDetails.Add("md5", dbDict["md5"]);
romDetails.Add("name", dbDict["name"]);
ImportedRoms.Add(romDetails);
}
}
}
if (sQLTransactionItems.Count > 0)
{
db.ExecuteTransactionCMD(sQLTransactionItems);
}
}
}
}

87
gaseous-tools/Database.cs
View File

@@ -4,6 +4,7 @@ using System.Data.SqlClient;
using System.Diagnostics;
using System.Reflection;
using MySql.Data.MySqlClient;
using static gaseous_tools.Database;
namespace gaseous_tools
{
@@ -154,6 +155,46 @@ namespace gaseous_tools
}
}
public void ExecuteTransactionCMD(List<SQLTransactionItem> CommandList, int Timeout = 60)
{
object conn;
switch (_ConnectorType)
{
case databaseType.MySql:
{
var commands = new List<Dictionary<string, object>>();
foreach (SQLTransactionItem CommandItem in CommandList)
{
var nCmd = new Dictionary<string, object>();
nCmd.Add("sql", CommandItem.SQLCommand);
nCmd.Add("values", CommandItem.Parameters);
commands.Add(nCmd);
}
conn = new MySQLServerConnector(_ConnectionString);
((MySQLServerConnector)conn).TransactionExecCMD(commands, Timeout);
break;
}
}
}
public class SQLTransactionItem
{
public SQLTransactionItem()
{
}
public SQLTransactionItem(string SQLCommand, Dictionary<string, object> Parameters)
{
this.SQLCommand = SQLCommand;
this.Parameters = Parameters;
}
public string? SQLCommand;
public Dictionary<string, object>? Parameters = new Dictionary<string, object>();
}
private partial class MySQLServerConnector
{
private string DBConn = "";
@@ -203,7 +244,49 @@ namespace gaseous_tools
return RetTable;
}
}
}
public void TransactionExecCMD(List<Dictionary<string, object>> Parameters, int Timeout)
{
var conn = new MySqlConnection(DBConn);
conn.Open();
var command = conn.CreateCommand();
MySqlTransaction transaction;
transaction = conn.BeginTransaction();
command.Connection = conn;
command.Transaction = transaction;
foreach (Dictionary<string, object> Parameter in Parameters)
{
var cmd = buildcommand(conn, Parameter["sql"].ToString(), (Dictionary<string, object>)Parameter["values"], Timeout);
cmd.Transaction = transaction;
cmd.ExecuteNonQuery();
}
transaction.Commit();
conn.Close();
}
private MySqlCommand buildcommand(MySqlConnection Conn, string SQL, Dictionary<string, object> Parameters, int Timeout)
{
var cmd = new MySqlCommand();
cmd.Connection = Conn;
cmd.CommandText = SQL;
cmd.CommandTimeout = Timeout;
{
var withBlock = cmd.Parameters;
if (Parameters is object)
{
if (Parameters.Count > 0)
{
foreach (string param in Parameters.Keys)
withBlock.AddWithValue(param, Parameters[param]);
}
}
}
return cmd;
}
}
}
}