refactor: tosec dat import is now batched using mysql transactions

gaseous-server/Classes/SignatureIngestors/TOSEC.cs

@@ -5,6 +5,7 @@ using gaseous_romsignatureobject;
 using gaseous_signature_parser.parsers;
 using gaseous_tools;
 using MySqlX.XDevAPI;
+using static gaseous_tools.Database;
 
 namespace gaseous_server.SignatureIngestors.TOSEC
 {
@@ -18,6 +19,7 @@ namespace gaseous_server.SignatureIngestors.TOSEC
             List<Dictionary<string, object>> ImportedPlatforms = new List<Dictionary<string, object>>();
             List<Dictionary<string, object>> ImportedPublishers = new List<Dictionary<string, object>>();
             List<Dictionary<string, object>> ImportedGames = new List<Dictionary<string, object>>();
+            List<Dictionary<string, object>> ImportedRoms = new List<Dictionary<string, object>>();
 
             // process provided files
             Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing from " + SearchPath);
@@ -236,6 +238,8 @@ namespace gaseous_server.SignatureIngestors.TOSEC
                                     }
 
                                     // store rom
+                                    int transactionCounterMax = 1000;
+                                    List<SQLTransactionItem> sQLTransactionItems = new List<SQLTransactionItem>();
                                     foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms)
                                     {
                                         if (romObject.Md5 != null)
@@ -271,23 +275,55 @@ namespace gaseous_server.SignatureIngestors.TOSEC
                                             dbDict.Add("medialabel", Common.ReturnValueIfNull(romObject.MediaLabel, ""));
                                             dbDict.Add("metadatasource", Classes.Roms.GameRomItem.SourceType.TOSEC);
 
+                                            bool ImportedRomFound = false;
+                                            foreach (Dictionary<string, object> ImportedRom in ImportedRoms)
+                                            {
+                                                if (((int)ImportedRom["gameid"] == gameId) && ((string)ImportedRom["md5"] == romObject.Md5))
+                                                {
+                                                    ImportedRomFound = true;
+                                                    break;
+                                                }
+                                            }
+
+                                            if (ImportedRomFound == false)
+                                            {
                                                 sigDB = db.ExecuteCMD(sql, dbDict);
                                                 if (sigDB.Rows.Count == 0)
                                                 {
+                                                    if (sQLTransactionItems.Count == transactionCounterMax)
+                                                    {
+                                                        db.ExecuteTransactionCMD(sQLTransactionItems);
+                                                    }
+
                                                     // entry not present, insert it
                                                     sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel, MetadataSource) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
-                                                sigDB = db.ExecuteCMD(sql, dbDict);
 
+                                                    SQLTransactionItem sQLTransactionItem = new SQLTransactionItem(
+                                                            SQLCommand: sql,
+                                                            Parameters: dbDict
+                                                        );
 
-                                                romId = Convert.ToInt32(sigDB.Rows[0][0]);
+                                                    sQLTransactionItems.Add(sQLTransactionItem);
                                                 }
                                                 else
                                                 {
                                                     romId = (int)sigDB.Rows[0][0];
                                                 }
+
+                                                Dictionary<string, object> romDetails = new Dictionary<string, object>();
+                                                romDetails.Add("gameid", dbDict["gameid"]);
+                                                romDetails.Add("md5", dbDict["md5"]);
+                                                romDetails.Add("name", dbDict["name"]);
+                                                ImportedRoms.Add(romDetails);
                                             }
                                         }
                                     }
+
+                                    if (sQLTransactionItems.Count > 0)
+                                    {
+                                        db.ExecuteTransactionCMD(sQLTransactionItems);
+                                    }
+                                }
                             }
                         }
                     }

gaseous-tools/Database.cs

@@ -4,6 +4,7 @@ using System.Data.SqlClient;
 using System.Diagnostics;
 using System.Reflection;
 using MySql.Data.MySqlClient;
+using static gaseous_tools.Database;
 
 namespace gaseous_tools
 {
@@ -154,6 +155,46 @@ namespace gaseous_tools
             }
         }
 
+        public void ExecuteTransactionCMD(List<SQLTransactionItem> CommandList, int Timeout = 60)
+        {
+            object conn;
+            switch (_ConnectorType)
+            {
+                case databaseType.MySql:
+                    {
+                        var commands = new List<Dictionary<string, object>>();
+                        foreach (SQLTransactionItem CommandItem in CommandList)
+                        {
+                            var nCmd = new Dictionary<string, object>();
+                            nCmd.Add("sql", CommandItem.SQLCommand);
+                            nCmd.Add("values", CommandItem.Parameters);
+                            commands.Add(nCmd);
+                        }
+
+                        conn = new MySQLServerConnector(_ConnectionString);
+                        ((MySQLServerConnector)conn).TransactionExecCMD(commands, Timeout);
+                        break;
+                    }
+            }
+        }
+
+        public class SQLTransactionItem
+        {
+			public SQLTransactionItem()
+			{
+
+			}
+
+			public SQLTransactionItem(string SQLCommand, Dictionary<string, object> Parameters)
+			{
+				this.SQLCommand = SQLCommand;
+				this.Parameters = Parameters;
+			}
+
+            public string? SQLCommand;
+            public Dictionary<string, object>? Parameters = new Dictionary<string, object>();
+        }
+
         private partial class MySQLServerConnector
 		{
 			private string DBConn = "";
@@ -203,6 +244,48 @@ namespace gaseous_tools
 
 				return RetTable;
 			}
+
+            public void TransactionExecCMD(List<Dictionary<string, object>> Parameters, int Timeout)
+            {
+                var conn = new MySqlConnection(DBConn);
+                conn.Open();
+                var command = conn.CreateCommand();
+                MySqlTransaction transaction;
+                transaction = conn.BeginTransaction();
+                command.Connection = conn;
+                command.Transaction = transaction;
+                foreach (Dictionary<string, object> Parameter in Parameters)
+                {
+                    var cmd = buildcommand(conn, Parameter["sql"].ToString(), (Dictionary<string, object>)Parameter["values"], Timeout);
+                    cmd.Transaction = transaction;
+                    cmd.ExecuteNonQuery();
+                }
+
+                transaction.Commit();
+                conn.Close();
+            }
+
+            private MySqlCommand buildcommand(MySqlConnection Conn, string SQL, Dictionary<string, object> Parameters, int Timeout)
+            {
+                var cmd = new MySqlCommand();
+                cmd.Connection = Conn;
+                cmd.CommandText = SQL;
+                cmd.CommandTimeout = Timeout;
+                {
+                    var withBlock = cmd.Parameters;
+                    if (Parameters is object)
+                    {
+                        if (Parameters.Count > 0)
+                        {
+                            foreach (string param in Parameters.Keys)
+                                withBlock.AddWithValue(param, Parameters[param]);
+                        }
+                    }
+                }
+
+                return cmd;
+            }
+
         }
     }
 }