Split LibraryScan and Rematching into separate background tasks (#166)

* Simplified LibraryScan service and moved rematching to it's own service

* Added rematcher to tasks blocked by OrganiseLibrary

gaseous-server/Classes/ImportGames.cs

@@ -7,6 +7,7 @@ using System.Threading.Tasks;
 using gaseous_server.Classes.Metadata;
 using gaseous_tools;
 using IGDB.Models;
+using NuGet.Common;
 using static gaseous_server.Classes.Metadata.Games;
 
 namespace gaseous_server.Classes
@@ -428,7 +429,7 @@ namespace gaseous_server.Classes
                 sql = "INSERT INTO Games_Roms (PlatformId, GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Attributes, RomType, RomTypeMedia, MediaLabel, Path, MetadataSource, MetadataGameName, MetadataVersion, LibraryId) VALUES (@platformid, @gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @Attributes, @romtype, @romtypemedia, @medialabel, @path, @metadatasource, @metadatagamename, @metadataversion, @libraryid); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
             } else
             {
-                sql = "UPDATE Games_Roms SET PlatformId=platformid, GameId=@gameid, Name=@name, Size=@size, DevelopmentStatus=@developmentstatus, Attributes=@Attributes, RomType=@romtype, RomTypeMedia=@romtypemedia, MediaLabel=@medialabel, MetadataSource=@metadatasource, MetadataGameName=@metadatagamename, MetadataVersion=@metadataversion WHERE Id=@id;";
+                sql = "UPDATE Games_Roms SET PlatformId=@platformid, GameId=@gameid, Name=@name, Size=@size, DevelopmentStatus=@developmentstatus, Attributes=@Attributes, RomType=@romtype, RomTypeMedia=@romtypemedia, MediaLabel=@medialabel, MetadataSource=@metadatasource, MetadataGameName=@metadatagamename, MetadataVersion=@metadataversion WHERE Id=@id;";
                 dbDict.Add("id", UpdateId);
             }
             dbDict.Add("platformid", Common.ReturnValueIfNull(determinedPlatform.Id, 0));
@@ -720,37 +721,6 @@ namespace gaseous_server.Classes
 
                         if (File.Exists(romPath))
                         {
-                            // file exists, so lets check to make sure the signature was matched, and update if a signature can be found
-                            if (
-                                romMetadataSource == gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None ||
-                                (int)dtRoms.Rows[i]["MetadataVersion"] == 1
-                            )
-                            {
-                                Common.hashObject hash = new Common.hashObject
-                                {
-                                    md5hash = (string)dtRoms.Rows[i]["MD5"],
-                                    sha1hash = (string)dtRoms.Rows[i]["SHA1"]
-                                };
-                                FileInfo fi = new FileInfo(romPath);
-
-                                Models.Signatures_Games sig = GetFileSignature(hash, fi, romPath);
-                                if (sig.Rom.SignatureSource != gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None)
-                                {
-                                    Logging.Log(Logging.LogType.Information, "Library Scan", " Update signature found for " + romPath);
-
-                                    // get discovered platform
-                                    IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(sig.Flags.IGDBPlatformId);
-                                    if (determinedPlatform == null)
-                                    {
-                                        determinedPlatform = new IGDB.Models.Platform();
-                                    }
-
-                                    IGDB.Models.Game determinedGame = SearchForGame(sig.Game.Name, sig.Flags.IGDBPlatformId);
-
-                                    StoreROM(library, hash, determinedGame, determinedPlatform, sig, romPath, romId);
-                                }
-                            }
-
                             if (library.IsDefaultLibrary == true)
                             {
                                 if (romPath != ComputeROMPath(romId))
@@ -776,6 +746,57 @@ namespace gaseous_server.Classes
                 Logging.Log(Logging.LogType.Information, "Library Scan", "Library scan completed");
             }
         }
+
+        public static void Rematcher()
+        {
+            // rescan all titles with an unknown platform or title and see if we can get a match
+            Logging.Log(Logging.LogType.Information, "Rematch Scan", "Rematch scan starting");
+
+            Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
+            string sql = "SELECT * FROM Games_Roms WHERE (PlatformId = 0 OR GameId = 0) AND (LastMatchAttemptDate IS NULL OR LastMatchAttemptDate < @lastmatchattemptdate) LIMIT 100;";
+            Dictionary<string, object> dbDict = new Dictionary<string, object>();
+            dbDict.Add("lastmatchattemptdate", DateTime.UtcNow.AddMonths(-1));
+            DataTable data = db.ExecuteCMD(sql, dbDict);
+            foreach (DataRow row in data.Rows)
+            {
+                // get library
+                GameLibrary.LibraryItem library = GameLibrary.GetLibrary((int)row["LibraryId"]);
+
+                // get rom info
+                long romId = (long)row["Id"];
+                string romPath = (string)row["Path"];
+                Common.hashObject hash = new Common.hashObject
+                {
+                    md5hash = (string)row["MD5"],
+                    sha1hash = (string)row["SHA1"]
+                };
+                FileInfo fi = new FileInfo(romPath);
+
+                Logging.Log(Logging.LogType.Information, "Rematch Scan", "Running rematch against " + romPath);
+
+                // determine rom signature
+                Models.Signatures_Games sig = GetFileSignature(hash, fi, romPath);
+
+                // determine rom platform
+                IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(sig.Flags.IGDBPlatformId);
+                if (determinedPlatform == null)
+                {
+                    determinedPlatform = new IGDB.Models.Platform();
+                }
+
+                IGDB.Models.Game determinedGame = SearchForGame(sig.Game.Name, sig.Flags.IGDBPlatformId);
+
+                StoreROM(library, hash, determinedGame, determinedPlatform, sig, romPath, romId);
+
+                string attemptSql = "UPDATE Games_Roms SET LastMatchAttemptDate=@lastmatchattemptdate WHERE Id=@id;";
+                Dictionary<string, object> dbLastAttemptDict = new Dictionary<string, object>();
+                dbLastAttemptDict.Add("id", romId);
+                dbLastAttemptDict.Add("lastmatchattemptdate", DateTime.UtcNow);
+                db.ExecuteCMD(attemptSql, dbLastAttemptDict);
+            }
+
+            Logging.Log(Logging.LogType.Information, "Rematch Scan", "Rematch scan completed");
+        }
     }
 }
 

gaseous-server/Models/PlatformMapping.cs

@@ -4,6 +4,7 @@ using System.Data;
 using System.Linq;
 using System.Reflection;
 using System.Text.Json.Serialization;
+using System.Web;
 using gaseous_server.Classes;
 using gaseous_server.Classes.Metadata;
 using gaseous_server.Controllers;
@@ -201,7 +202,7 @@ namespace gaseous_server.Models
                         sql = "INSERT INTO PlatformMap_AlternateNames (Id, Name) VALUES (@Id, @Name);";
                         dbDict.Clear();
                         dbDict.Add("Id", item.IGDBId);
-                        dbDict.Add("Name", alternateName);
+                        dbDict.Add("Name", HttpUtility.HtmlDecode(alternateName));
                         db.ExecuteCMD(sql, dbDict);
                     }
                 }

gaseous-server/ProcessQueue.cs

@@ -146,6 +146,14 @@ namespace gaseous_server
 
                                     break;
 
+                                case QueueItemType.Rematcher:
+                                    Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Rematch");
+                                    Classes.ImportGame.Rematcher();
+
+                                    _SaveLastRunTime = true;
+
+                                    break;
+
                                 case QueueItemType.CollectionCompiler:
                                     Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Collection Compiler");
                                     Classes.Collections.CompileCollections((long)Options);
@@ -168,6 +176,8 @@ namespace gaseous_server
                         _ForceExecute = false;
                         _ItemState = QueueItemState.Stopped;
                         _LastFinishTime = DateTime.UtcNow;
+
+                        Logging.Log(Logging.LogType.Information, "Timered Event", "Total " + _ItemType + " run time = " + (DateTime.UtcNow - _LastRunTime).TotalSeconds);
                     }
                 }
             }
@@ -220,6 +230,11 @@ namespace gaseous_server
             /// </summary>
             LibraryScan,
 
+            /// <summary>
+            /// Looks for roms in the library that have an unknown platform or game match
+            /// </summary>
+            Rematcher,
+
             /// <summary>
             /// Builds collections - set the options attribute to the id of the collection to build
             /// </summary>

gaseous-server/Program.cs

@@ -182,29 +182,54 @@ gaseous_server.Classes.Metadata.Platforms.GetPlatform(0);
 PlatformMapping.ExtractPlatformMap();
 
 // add background tasks
-ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.SignatureIngestor, 60));
 ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
-    ProcessQueue.QueueItemType.TitleIngestor, 1,
+    ProcessQueue.QueueItemType.SignatureIngestor,
+    60
+    )
+    );
+ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
+    ProcessQueue.QueueItemType.TitleIngestor,
+    1,
     new List<ProcessQueue.QueueItemType>
     {
         ProcessQueue.QueueItemType.OrganiseLibrary,
         ProcessQueue.QueueItemType.LibraryScan
     })
     );
-ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.MetadataRefresh, 360));
 ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
-    ProcessQueue.QueueItemType.OrganiseLibrary, 1440, new List<ProcessQueue.QueueItemType>
+    ProcessQueue.QueueItemType.MetadataRefresh,
+    360
+    )
+    );
+ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
+    ProcessQueue.QueueItemType.OrganiseLibrary,
+    1440,
+    new List<ProcessQueue.QueueItemType>
     {
         ProcessQueue.QueueItemType.LibraryScan,
-        ProcessQueue.QueueItemType.TitleIngestor
+        ProcessQueue.QueueItemType.TitleIngestor,
+        ProcessQueue.QueueItemType.Rematcher
     })
     );
 ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
-    ProcessQueue.QueueItemType.LibraryScan, 1440, new List<ProcessQueue.QueueItemType>
+    ProcessQueue.QueueItemType.LibraryScan,
+    60,
+    new List<ProcessQueue.QueueItemType>
     {
-        ProcessQueue.QueueItemType.OrganiseLibrary
+        ProcessQueue.QueueItemType.OrganiseLibrary,
+        ProcessQueue.QueueItemType.Rematcher
     })
     );
+ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
+    ProcessQueue.QueueItemType.Rematcher,
+    1440,
+    new List<ProcessQueue.QueueItemType>
+    {
+        ProcessQueue.QueueItemType.OrganiseLibrary,
+        ProcessQueue.QueueItemType.LibraryScan
+    })
+    );
+
 
 Logging.WriteToDiskOnly = false;
 

gaseous-server/Support/PlatformMap.json

@@ -882,7 +882,7 @@
         "alternateNames": [
             "NES",
             "Nintendo Entertainment System",
-            "Nintendo Famicom & Entertainment System"
+            "Nintendo Famicom & Entertainment System"
         ],
         "extensions": {
             "supportedFileExtensions": [
@@ -1086,7 +1086,7 @@
         "alternateNames": [
             "Mega CD",
             "Sega CD",
-            "Sega Mega-CD & Sega CD",
+            "Sega Mega-CD & Sega CD",
             "segacd"
         ],
         "extensions": {
@@ -1188,7 +1188,7 @@
         "igdbName": "Sega Master System/Mark III",
         "igdbSlug": "sms",
         "alternateNames": [
-            "Sega Mark III & Master System",
+            "Sega Mark III & Master System",
             "Sega Master System",
             "Sega Master System/Mark III",
             "sms",
@@ -1248,7 +1248,7 @@
             "genesis-slash-megadrive",
             "Sega Genesis",
             "Sega Mega Drive",
-            "Sega Mega Drive & Genesis",
+            "Sega Mega Drive & Genesis",
             "Sega Mega Drive/Genesis"
         ],
         "extensions": {
@@ -1351,7 +1351,7 @@
         "igdbName": "Super Nintendo Entertainment System",
         "igdbSlug": "snes",
         "alternateNames": [
-            "Nintendo Super Famicom & Super Entertainment System",
+            "Nintendo Super Famicom & Super Entertainment System",
             "SNES",
             "SNES, Super Nintendo",
             "Super Nintendo",

gaseous-tools/Database/MySQL/gaseous-1004.sql

@@ -37,3 +37,6 @@ CREATE TABLE `Relation_Game_Themes` (
   PRIMARY KEY (`GameId`, `ThemesId`),
   INDEX `idx_PrimaryColumn` (`GameId` ASC) VISIBLE
 );
+
+ALTER TABLE `Games_Roms` 
+ADD COLUMN `LastMatchAttemptDate` DATETIME NULL AFTER `LibraryId`;