sendnrw/gaseous-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Split LibraryScan and Rematching into separate background tasks (#166)

Browse Source 
* Simplified LibraryScan service and moved rematching to it's own service

* Added rematcher to tasks blocked by OrganiseLibrary
This commit is contained in:
Michael Green
2023-10-18 08:56:02 +11:00
committed by GitHub
parent 8688e1d5c0
commit 7ae6eb82f0
6 changed files with 111 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

85
gaseous-server/Classes/ImportGames.cs
View File

@@ -7,6 +7,7 @@ using System.Threading.Tasks;
using gaseous_server.Classes.Metadata;
using gaseous_tools;
using IGDB.Models;
using NuGet.Common;
using static gaseous_server.Classes.Metadata.Games;
namespace gaseous_server.Classes
@@ -428,7 +429,7 @@ namespace gaseous_server.Classes
sql = "INSERT INTO Games_Roms (PlatformId, GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Attributes, RomType, RomTypeMedia, MediaLabel, Path, MetadataSource, MetadataGameName, MetadataVersion, LibraryId) VALUES (@platformid, @gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @Attributes, @romtype, @romtypemedia, @medialabel, @path, @metadatasource, @metadatagamename, @metadataversion, @libraryid); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
} else
{
sql = "UPDATE Games_Roms SET PlatformId=platformid, GameId=@gameid, Name=@name, Size=@size, DevelopmentStatus=@developmentstatus, Attributes=@Attributes, RomType=@romtype, RomTypeMedia=@romtypemedia, MediaLabel=@medialabel, MetadataSource=@metadatasource, MetadataGameName=@metadatagamename, MetadataVersion=@metadataversion WHERE Id=@id;";
sql = "UPDATE Games_Roms SET PlatformId=@platformid, GameId=@gameid, Name=@name, Size=@size, DevelopmentStatus=@developmentstatus, Attributes=@Attributes, RomType=@romtype, RomTypeMedia=@romtypemedia, MediaLabel=@medialabel, MetadataSource=@metadatasource, MetadataGameName=@metadatagamename, MetadataVersion=@metadataversion WHERE Id=@id;";
dbDict.Add("id", UpdateId);
}
dbDict.Add("platformid", Common.ReturnValueIfNull(determinedPlatform.Id, 0));
@@ -720,37 +721,6 @@ namespace gaseous_server.Classes
if (File.Exists(romPath))
{
// file exists, so lets check to make sure the signature was matched, and update if a signature can be found
if (
romMetadataSource == gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None ||
(int)dtRoms.Rows[i]["MetadataVersion"] == 1
)
{
Common.hashObject hash = new Common.hashObject
{
md5hash = (string)dtRoms.Rows[i]["MD5"],
sha1hash = (string)dtRoms.Rows[i]["SHA1"]
};
FileInfo fi = new FileInfo(romPath);
Models.Signatures_Games sig = GetFileSignature(hash, fi, romPath);
if (sig.Rom.SignatureSource != gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None)
{
Logging.Log(Logging.LogType.Information, "Library Scan", " Update signature found for " + romPath);
// get discovered platform
IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(sig.Flags.IGDBPlatformId);
if (determinedPlatform == null)
{
determinedPlatform = new IGDB.Models.Platform();
}
IGDB.Models.Game determinedGame = SearchForGame(sig.Game.Name, sig.Flags.IGDBPlatformId);
StoreROM(library, hash, determinedGame, determinedPlatform, sig, romPath, romId);
}
}
if (library.IsDefaultLibrary == true)
{
if (romPath != ComputeROMPath(romId))
@@ -776,6 +746,57 @@ namespace gaseous_server.Classes
Logging.Log(Logging.LogType.Information, "Library Scan", "Library scan completed");
}
}
public static void Rematcher()
{
// rescan all titles with an unknown platform or title and see if we can get a match
Logging.Log(Logging.LogType.Information, "Rematch Scan", "Rematch scan starting");
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "SELECT * FROM Games_Roms WHERE (PlatformId = 0 OR GameId = 0) AND (LastMatchAttemptDate IS NULL OR LastMatchAttemptDate < @lastmatchattemptdate) LIMIT 100;";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
dbDict.Add("lastmatchattemptdate", DateTime.UtcNow.AddMonths(-1));
DataTable data = db.ExecuteCMD(sql, dbDict);
foreach (DataRow row in data.Rows)
{
// get library
GameLibrary.LibraryItem library = GameLibrary.GetLibrary((int)row["LibraryId"]);
// get rom info
long romId = (long)row["Id"];
string romPath = (string)row["Path"];
Common.hashObject hash = new Common.hashObject
{
md5hash = (string)row["MD5"],
sha1hash = (string)row["SHA1"]
};
FileInfo fi = new FileInfo(romPath);
Logging.Log(Logging.LogType.Information, "Rematch Scan", "Running rematch against " + romPath);
// determine rom signature
Models.Signatures_Games sig = GetFileSignature(hash, fi, romPath);
// determine rom platform
IGDB.Models.Platform determinedPlatform = Metadata.Platforms.GetPlatform(sig.Flags.IGDBPlatformId);
if (determinedPlatform == null)
{
determinedPlatform = new IGDB.Models.Platform();
}
IGDB.Models.Game determinedGame = SearchForGame(sig.Game.Name, sig.Flags.IGDBPlatformId);
StoreROM(library, hash, determinedGame, determinedPlatform, sig, romPath, romId);
string attemptSql = "UPDATE Games_Roms SET LastMatchAttemptDate=@lastmatchattemptdate WHERE Id=@id;";
Dictionary<string, object> dbLastAttemptDict = new Dictionary<string, object>();
dbLastAttemptDict.Add("id", romId);
dbLastAttemptDict.Add("lastmatchattemptdate", DateTime.UtcNow);
db.ExecuteCMD(attemptSql, dbLastAttemptDict);
}
Logging.Log(Logging.LogType.Information, "Rematch Scan", "Rematch scan completed");
}
}
}

3
gaseous-server/Models/PlatformMapping.cs
View File

@@ -4,6 +4,7 @@ using System.Data;
using System.Linq;
using System.Reflection;
using System.Text.Json.Serialization;
using System.Web;
using gaseous_server.Classes;
using gaseous_server.Classes.Metadata;
using gaseous_server.Controllers;
@@ -201,7 +202,7 @@ namespace gaseous_server.Models
sql = "INSERT INTO PlatformMap_AlternateNames (Id, Name) VALUES (@Id, @Name);";
dbDict.Clear();
dbDict.Add("Id", item.IGDBId);
dbDict.Add("Name", alternateName);
dbDict.Add("Name", HttpUtility.HtmlDecode(alternateName));
db.ExecuteCMD(sql, dbDict);
}
}

15
gaseous-server/ProcessQueue.cs
View File

@@ -146,6 +146,14 @@ namespace gaseous_server
break;
case QueueItemType.Rematcher:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Rematch");
Classes.ImportGame.Rematcher();
_SaveLastRunTime = true;
break;
case QueueItemType.CollectionCompiler:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Collection Compiler");
Classes.Collections.CompileCollections((long)Options);
@@ -168,6 +176,8 @@ namespace gaseous_server
_ForceExecute = false;
_ItemState = QueueItemState.Stopped;
_LastFinishTime = DateTime.UtcNow;
Logging.Log(Logging.LogType.Information, "Timered Event", "Total " + _ItemType + " run time = " + (DateTime.UtcNow - _LastRunTime).TotalSeconds);
}
}
}
@@ -220,6 +230,11 @@ namespace gaseous_server
/// </summary>
LibraryScan,
/// <summary>
/// Looks for roms in the library that have an unknown platform or game match
/// </summary>
Rematcher,
/// <summary>
/// Builds collections - set the options attribute to the id of the collection to build
/// </summary>

39
gaseous-server/Program.cs
View File

@@ -182,29 +182,54 @@ gaseous_server.Classes.Metadata.Platforms.GetPlatform(0);
PlatformMapping.ExtractPlatformMap();
// add background tasks
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.SignatureIngestor, 60));
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.TitleIngestor, 1,
ProcessQueue.QueueItemType.SignatureIngestor,
60
)
);
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.TitleIngestor,
1,
new List<ProcessQueue.QueueItemType>
{
ProcessQueue.QueueItemType.OrganiseLibrary,
ProcessQueue.QueueItemType.LibraryScan
})
);
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.MetadataRefresh, 360));
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.OrganiseLibrary, 1440, new List<ProcessQueue.QueueItemType>
ProcessQueue.QueueItemType.MetadataRefresh,
360
)
);
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.OrganiseLibrary,
1440,
new List<ProcessQueue.QueueItemType>
{
ProcessQueue.QueueItemType.LibraryScan,
ProcessQueue.QueueItemType.TitleIngestor
ProcessQueue.QueueItemType.TitleIngestor,
ProcessQueue.QueueItemType.Rematcher
})
);
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.LibraryScan, 1440, new List<ProcessQueue.QueueItemType>
ProcessQueue.QueueItemType.LibraryScan,
60,
new List<ProcessQueue.QueueItemType>
{
ProcessQueue.QueueItemType.OrganiseLibrary
ProcessQueue.QueueItemType.OrganiseLibrary,
ProcessQueue.QueueItemType.Rematcher
})
);
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.Rematcher,
1440,
new List<ProcessQueue.QueueItemType>
{
ProcessQueue.QueueItemType.OrganiseLibrary,
ProcessQueue.QueueItemType.LibraryScan
})
);
Logging.WriteToDiskOnly = false;

10
gaseous-server/Support/PlatformMap.json
View File

@@ -882,7 +882,7 @@
"alternateNames": [
"NES",
"Nintendo Entertainment System",
"Nintendo Famicom &amp;amp; Entertainment System"
"Nintendo Famicom & Entertainment System"
],
"extensions": {
"supportedFileExtensions": [
@@ -1086,7 +1086,7 @@
"alternateNames": [
"Mega CD",
"Sega CD",
"Sega Mega-CD &amp;amp; Sega CD",
"Sega Mega-CD & Sega CD",
"segacd"
],
"extensions": {
@@ -1188,7 +1188,7 @@
"igdbName": "Sega Master System/Mark III",
"igdbSlug": "sms",
"alternateNames": [
"Sega Mark III &amp;amp; Master System",
"Sega Mark III & Master System",
"Sega Master System",
"Sega Master System/Mark III",
"sms",
@@ -1248,7 +1248,7 @@
"genesis-slash-megadrive",
"Sega Genesis",
"Sega Mega Drive",
"Sega Mega Drive &amp; Genesis",
"Sega Mega Drive & Genesis",
"Sega Mega Drive/Genesis"
],
"extensions": {
@@ -1351,7 +1351,7 @@
"igdbName": "Super Nintendo Entertainment System",
"igdbSlug": "snes",
"alternateNames": [
"Nintendo Super Famicom &amp;amp; Super Entertainment System",
"Nintendo Super Famicom & Super Entertainment System",
"SNES",
"SNES, Super Nintendo",
"Super Nintendo",

3
gaseous-tools/Database/MySQL/gaseous-1004.sql
View File

@@ -37,3 +37,6 @@ CREATE TABLE `Relation_Game_Themes` (
PRIMARY KEY (`GameId`, `ThemesId`),
INDEX `idx_PrimaryColumn` (`GameId` ASC) VISIBLE
);
ALTER TABLE `Games_Roms`
ADD COLUMN `LastMatchAttemptDate` DATETIME NULL AFTER `LibraryId`;