This commit is contained in:
Michael Green
2025-01-06 22:55:06 +11:00
parent 8153c42fe1
commit fc0d164a96
10 changed files with 98 additions and 248 deletions

View File

@@ -394,9 +394,8 @@ namespace gaseous_server.Classes
);
HasheousClient.Models.Metadata.IGDB.Platform platform = Platforms.GetPlatform((long)row["PlatformId"]);
Game game = Games.GetGame(HasheousClient.Models.MetadataSources.IGDB, (long)row["GameId"]);
ImportGame.StoreROM(library, hash, game, platform, signature, (string)row["Path"], (long)row["Id"]);
ImportGame.StoreGame(library, hash, signature, platform, (string)row["Path"], (long)row["Id"]);
count += 1;
}

View File

@@ -465,83 +465,6 @@ namespace gaseous_server.Classes
return SearchCandidates;
}
public static long StoreROM(GameLibrary.LibraryItem library, Common.hashObject hash, gaseous_server.Models.Game determinedGame, Platform determinedPlatform, gaseous_server.Models.Signatures_Games discoveredSignature, string GameFileImportPath, long UpdateId = 0, bool SourceIsExternal = false)
{
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
if (UpdateId == 0)
{
sql = "INSERT INTO Games_Roms (PlatformId, GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Attributes, RomType, RomTypeMedia, MediaLabel, RelativePath, MetadataSource, MetadataGameName, MetadataVersion, LibraryId, RomDataVersion) VALUES (@platformid, @gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @Attributes, @romtype, @romtypemedia, @medialabel, @path, @metadatasource, @metadatagamename, @metadataversion, @libraryid, @romdataversion); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
}
else
{
sql = "UPDATE Games_Roms SET PlatformId=@platformid, GameId=@gameid, Name=@name, Size=@size, DevelopmentStatus=@developmentstatus, Attributes=@Attributes, RomType=@romtype, RomTypeMedia=@romtypemedia, MediaLabel=@medialabel, MetadataSource=@metadatasource, MetadataGameName=@metadatagamename, MetadataVersion=@metadataversion, RomDataVersion=@romdataversion WHERE Id=@id;";
dbDict.Add("id", UpdateId);
}
dbDict.Add("platformid", Common.ReturnValueIfNull(determinedPlatform.Id, 0));
dbDict.Add("gameid", Common.ReturnValueIfNull(determinedGame.Id, 0));
dbDict.Add("name", Common.ReturnValueIfNull(discoveredSignature.Rom.Name, 0));
dbDict.Add("size", Common.ReturnValueIfNull(discoveredSignature.Rom.Size, 0));
dbDict.Add("md5", hash.md5hash);
dbDict.Add("sha1", hash.sha1hash);
dbDict.Add("crc", Common.ReturnValueIfNull(discoveredSignature.Rom.Crc, ""));
dbDict.Add("developmentstatus", Common.ReturnValueIfNull(discoveredSignature.Rom.DevelopmentStatus, ""));
dbDict.Add("metadatasource", discoveredSignature.Rom.SignatureSource);
dbDict.Add("metadatagamename", discoveredSignature.Game.Name);
dbDict.Add("metadataversion", 2);
dbDict.Add("libraryid", library.Id);
dbDict.Add("romdataversion", 2);
if (discoveredSignature.Rom.Attributes != null)
{
if (discoveredSignature.Rom.Attributes.Count > 0)
{
dbDict.Add("attributes", Newtonsoft.Json.JsonConvert.SerializeObject(discoveredSignature.Rom.Attributes));
}
else
{
dbDict.Add("attributes", "[ ]");
}
}
else
{
dbDict.Add("attributes", "[ ]");
}
dbDict.Add("romtype", (int)discoveredSignature.Rom.RomType);
dbDict.Add("romtypemedia", Common.ReturnValueIfNull(discoveredSignature.Rom.RomTypeMedia, ""));
dbDict.Add("medialabel", Common.ReturnValueIfNull(discoveredSignature.Rom.MediaLabel, ""));
string libraryRootPath = library.Path;
if (libraryRootPath.EndsWith(Path.DirectorySeparatorChar.ToString()) == false)
{
libraryRootPath += Path.DirectorySeparatorChar;
}
dbDict.Add("path", GameFileImportPath.Replace(libraryRootPath, ""));
DataTable romInsert = db.ExecuteCMD(sql, dbDict);
long romId = 0;
if (UpdateId == 0)
{
romId = (long)romInsert.Rows[0][0];
}
else
{
romId = UpdateId;
}
// move to destination
if (library.IsDefaultLibrary == true)
{
MoveGameFile(romId, SourceIsExternal);
}
return romId;
}
public static string ComputeROMPath(long RomId)
{
Classes.Roms.GameRomItem rom = Classes.Roms.GetRom(RomId);
@@ -716,8 +639,7 @@ namespace gaseous_server.Classes
1,
new List<ProcessQueue.QueueItemType>
{
ProcessQueue.QueueItemType.OrganiseLibrary,
ProcessQueue.QueueItemType.Rematcher
ProcessQueue.QueueItemType.OrganiseLibrary
},
false,
true)
@@ -876,7 +798,7 @@ namespace gaseous_server.Classes
gaseous_server.Models.Game determinedGame = SearchForGame(sig, PlatformId, true);
StoreROM(library, hash, determinedGame, determinedPlatform, sig, LibraryFile);
StoreGame(library, hash, sig, determinedPlatform, LibraryFile, 0, false);
}
catch (Exception ex)
{
@@ -938,86 +860,6 @@ namespace gaseous_server.Classes
Logging.Log(Logging.LogType.Information, "Library Scan", "Library scan completed");
}
public void Rematcher(bool ForceExecute = false)
{
// rescan all titles with an unknown platform or title and see if we can get a match
Logging.Log(Logging.LogType.Information, "Rematch Scan", "Rematch scan starting");
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
foreach (GameLibrary.LibraryItem library in GameLibrary.GetLibraries)
{
Logging.Log(Logging.LogType.Information, "Rematch Scan", "Rematch on library " + library.Name);
string sql = "";
if (ForceExecute == false)
{
sql = "SELECT * FROM view_Games_Roms WHERE (PlatformId = 0 AND GameId <> 0) OR (((PlatformId = 0 OR GameId = 0) AND MetadataSource = 0) OR (PlatformId = 0 AND GameId = 0)) AND (LastMatchAttemptDate IS NULL OR LastMatchAttemptDate < @lastmatchattemptdate) AND LibraryId = @libraryid LIMIT 100;";
}
else
{
sql = "SELECT * FROM view_Games_Roms WHERE (PlatformId = 0 AND GameId <> 0) OR (((PlatformId = 0 OR GameId = 0) AND MetadataSource = 0) OR (PlatformId = 0 AND GameId = 0)) AND LibraryId = @libraryid;";
}
Dictionary<string, object> dbDict = new Dictionary<string, object>();
dbDict.Add("lastmatchattemptdate", DateTime.UtcNow.AddDays(-7));
dbDict.Add("libraryid", library.Id);
DataTable data = db.ExecuteCMD(sql, dbDict);
int StatusCount = -0;
foreach (DataRow row in data.Rows)
{
SetStatus(StatusCount, data.Rows.Count, "Running rematcher");
// get rom info
long romId = (long)row["Id"];
string romPath = (string)row["Path"];
Common.hashObject hash = new Common.hashObject
{
md5hash = (string)row["MD5"],
sha1hash = (string)row["SHA1"]
};
FileInfo fi = new FileInfo(romPath);
Logging.Log(Logging.LogType.Information, "Rematch Scan", "Running rematch against " + romPath);
// determine rom signature
FileSignature fileSignature = new FileSignature();
gaseous_server.Models.Signatures_Games sig = fileSignature.GetFileSignature(library, hash, fi, romPath);
// get discovered platform
long PlatformId;
Platform determinedPlatform;
if (sig.Flags.PlatformId == null || sig.Flags.PlatformId == 0)
{
// no platform discovered in the signature
PlatformId = library.DefaultPlatformId;
}
else
{
// use the platform discovered in the signature
PlatformId = (long)sig.Flags.PlatformId;
}
determinedPlatform = Platforms.GetPlatform(PlatformId);
gaseous_server.Models.Game determinedGame = SearchForGame(sig, PlatformId, true);
StoreROM(library, hash, determinedGame, determinedPlatform, sig, romPath, romId);
string attemptSql = "UPDATE Games_Roms SET LastMatchAttemptDate=@lastmatchattemptdate WHERE Id=@id;";
Dictionary<string, object> dbLastAttemptDict = new Dictionary<string, object>();
dbLastAttemptDict.Add("id", romId);
dbLastAttemptDict.Add("lastmatchattemptdate", DateTime.UtcNow);
db.ExecuteCMD(attemptSql, dbLastAttemptDict);
StatusCount += 1;
}
ClearStatus();
Logging.Log(Logging.LogType.Information, "Rematch Scan", "Rematch scan completed");
ClearStatus();
}
}
}
}

View File

@@ -231,7 +231,7 @@ namespace gaseous_server.Classes.Metadata
}
}
public static List<AvailablePlatformItem> GetAvailablePlatforms(string UserId, long GameId)
public static List<AvailablePlatformItem> GetAvailablePlatforms(string UserId, HasheousClient.Models.MetadataSources SourceType, long GameId)
{
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = @"
@@ -273,10 +273,11 @@ FROM
LEFT JOIN
view_Games_Roms AS GFV ON GFV.Id = User_GameFavouriteRoms.RomId
WHERE
view_Games_Roms.MetadataMapId = @gameid
view_Games_Roms.GameIdType = @sourcetype AND view_Games_Roms.GameId = @gameid
ORDER BY Platform.`Name`;";
Dictionary<string, object> dbDict = new Dictionary<string, object>
{
{ "sourcetype", (int)SourceType },
{ "gameid", GameId },
{ "userid", UserId }
};

View File

@@ -1,3 +1,5 @@
using System.Data;
namespace gaseous_server.Classes.Metadata
{
public class Metadata
@@ -174,17 +176,43 @@ namespace gaseous_server.Classes.Metadata
// get T type as string
string type = typeof(T).Name;
// get metadata from the server
Communications comms = new Communications();
var results = await comms.APIComm<T>(SourceType, (Communications.MetadataEndpoint)Enum.Parse(typeof(Communications.MetadataEndpoint), type, true), Id);
// check for errors
if (results == null)
if (SourceType == HasheousClient.Models.MetadataSources.None)
{
throw new InvalidMetadataId(SourceType, Id);
}
// generate a dummy object
var returnObject = (T)Activator.CreateInstance(typeof(T));
returnObject.GetType().GetProperty("Id").SetValue(returnObject, Id);
return results.FirstOrDefault<T>();
// if returnObject has a property called "name", query the metadatamap view for the name
if (returnObject.GetType().GetProperty("Name") != null)
{
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "SELECT SignatureGameName FROM view_MetadataMap WHERE `Id` = @id;";
DataTable dataTable = db.ExecuteCMD(sql, new Dictionary<string, object>
{
{ "@id", Id }
});
if (dataTable.Rows.Count > 0)
{
returnObject.GetType().GetProperty("Name").SetValue(returnObject, dataTable.Rows[0]["SignatureGameName"].ToString());
}
}
return returnObject;
}
else
{
// get metadata from the server
Communications comms = new Communications();
var results = await comms.APIComm<T>(SourceType, (Communications.MetadataEndpoint)Enum.Parse(typeof(Communications.MetadataEndpoint), type, true), Id);
// check for errors
if (results == null)
{
throw new InvalidMetadataId(SourceType, Id);
}
return results.FirstOrDefault<T>();
}
}
private static async Task<T> GetMetadataFromServer<T>(HasheousClient.Models.MetadataSources SourceType, string Id) where T : class
@@ -192,17 +220,25 @@ namespace gaseous_server.Classes.Metadata
// get T type as string
string type = typeof(T).Name;
// get metadata from the server
Communications comms = new Communications();
var results = await comms.APIComm<T>(SourceType, (Communications.MetadataEndpoint)Enum.Parse(typeof(Communications.MetadataEndpoint), type, true), Id);
// check for errors
if (results == null)
if (SourceType == HasheousClient.Models.MetadataSources.None)
{
throw new InvalidMetadataId(SourceType, Id);
// generate a dummy object
return (T)Activator.CreateInstance(typeof(T));
}
else
{
// get metadata from the server
Communications comms = new Communications();
var results = await comms.APIComm<T>(SourceType, (Communications.MetadataEndpoint)Enum.Parse(typeof(Communications.MetadataEndpoint), type, true), Id);
return results.FirstOrDefault<T>();
// check for errors
if (results == null)
{
throw new InvalidMetadataId(SourceType, Id);
}
return results.FirstOrDefault<T>();
}
}
#endregion

View File

@@ -217,13 +217,13 @@ namespace gaseous_server.Classes.Metadata
newObjectValue = Newtonsoft.Json.JsonConvert.SerializeObject(newDict["Ids"]);
objectDict[key.Key] = newObjectValue;
StoreRelations(ObjectTypeName, key.Key, (long)objectDict["Id"], newObjectValue);
StoreRelations(SourceType, ObjectTypeName, key.Key, (long)objectDict["Id"], newObjectValue);
break;
case "list":
newObjectValue = Newtonsoft.Json.JsonConvert.SerializeObject(objectValue);
objectDict[key.Key] = newObjectValue;
StoreRelations(ObjectTypeName, key.Key, (long)objectDict["Id"], newObjectValue);
StoreRelations(SourceType, ObjectTypeName, key.Key, (long)objectDict["Id"], newObjectValue);
break;
case "int32[]":
@@ -372,7 +372,7 @@ namespace gaseous_server.Classes.Metadata
return EndpointType;
}
private static void StoreRelations(string PrimaryTable, string SecondaryTable, long ObjectId, string Relations)
private static void StoreRelations(HasheousClient.Models.MetadataSources SourceType, string PrimaryTable, string SecondaryTable, long ObjectId, string Relations)
{
string TableName = "Relation_" + PrimaryTable + "_" + SecondaryTable;
Database db = new Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
@@ -381,7 +381,13 @@ namespace gaseous_server.Classes.Metadata
if (data.Rows.Count == 0)
{
// table doesn't exist, create it
sql = "CREATE TABLE `" + Config.DatabaseConfiguration.DatabaseName + "`.`" + TableName + "` (`" + PrimaryTable + "Id` BIGINT NOT NULL, `" + SecondaryTable + "Id` BIGINT NOT NULL, PRIMARY KEY (`" + PrimaryTable + "Id`, `" + SecondaryTable + "Id`), INDEX `idx_PrimaryColumn` (`" + PrimaryTable + "Id` ASC) VISIBLE);";
sql = @"
CREATE TABLE
`" + Config.DatabaseConfiguration.DatabaseName + "`.`" + TableName + @"`
(`" + PrimaryTable + @"SourceId` INT NOT NULL,
`" + PrimaryTable + @"Id` BIGINT NOT NULL,
`" + SecondaryTable + @"Id` BIGINT NOT NULL,
PRIMARY KEY (`" + PrimaryTable + "SourceId`, `" + PrimaryTable + "Id`, `" + SecondaryTable + "Id`), INDEX `idx_PrimaryColumn` (`" + PrimaryTable + "Id` ASC) VISIBLE);";
db.ExecuteCMD(sql);
}
else
@@ -397,10 +403,13 @@ namespace gaseous_server.Classes.Metadata
long[] RelationValues = Newtonsoft.Json.JsonConvert.DeserializeObject<long[]>(Relations);
foreach (long RelationValue in RelationValues)
{
sql = "INSERT INTO " + TableName + " (`" + PrimaryTable + "Id`, `" + SecondaryTable + "Id`) VALUES (@objectid, @relationvalue);";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
dbDict.Add("objectid", ObjectId);
dbDict.Add("relationvalue", RelationValue);
sql = "INSERT INTO " + TableName + " (`" + PrimaryTable + "SourceId`, `" + PrimaryTable + "Id`, `" + SecondaryTable + "Id`) VALUES (@sourceid, @objectid, @relationvalue);";
Dictionary<string, object> dbDict = new Dictionary<string, object>
{
{ "sourceid", SourceType },
{ "objectid", ObjectId },
{ "relationvalue", RelationValue }
};
db.ExecuteCMD(sql, dbDict);
}
}
@@ -422,7 +431,13 @@ namespace gaseous_server.Classes.Metadata
if (data.Rows.Count == 0)
{
// table doesn't exist, create it
sql = "CREATE TABLE `" + Config.DatabaseConfiguration.DatabaseName + "`.`" + TableName + "` (`" + PrimaryTable + "Id` BIGINT NOT NULL, `" + SecondaryTable + "Id` BIGINT NOT NULL, PRIMARY KEY (`" + PrimaryTable + "Id`, `" + SecondaryTable + "Id`), INDEX `idx_PrimaryColumn` (`" + PrimaryTable + "Id` ASC) VISIBLE);";
sql = @"
CREATE TABLE
`" + Config.DatabaseConfiguration.DatabaseName + "`.`" + TableName + @"`
(`" + PrimaryTable + @"SourceId` INT NOT NULL,
`" + PrimaryTable + @"Id` BIGINT NOT NULL,
`" + SecondaryTable + @"Id` BIGINT NOT NULL,
PRIMARY KEY (`" + PrimaryTable + "SourceId`, `" + PrimaryTable + "Id`, `" + SecondaryTable + "Id`), INDEX `idx_PrimaryColumn` (`" + PrimaryTable + "Id` ASC) VISIBLE);";
db.ExecuteCMD(sql);
}
}

View File

@@ -334,7 +334,7 @@ namespace gaseous_server.Classes
else
{
// when run normally, update all games (since this will honour cache timeouts)
sql = "SELECT MetadataSourceId AS `Id`, MetadataSourceType AS `GameIdType`, SignatureGameName AS `Name` FROM gaseous.view_MetadataMap;";
sql = "SELECT DISTINCT MetadataSourceId AS `Id`, MetadataSourceType AS `GameIdType`, SignatureGameName AS `Name` FROM gaseous.view_MetadataMap;";
}
dt = db.ExecuteCMD(sql);

View File

@@ -1060,7 +1060,8 @@ namespace gaseous_server.Controllers
if (user != null)
{
return Ok(Games.GetAvailablePlatforms(user.Id, MetadataMapId));
MetadataMap.MetadataMapItem metadataMap = Classes.MetadataManagement.GetMetadataMap(MetadataMapId).PreferredMetadataMapItem;
return Ok(Games.GetAvailablePlatforms(user.Id, metadataMap.SourceType, metadataMap.SourceId));
}
else
{

View File

@@ -525,8 +525,7 @@ ORDER BY Platform.`Name`; ";
this._Blocks = new List<ProcessQueue.QueueItemType>{
ProcessQueue.QueueItemType.LibraryScan,
ProcessQueue.QueueItemType.LibraryScanWorker,
ProcessQueue.QueueItemType.TitleIngestor,
ProcessQueue.QueueItemType.Rematcher
ProcessQueue.QueueItemType.TitleIngestor
};
break;
@@ -548,32 +547,7 @@ ORDER BY Platform.`Name`; ";
this.DefaultAllowedEndHours = 23;
this.DefaultAllowedEndMinutes = 59;
this._Blocks = new List<ProcessQueue.QueueItemType>{
ProcessQueue.QueueItemType.OrganiseLibrary,
ProcessQueue.QueueItemType.Rematcher
};
break;
case ProcessQueue.QueueItemType.Rematcher:
this._UserManageable = true;
this.DefaultInterval = 1440;
this.MinimumAllowedInterval = 360;
this.DefaultAllowedDays = new List<DayOfWeek>{
DayOfWeek.Sunday,
DayOfWeek.Monday,
DayOfWeek.Tuesday,
DayOfWeek.Wednesday,
DayOfWeek.Thursday,
DayOfWeek.Friday,
DayOfWeek.Saturday
};
this.DefaultAllowedStartHours = 0;
this.DefaultAllowedStartMinutes = 0;
this.DefaultAllowedEndHours = 23;
this.DefaultAllowedEndMinutes = 59;
this._Blocks = new List<ProcessQueue.QueueItemType>{
ProcessQueue.QueueItemType.OrganiseLibrary,
ProcessQueue.QueueItemType.LibraryScan,
ProcessQueue.QueueItemType.LibraryScanWorker
ProcessQueue.QueueItemType.OrganiseLibrary
};
break;

View File

@@ -9,8 +9,8 @@ using NuGet.Packaging;
namespace gaseous_server
{
public static class ProcessQueue
{
public static class ProcessQueue
{
public static List<QueueItem> QueueItems = new List<QueueItem>();
public class QueueItem
@@ -115,8 +115,8 @@ namespace gaseous_server
};
private List<QueueItemType> _Blocks = new List<QueueItemType>();
public List<DayOfWeek> AllowedDays
{
public List<DayOfWeek> AllowedDays
{
get
{
return _AllowedDays;
@@ -124,7 +124,7 @@ namespace gaseous_server
set
{
_AllowedDays = value;
}
}
}
public int AllowedStartHours { get; set; } = 0;
public int AllowedStartMinutes { get; set; } = 0;
@@ -135,7 +135,7 @@ namespace gaseous_server
public DateTime LastRunTime => _LastRunTime;
public DateTime LastFinishTime => _LastFinishTime;
public double LastRunDuration => _LastRunDuration;
public DateTime NextRunTime
public DateTime NextRunTime
{
get
{
@@ -338,18 +338,6 @@ namespace gaseous_server
break;
case QueueItemType.Rematcher:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Rematch");
Classes.ImportGame importRematch = new ImportGame
{
CallingQueueItem = this
};
importRematch.Rematcher(_ForceExecute);
_SaveLastRunTime = true;
break;
case QueueItemType.CollectionCompiler:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Collection Compiler");
Dictionary<string, object> collectionOptions = (Dictionary<string, object>)Options;
@@ -368,7 +356,8 @@ namespace gaseous_server
case QueueItemType.DailyMaintainer:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Daily Maintenance");
Classes.Maintenance maintenance = new Maintenance{
Classes.Maintenance maintenance = new Maintenance
{
CallingQueueItem = this
};
maintenance.RunDailyMaintenance();
@@ -379,7 +368,8 @@ namespace gaseous_server
case QueueItemType.WeeklyMaintainer:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Weekly Maintenance");
Classes.Maintenance weeklyMaintenance = new Maintenance{
Classes.Maintenance weeklyMaintenance = new Maintenance
{
CallingQueueItem = this
};
weeklyMaintenance.RunWeeklyMaintenance();
@@ -572,11 +562,6 @@ namespace gaseous_server
/// </summary>
LibraryScanWorker,
/// <summary>
/// Looks for roms in the library that have an unknown platform or game match
/// </summary>
Rematcher,
/// <summary>
/// Builds collections - set the options attribute to the id of the collection to build
/// </summary>

View File

@@ -351,9 +351,6 @@ ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.LibraryScan)
);
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.Rematcher)
);
// maintenance tasks
ProcessQueue.QueueItem dailyMaintenance = new ProcessQueue.QueueItem(