sendnrw/gaseous-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Expand the dat file ingestor to handle more formats (#84)

Browse Source 
* MAME DAT's can now be imported, beginning of DB updates

* Fixed various database bugs that occur during upgrade from earlier versions

* Removed collation and MySQL specific options from database scripts
This commit is contained in:
Michael Green
2023-09-06 07:52:11 +10:00
committed by GitHub
parent 6b391bc357
commit bd7124a5be
18 changed files with 574 additions and 554 deletions
Download Patch File Download Diff File Expand all files Collapse all files

73
gaseous-server/Classes/ImportGames.cs
View File

@@ -5,6 +5,7 @@ using System.Security.Policy;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using gaseous_tools;
using IGDB.Models;
using MySqlX.XDevAPI;
using Org.BouncyCastle.Utilities.IO.Pem;
using static gaseous_server.Classes.Metadata.Games;
@@ -43,11 +44,7 @@ namespace gaseous_server.Classes
string sql = "";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
string[] SkippableFiles = {
".DS_STORE",
"desktop.ini"
};
if (SkippableFiles.Contains<string>(Path.GetFileName(GameFileImportPath), StringComparer.OrdinalIgnoreCase))
if (Common.SkippableFiles.Contains<string>(Path.GetFileName(GameFileImportPath), StringComparer.OrdinalIgnoreCase))
{
Logging.Log(Logging.LogType.Debug, "Import Game", "Skipping item " + GameFileImportPath);
}
@@ -144,6 +141,17 @@ namespace gaseous_server.Classes
if (zDiscoveredSignature.Score > discoveredSignature.Score)
{
if (
zDiscoveredSignature.Rom.SignatureSource == gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.MAMEArcade ||
zDiscoveredSignature.Rom.SignatureSource == gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.MAMEMess
)
{
zDiscoveredSignature.Rom.Name = zDiscoveredSignature.Game.Description + ".zip";
}
zDiscoveredSignature.Rom.Crc = discoveredSignature.Rom.Crc;
zDiscoveredSignature.Rom.Md5 = discoveredSignature.Rom.Md5;
zDiscoveredSignature.Rom.Sha1 = discoveredSignature.Rom.Sha1;
zDiscoveredSignature.Rom.Size = discoveredSignature.Rom.Size;
discoveredSignature = zDiscoveredSignature;
break;
@@ -238,7 +246,7 @@ namespace gaseous_server.Classes
ri.Md5 = hash.md5hash;
ri.Sha1 = hash.sha1hash;
ri.Size = fi.Length;
ri.SignatureSource = Models.Signatures_Games.RomItem.SignatureSourceType.None;
ri.SignatureSource = gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None;
}
}
@@ -275,6 +283,17 @@ namespace gaseous_server.Classes
else if (games.Length > 0)
{
Logging.Log(Logging.LogType.Information, "Import Game", " " + games.Length + " search results found");
// quite likely we've found sequels and alternate types
foreach (Game game in games) {
if (game.Name == SearchCandidate) {
// found game title matches the search candidate
determinedGame = Metadata.Games.GetGame((long)games[0].Id, false, false, false);
Logging.Log(Logging.LogType.Information, "Import Game", "Found exact match!");
GameFound = true;
break;
}
}
}
else
{
@@ -340,11 +359,17 @@ namespace gaseous_server.Classes
GameName = Regex.Replace(GameName, @"v(\d+\.)?(\d+\.)?(\*|\d+)$", "").Trim();
GameName = Regex.Replace(GameName, @"Rev (\d+\.)?(\d+\.)?(\*|\d+)$", "").Trim();
// assumption: no games have () in their titles so we'll remove them
int idx = GameName.IndexOf('(');
if (idx >= 0) {
GameName = GameName.Substring(0, idx);
}
List<string> SearchCandidates = new List<string>();
SearchCandidates.Add(GameName);
SearchCandidates.Add(GameName.Trim());
if (GameName.Contains(" - "))
{
SearchCandidates.Add(GameName.Replace(" - ", ": "));
SearchCandidates.Add(GameName.Replace(" - ", ": ").Trim());
SearchCandidates.Add(GameName.Substring(0, GameName.IndexOf(" - ")).Trim());
}
if (GameName.Contains(": "))
@@ -367,10 +392,10 @@ namespace gaseous_server.Classes
if (UpdateId == 0)
{
sql = "INSERT INTO Games_Roms (PlatformId, GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel, Path, MetadataSource) VALUES (@platformid, @gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @path, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
sql = "INSERT INTO Games_Roms (PlatformId, GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Attributes, RomType, RomTypeMedia, MediaLabel, Path, MetadataSource, MetadataGameName, MetadataVersion) VALUES (@platformid, @gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @Attributes, @romtype, @romtypemedia, @medialabel, @path, @metadatasource, @metadatagamename, @metadataversion); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
} else
{
sql = "UPDATE Games_Roms SET PlatformId=platformid, GameId=@gameid, Name=@name, Size=@size, DevelopmentStatus=@developmentstatus, Flags=@flags, RomType=@romtype, RomTypeMedia=@romtypemedia, MediaLabel=@medialabel, MetadataSource=@metadatasource WHERE Id=@id;";
sql = "UPDATE Games_Roms SET PlatformId=platformid, GameId=@gameid, Name=@name, Size=@size, DevelopmentStatus=@developmentstatus, Attributes=@Attributes, RomType=@romtype, RomTypeMedia=@romtypemedia, MediaLabel=@medialabel, MetadataSource=@metadatasource, MetadataGameName=@metadatagamename, MetadataVersion=@metadataversion WHERE Id=@id;";
dbDict.Add("id", UpdateId);
}
dbDict.Add("platformid", Common.ReturnValueIfNull(determinedPlatform.Id, 0));
@@ -382,21 +407,23 @@ namespace gaseous_server.Classes
dbDict.Add("crc", Common.ReturnValueIfNull(discoveredSignature.Rom.Crc, ""));
dbDict.Add("developmentstatus", Common.ReturnValueIfNull(discoveredSignature.Rom.DevelopmentStatus, ""));
dbDict.Add("metadatasource", discoveredSignature.Rom.SignatureSource);
dbDict.Add("metadatagamename", discoveredSignature.Game.Name);
dbDict.Add("metadataversion", 2);
if (discoveredSignature.Rom.flags != null)
if (discoveredSignature.Rom.Attributes != null)
{
if (discoveredSignature.Rom.flags.Count > 0)
if (discoveredSignature.Rom.Attributes.Count > 0)
{
dbDict.Add("flags", Newtonsoft.Json.JsonConvert.SerializeObject(discoveredSignature.Rom.flags));
dbDict.Add("attributes", Newtonsoft.Json.JsonConvert.SerializeObject(discoveredSignature.Rom.Attributes));
}
else
{
dbDict.Add("flags", "[ ]");
dbDict.Add("attributes", "[ ]");
}
}
else
{
dbDict.Add("flags", "[ ]");
dbDict.Add("attributes", "[ ]");
}
dbDict.Add("romtype", (int)discoveredSignature.Rom.RomType);
dbDict.Add("romtypemedia", Common.ReturnValueIfNull(discoveredSignature.Rom.RomTypeMedia, ""));
@@ -571,6 +598,8 @@ namespace gaseous_server.Classes
Logging.Log(Logging.LogType.Information, "Library Scan", "Looking for orphaned library files to add");
string[] LibraryFiles = Directory.GetFiles(Config.LibraryConfiguration.LibraryDataDirectory, "*.*", SearchOption.AllDirectories);
foreach (string LibraryFile in LibraryFiles)
{
if (!Common.SkippableFiles.Contains<string>(Path.GetFileName(LibraryFile), StringComparer.OrdinalIgnoreCase))
{
Common.hashObject LibraryFileHash = new Common.hashObject(LibraryFile);
@@ -611,6 +640,7 @@ namespace gaseous_server.Classes
StoreROM(hash, determinedGame, determinedPlatform, sig, LibraryFile);
}
}
}
sql = "SELECT * FROM Games_Roms ORDER BY `name`";
dtRoms = db.ExecuteCMD(sql);
@@ -623,24 +653,27 @@ namespace gaseous_server.Classes
{
long romId = (long)dtRoms.Rows[i]["Id"];
string romPath = (string)dtRoms.Rows[i]["Path"];
Classes.Roms.GameRomItem.SourceType romMetadataSource = (Classes.Roms.GameRomItem.SourceType)(int)dtRoms.Rows[i]["MetadataSource"];
gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType romMetadataSource = (gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType)(int)dtRoms.Rows[i]["MetadataSource"];
Logging.Log(Logging.LogType.Information, "Library Scan", " Processing ROM at path " + romPath);
if (File.Exists(romPath))
{
// file exists, so lets check to make sure the signature was matched, and update if a signature can be found
if (romMetadataSource == Roms.GameRomItem.SourceType.None)
if (
romMetadataSource == gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None ||
(int)dtRoms.Rows[i]["MetadataVersion"] == 1
)
{
Common.hashObject hash = new Common.hashObject
{
md5hash = "",
sha1hash = ""
md5hash = (string)dtRoms.Rows[i]["MD5"],
sha1hash = (string)dtRoms.Rows[i]["SHA1"]
};
FileInfo fi = new FileInfo(romPath);
Models.Signatures_Games sig = GetFileSignature(hash, fi, romPath);
if (sig.Rom.SignatureSource != Models.Signatures_Games.RomItem.SignatureSourceType.None)
if (sig.Rom.SignatureSource != gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType.None)
{
Logging.Log(Logging.LogType.Information, "Library Scan", " Update signature found for " + romPath);

15
gaseous-server/Classes/Roms.cs
View File

@@ -107,12 +107,13 @@ namespace gaseous_server.Classes
MD5 = (string)romDR["md5"],
SHA1 = (string)romDR["sha1"],
DevelopmentStatus = (string)romDR["developmentstatus"],
Flags = Newtonsoft.Json.JsonConvert.DeserializeObject<string[]>((string)romDR["flags"]),
Attributes = Newtonsoft.Json.JsonConvert.DeserializeObject<List<KeyValuePair<string, object>>>((string)Common.ReturnValueIfNull(romDR["attributes"], "[ ]")),
RomType = (int)romDR["romtype"],
RomTypeMedia = (string)romDR["romtypemedia"],
MediaLabel = (string)romDR["medialabel"],
Path = (string)romDR["path"],
Source = (GameRomItem.SourceType)(Int32)romDR["metadatasource"]
Source = (gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType)(Int32)romDR["metadatasource"],
SignatureSourceGameTitle = (string)Common.ReturnValueIfNull(romDR["MetadataGameName"], "")
};
// check for a web emulator and update the romItem
@@ -145,17 +146,13 @@ namespace gaseous_server.Classes
public string? SHA1 { get; set; }
public string? DevelopmentStatus { get; set; }
public string[]? Flags { get; set; }
public List<KeyValuePair<string, object>>? Attributes { get; set;}
public int RomType { get; set; }
public string? RomTypeMedia { get; set; }
public string? MediaLabel { get; set; }
public string? Path { get; set; }
public SourceType Source { get; set; }
public enum SourceType
{
None = 0,
TOSEC = 1
}
public gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType Source { get; set; }
public string? SignatureSourceGameTitle { get; set;}
}
}
}

234
gaseous-server/Classes/SignatureIngestors/TOSEC.cs
View File

@@ -1,234 +0,0 @@
using System;
using System.IO;
using MySql.Data.MySqlClient;
using gaseous_signature_parser.models.RomSignatureObject;
using gaseous_signature_parser.classes.parsers;
using gaseous_tools;
using MySqlX.XDevAPI;
namespace gaseous_server.SignatureIngestors.TOSEC
{
public class TOSECIngestor
{
public void Import(string SearchPath)
{
// connect to database
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
// process provided files
Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing from " + SearchPath);
if (Directory.Exists(Config.LibraryConfiguration.LibrarySignatureImportDirectory_TOSEC))
{
string[] tosecPathContents = Directory.GetFiles(SearchPath);
Array.Sort(tosecPathContents);
string sql = "";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
System.Data.DataTable sigDB;
for (UInt16 i = 0; i < tosecPathContents.Length; ++i)
{
string tosecXMLFile = tosecPathContents[i];
// check tosec file md5
Common.hashObject hashObject = new Common.hashObject(tosecXMLFile);
sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
dbDict = new Dictionary<string, object>();
dbDict.Add("sourcemd5", hashObject.md5hash);
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
try
{
Logging.Log(Logging.LogType.Information, "Signature Ingestor - TOSEC", "Importing file: " + tosecXMLFile);
// start parsing file
TosecParser tosecParser = new TosecParser();
RomSignatureObject tosecObject = tosecParser.Parse(tosecXMLFile);
// store in database
// store source object
bool processGames = false;
if (tosecObject.SourceMd5 != null)
{
sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
dbDict = new Dictionary<string, object>();
dbDict.Add("name", Common.ReturnValueIfNull(tosecObject.Name, ""));
dbDict.Add("description", Common.ReturnValueIfNull(tosecObject.Description, ""));
dbDict.Add("category", Common.ReturnValueIfNull(tosecObject.Category, ""));
dbDict.Add("version", Common.ReturnValueIfNull(tosecObject.Version, ""));
dbDict.Add("author", Common.ReturnValueIfNull(tosecObject.Author, ""));
dbDict.Add("email", Common.ReturnValueIfNull(tosecObject.Email, ""));
dbDict.Add("homepage", Common.ReturnValueIfNull(tosecObject.Homepage, ""));
dbDict.Add("uri", Common.ReturnValueIfNull(tosecObject.Url, ""));
dbDict.Add("sourcetype", Common.ReturnValueIfNull(tosecObject.SourceType, ""));
dbDict.Add("sourcemd5", tosecObject.SourceMd5);
dbDict.Add("sourcesha1", tosecObject.SourceSHA1);
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
sql = "INSERT INTO Signatures_Sources (Name, Description, Category, Version, Author, Email, Homepage, Url, SourceType, SourceMD5, SourceSHA1) VALUES (@name, @description, @category, @version, @author, @email, @homepage, @uri, @sourcetype, @sourcemd5, @sourcesha1)";
db.ExecuteCMD(sql, dbDict);
processGames = true;
}
if (processGames == true)
{
for (int x = 0; x < tosecObject.Games.Count; ++x)
{
RomSignatureObject.Game gameObject = tosecObject.Games[x];
// set up game dictionary
dbDict = new Dictionary<string, object>();
dbDict.Add("name", Common.ReturnValueIfNull(gameObject.Name, ""));
dbDict.Add("description", Common.ReturnValueIfNull(gameObject.Description, ""));
dbDict.Add("year", Common.ReturnValueIfNull(gameObject.Year, ""));
dbDict.Add("publisher", Common.ReturnValueIfNull(gameObject.Publisher, ""));
dbDict.Add("demo", (int)gameObject.Demo);
dbDict.Add("system", Common.ReturnValueIfNull(gameObject.System, ""));
dbDict.Add("platform", Common.ReturnValueIfNull(gameObject.System, ""));
dbDict.Add("systemvariant", Common.ReturnValueIfNull(gameObject.SystemVariant, ""));
dbDict.Add("video", Common.ReturnValueIfNull(gameObject.Video, ""));
dbDict.Add("country", Common.ReturnValueIfNull(gameObject.Country, ""));
dbDict.Add("language", Common.ReturnValueIfNull(gameObject.Language, ""));
dbDict.Add("copyright", Common.ReturnValueIfNull(gameObject.Copyright, ""));
// store platform
int gameSystem = 0;
if (gameObject.System != null)
{
sql = "SELECT Id FROM Signatures_Platforms WHERE Platform=@platform";
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
sql = "INSERT INTO Signatures_Platforms (Platform) VALUES (@platform); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
sigDB = db.ExecuteCMD(sql, dbDict);
gameSystem = Convert.ToInt32(sigDB.Rows[0][0]);
}
else
{
gameSystem = (int)sigDB.Rows[0][0];
}
}
dbDict.Add("systemid", gameSystem);
// store publisher
int gamePublisher = 0;
if (gameObject.Publisher != null)
{
sql = "SELECT * FROM Signatures_Publishers WHERE Publisher=@publisher";
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
sql = "INSERT INTO Signatures_Publishers (Publisher) VALUES (@publisher); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
sigDB = db.ExecuteCMD(sql, dbDict);
gamePublisher = Convert.ToInt32(sigDB.Rows[0][0]);
}
else
{
gamePublisher = (int)sigDB.Rows[0][0];
}
}
dbDict.Add("publisherid", gamePublisher);
// store game
int gameId = 0;
sql = "SELECT * FROM Signatures_Games WHERE Name=@name AND Year=@year AND Publisherid=@publisher AND Systemid=@systemid AND Country=@country AND Language=@language";
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
sql = "INSERT INTO Signatures_Games " +
"(Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " +
"(@name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
sigDB = db.ExecuteCMD(sql, dbDict);
gameId = Convert.ToInt32(sigDB.Rows[0][0]);
}
else
{
gameId = (int)sigDB.Rows[0][0];
}
// store rom
foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms)
{
if (romObject.Md5 != null)
{
int romId = 0;
sql = "SELECT * FROM Signatures_Roms WHERE GameId=@gameid AND MD5=@md5";
dbDict = new Dictionary<string, object>();
dbDict.Add("gameid", gameId);
dbDict.Add("name", Common.ReturnValueIfNull(romObject.Name, ""));
dbDict.Add("size", Common.ReturnValueIfNull(romObject.Size, ""));
dbDict.Add("crc", Common.ReturnValueIfNull(romObject.Crc, ""));
dbDict.Add("md5", romObject.Md5);
dbDict.Add("sha1", Common.ReturnValueIfNull(romObject.Sha1, ""));
dbDict.Add("developmentstatus", Common.ReturnValueIfNull(romObject.DevelopmentStatus, ""));
if (romObject.flags != null)
{
if (romObject.flags.Count > 0)
{
dbDict.Add("flags", Newtonsoft.Json.JsonConvert.SerializeObject(romObject.flags));
}
else
{
dbDict.Add("flags", "[ ]");
}
}
else
{
dbDict.Add("flags", "[ ]");
}
dbDict.Add("romtype", (int)romObject.RomType);
dbDict.Add("romtypemedia", Common.ReturnValueIfNull(romObject.RomTypeMedia, ""));
dbDict.Add("medialabel", Common.ReturnValueIfNull(romObject.MediaLabel, ""));
dbDict.Add("metadatasource", Classes.Roms.GameRomItem.SourceType.TOSEC);
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Flags, RomType, RomTypeMedia, MediaLabel, MetadataSource) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
sigDB = db.ExecuteCMD(sql, dbDict);
romId = Convert.ToInt32(sigDB.Rows[0][0]);
}
else
{
romId = (int)sigDB.Rows[0][0];
}
}
}
}
}
}
}
catch (Exception ex)
{
Logging.Log(Logging.LogType.Warning, "Signature Ingestor - TOSEC", "Invalid import file: " + tosecXMLFile, ex);
}
}
else
{
Logging.Log(Logging.LogType.Debug, "Signature Ingestor - TOSEC", "Rejecting already imported file: " + tosecXMLFile);
}
}
}
}
}
}

344
gaseous-server/Classes/SignatureIngestors/XML.cs Normal file
View File

@@ -0,0 +1,344 @@
using System;
using System.IO;
using MySql.Data.MySqlClient;
using gaseous_signature_parser.models.RomSignatureObject;
using gaseous_tools;
using MySqlX.XDevAPI;
using System.Data;
namespace gaseous_server.SignatureIngestors.XML
{
public class XMLIngestor
{
public void Import(string SearchPath, gaseous_signature_parser.parser.SignatureParser XMLType)
{
// connect to database
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
// process provided files
Logging.Log(Logging.LogType.Information, "Signature Ingestor - XML", "Importing from " + SearchPath);
if (!Directory.Exists(SearchPath))
{
Directory.CreateDirectory(SearchPath);
}
string[] PathContents = Directory.GetFiles(SearchPath);
Array.Sort(PathContents);
string sql = "";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
System.Data.DataTable sigDB;
for (UInt16 i = 0; i < PathContents.Length; ++i)
{
string XMLFile = PathContents[i];
// check xml file md5
Common.hashObject hashObject = new Common.hashObject(XMLFile);
sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
dbDict = new Dictionary<string, object>();
dbDict.Add("sourcemd5", hashObject.md5hash);
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
try
{
Logging.Log(Logging.LogType.Information, "Signature Ingestor - XML", "Importing file: " + XMLFile);
// start parsing file
gaseous_signature_parser.parser Parser = new gaseous_signature_parser.parser();
RomSignatureObject Object = Parser.ParseSignatureDAT(XMLFile, XMLType);
// store in database
string[] flipNameAndDescription = {
"MAMEArcade",
"MAMEMess"
};
// store source object
bool processGames = false;
if (Object.SourceMd5 != null)
{
sql = "SELECT * FROM Signatures_Sources WHERE SourceMD5=@sourcemd5";
dbDict = new Dictionary<string, object>();
dbDict.Add("name", Common.ReturnValueIfNull(Object.Name, ""));
dbDict.Add("description", Common.ReturnValueIfNull(Object.Description, ""));
dbDict.Add("category", Common.ReturnValueIfNull(Object.Category, ""));
dbDict.Add("version", Common.ReturnValueIfNull(Object.Version, ""));
dbDict.Add("author", Common.ReturnValueIfNull(Object.Author, ""));
dbDict.Add("email", Common.ReturnValueIfNull(Object.Email, ""));
dbDict.Add("homepage", Common.ReturnValueIfNull(Object.Homepage, ""));
dbDict.Add("uri", Common.ReturnValueIfNull(Object.Url, ""));
dbDict.Add("sourcetype", Common.ReturnValueIfNull(Object.SourceType, ""));
dbDict.Add("sourcemd5", Object.SourceMd5);
dbDict.Add("sourcesha1", Object.SourceSHA1);
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
sql = "INSERT INTO Signatures_Sources (Name, Description, Category, Version, Author, Email, Homepage, Url, SourceType, SourceMD5, SourceSHA1) VALUES (@name, @description, @category, @version, @author, @email, @homepage, @uri, @sourcetype, @sourcemd5, @sourcesha1)";
db.ExecuteCMD(sql, dbDict);
processGames = true;
}
if (processGames == true)
{
for (int x = 0; x < Object.Games.Count; ++x)
{
RomSignatureObject.Game gameObject = Object.Games[x];
// set up game dictionary
dbDict = new Dictionary<string, object>();
if (flipNameAndDescription.Contains(Object.SourceType))
{
dbDict.Add("name", Common.ReturnValueIfNull(gameObject.Description, ""));
dbDict.Add("description", Common.ReturnValueIfNull(gameObject.Name, ""));
}
else
{
dbDict.Add("name", Common.ReturnValueIfNull(gameObject.Name, ""));
dbDict.Add("description", Common.ReturnValueIfNull(gameObject.Description, ""));
}
dbDict.Add("year", Common.ReturnValueIfNull(gameObject.Year, ""));
dbDict.Add("publisher", Common.ReturnValueIfNull(gameObject.Publisher, ""));
dbDict.Add("demo", (int)gameObject.Demo);
dbDict.Add("system", Common.ReturnValueIfNull(gameObject.System, ""));
dbDict.Add("platform", Common.ReturnValueIfNull(gameObject.System, ""));
dbDict.Add("systemvariant", Common.ReturnValueIfNull(gameObject.SystemVariant, ""));
dbDict.Add("video", Common.ReturnValueIfNull(gameObject.Video, ""));
dbDict.Add("country", Common.ReturnValueIfNull(gameObject.Country, ""));
dbDict.Add("language", Common.ReturnValueIfNull(gameObject.Language, ""));
dbDict.Add("copyright", Common.ReturnValueIfNull(gameObject.Copyright, ""));
// store platform
int gameSystem = 0;
if (gameObject.System != null)
{
sql = "SELECT Id FROM Signatures_Platforms WHERE Platform=@platform";
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
sql = "INSERT INTO Signatures_Platforms (Platform) VALUES (@platform); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
sigDB = db.ExecuteCMD(sql, dbDict);
gameSystem = Convert.ToInt32(sigDB.Rows[0][0]);
}
else
{
gameSystem = (int)sigDB.Rows[0][0];
}
}
dbDict.Add("systemid", gameSystem);
// store publisher
int gamePublisher = 0;
if (gameObject.Publisher != null)
{
sql = "SELECT * FROM Signatures_Publishers WHERE Publisher=@publisher";
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
sql = "INSERT INTO Signatures_Publishers (Publisher) VALUES (@publisher); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
sigDB = db.ExecuteCMD(sql, dbDict);
gamePublisher = Convert.ToInt32(sigDB.Rows[0][0]);
}
else
{
gamePublisher = (int)sigDB.Rows[0][0];
}
}
dbDict.Add("publisherid", gamePublisher);
// store game
int gameId = 0;
sql = "SELECT * FROM Signatures_Games WHERE Name=@name AND Year=@year AND Publisherid=@publisher AND Systemid=@systemid AND Country=@country AND Language=@language";
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
sql = "INSERT INTO Signatures_Games " +
"(Name, Description, Year, PublisherId, Demo, SystemId, SystemVariant, Video, Country, Language, Copyright) VALUES " +
"(@name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
sigDB = db.ExecuteCMD(sql, dbDict);
gameId = Convert.ToInt32(sigDB.Rows[0][0]);
}
else
{
gameId = (int)sigDB.Rows[0][0];
}
// store rom
foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms)
{
if (romObject.Md5 != null || romObject.Sha1 != null)
{
int romId = 0;
sql = "SELECT * FROM Signatures_Roms WHERE GameId=@gameid AND MD5=@md5";
dbDict = new Dictionary<string, object>();
dbDict.Add("gameid", gameId);
dbDict.Add("name", Common.ReturnValueIfNull(romObject.Name, ""));
dbDict.Add("size", Common.ReturnValueIfNull(romObject.Size, ""));
dbDict.Add("crc", Common.ReturnValueIfNull(romObject.Crc, ""));
dbDict.Add("md5", Common.ReturnValueIfNull(romObject.Md5, ""));
dbDict.Add("sha1", Common.ReturnValueIfNull(romObject.Sha1, ""));
dbDict.Add("developmentstatus", Common.ReturnValueIfNull(romObject.DevelopmentStatus, ""));
if (romObject.Attributes != null)
{
if (romObject.Attributes.Count > 0)
{
dbDict.Add("attributes", Newtonsoft.Json.JsonConvert.SerializeObject(romObject.Attributes));
}
else
{
dbDict.Add("attributes", "[ ]");
}
}
else
{
dbDict.Add("attributes", "[ ]");
}
dbDict.Add("romtype", (int)romObject.RomType);
dbDict.Add("romtypemedia", Common.ReturnValueIfNull(romObject.RomTypeMedia, ""));
dbDict.Add("medialabel", Common.ReturnValueIfNull(romObject.MediaLabel, ""));
dbDict.Add("metadatasource", romObject.SignatureSource);
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
sql = "INSERT INTO Signatures_Roms (GameId, Name, Size, CRC, MD5, SHA1, DevelopmentStatus, Attributes, RomType, RomTypeMedia, MediaLabel, MetadataSource) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @attributes, @romtype, @romtypemedia, @medialabel, @metadatasource); SELECT CAST(LAST_INSERT_ID() AS SIGNED);";
sigDB = db.ExecuteCMD(sql, dbDict);
romId = Convert.ToInt32(sigDB.Rows[0][0]);
}
else
{
romId = (int)sigDB.Rows[0][0];
}
}
}
}
}
}
}
catch (Exception ex)
{
Logging.Log(Logging.LogType.Warning, "Signature Ingestor - XML", "Invalid import file: " + XMLFile, ex);
}
}
else
{
Logging.Log(Logging.LogType.Debug, "Signature Ingestor - XML", "Rejecting already imported file: " + XMLFile);
}
}
}
public void MigrateMetadatVersion() {
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
// update signature roms to v2
sql = "SELECT Id, Flags, Attributes, IngestorVersion FROM Signatures_Roms WHERE IngestorVersion = 1";
DataTable data = db.ExecuteCMD(sql);
if (data.Rows.Count > 0)
{
Logging.Log(Logging.LogType.Information, "Signature Ingestor - Database Update", "Updating " + data.Rows.Count + " database entries");
int Counter = 0;
int LastCounterCheck = 0;
foreach (DataRow row in data.Rows)
{
List<string> Flags = Newtonsoft.Json.JsonConvert.DeserializeObject<List<string>>((string)Common.ReturnValueIfNull(row["flags"], "[]"));
List<KeyValuePair<string, object>> Attributes = new List<KeyValuePair<string, object>>();
foreach (string Flag in Flags)
{
if (Flag.StartsWith("a"))
{
Attributes.Add(
new KeyValuePair<string, object>(
"a",
Flag
)
);
}
else
{
string[] FlagCompare = Flag.Split(' ');
switch (FlagCompare[0].Trim().ToLower())
{
case "cr":
// cracked
case "f":
// fixed
case "h":
// hacked
case "m":
// modified
case "p":
// pirated
case "t":
// trained
case "tr":
// translated
case "o":
// overdump
case "u":
// underdump
case "v":
// virus
case "b":
// bad dump
case "a":
// alternate
case "!":
// known verified dump
// -------------------
string shavedToken = Flag.Substring(FlagCompare[0].Trim().Length).Trim();
Attributes.Add(new KeyValuePair<string, object>(
FlagCompare[0].Trim().ToLower(),
shavedToken
));
break;
}
}
}
string AttributesJson;
if (Attributes.Count > 0)
{
AttributesJson = Newtonsoft.Json.JsonConvert.SerializeObject(Attributes);
}
else
{
AttributesJson = "[]";
}
string updateSQL = "UPDATE Signatures_Roms SET Attributes=@attributes, IngestorVersion=2 WHERE Id=@id";
dbDict = new Dictionary<string, object>();
dbDict.Add("attributes", AttributesJson);
dbDict.Add("id", (int)row["Id"]);
db.ExecuteCMD(updateSQL, dbDict);
if ((Counter - LastCounterCheck) > 10)
{
LastCounterCheck = Counter;
Logging.Log(Logging.LogType.Information, "Signature Ingestor - Database Update", "Updating " + Counter + " / " + data.Rows.Count + " database entries");
}
Counter += 1;
}
}
}
}
}

6
gaseous-server/Controllers/SignaturesController.cs
View File

@@ -55,7 +55,7 @@ namespace gaseous_server.Controllers
private List<Models.Signatures_Games> _GetSignature(string sqlWhere, string searchString)
{
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "SELECT view_Signatures_Games.*, Signatures_Roms.Id AS romid, Signatures_Roms.Name AS romname, Signatures_Roms.Size, Signatures_Roms.CRC, Signatures_Roms.MD5, Signatures_Roms.SHA1, Signatures_Roms.DevelopmentStatus, Signatures_Roms.Flags, Signatures_Roms.RomType, Signatures_Roms.RomTypeMedia, Signatures_Roms.MediaLabel, Signatures_Roms.MetadataSource FROM Signatures_Roms INNER JOIN view_Signatures_Games ON Signatures_Roms.GameId = view_Signatures_Games.Id WHERE " + sqlWhere;
string sql = "SELECT view_Signatures_Games.*, Signatures_Roms.Id AS romid, Signatures_Roms.Name AS romname, Signatures_Roms.Size, Signatures_Roms.CRC, Signatures_Roms.MD5, Signatures_Roms.SHA1, Signatures_Roms.DevelopmentStatus, Signatures_Roms.Attributes, Signatures_Roms.RomType, Signatures_Roms.RomTypeMedia, Signatures_Roms.MediaLabel, Signatures_Roms.MetadataSource FROM Signatures_Roms INNER JOIN view_Signatures_Games ON Signatures_Roms.GameId = view_Signatures_Games.Id WHERE " + sqlWhere;
Dictionary<string, object> dbDict = new Dictionary<string, object>();
dbDict.Add("searchString", searchString);
@@ -91,11 +91,11 @@ namespace gaseous_server.Controllers
Md5 = (string)sigDbRow["MD5"],
Sha1 = (string)sigDbRow["SHA1"],
DevelopmentStatus = (string)sigDbRow["DevelopmentStatus"],
flags = Newtonsoft.Json.JsonConvert.DeserializeObject<List<string>>((string)sigDbRow["Flags"]),
Attributes = Newtonsoft.Json.JsonConvert.DeserializeObject<List<KeyValuePair<string, object>>>((string)Common.ReturnValueIfNull(sigDbRow["Attributes"], "[]")),
RomType = (Models.Signatures_Games.RomItem.RomTypes)(int)sigDbRow["RomType"],
RomTypeMedia = (string)sigDbRow["RomTypeMedia"],
MediaLabel = (string)sigDbRow["MediaLabel"],
SignatureSource = (Models.Signatures_Games.RomItem.SignatureSourceType)(Int32)sigDbRow["MetadataSource"]
SignatureSource = (gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType)(Int32)sigDbRow["MetadataSource"]
}
};
GamesList.Add(gameItem);

5
gaseous-server/Models/PlatformMapping.cs
View File

@@ -56,7 +56,10 @@ namespace gaseous_server.Models
{
foreach (Models.PlatformMapping.PlatformMapItem PlatformMapping in Models.PlatformMapping.PlatformMap)
{
if (PlatformMapping.AlternateNames.Contains(Signature.Game.System, StringComparer.OrdinalIgnoreCase))
if (
PlatformMapping.IGDBName == Signature.Game.System ||
PlatformMapping.AlternateNames.Contains(Signature.Game.System, StringComparer.OrdinalIgnoreCase)
)
{
if (SetSystemName == true)
{

11
gaseous-server/Models/Signatures_Games.cs
View File

@@ -126,19 +126,13 @@ namespace gaseous_server.Models
public string? DevelopmentStatus { get; set; }
public List<string> flags { get; set; } = new List<string>();
public List<KeyValuePair<string, object>> Attributes { get; set; } = new List<KeyValuePair<string, object>>();
public RomTypes RomType { get; set; }
public string? RomTypeMedia { get; set; }
public string? MediaLabel { get; set; }
public SignatureSourceType SignatureSource { get; set; }
public enum SignatureSourceType
{
None = 0,
TOSEC = 1
}
public gaseous_signature_parser.models.RomSignatureObject.RomSignatureObject.Game.Rom.SignatureSourceType SignatureSource { get; set; }
public enum RomTypes
{
@@ -197,6 +191,7 @@ namespace gaseous_server.Models
case "crc":
case "developmentstatus":
case "flags":
case "attributes":
case "romtypemedia":
case "medialabel":
if (prop.PropertyType == typeof(string) || prop.PropertyType == typeof(Int64) || prop.PropertyType == typeof(List<string>))

13
gaseous-server/ProcessQueue.cs
View File

@@ -76,8 +76,17 @@ namespace gaseous_server
{
case QueueItemType.SignatureIngestor:
Logging.Log(Logging.LogType.Information, "Timered Event", "Starting Signature Ingestor");
SignatureIngestors.TOSEC.TOSECIngestor tIngest = new SignatureIngestors.TOSEC.TOSECIngestor();
tIngest.Import(Config.LibraryConfiguration.LibrarySignatureImportDirectory_TOSEC);
SignatureIngestors.XML.XMLIngestor tIngest = new SignatureIngestors.XML.XMLIngestor();
Logging.Log(Logging.LogType.Information, "Signature Import", "Processing TOSEC files");
tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "TOSEC"), gaseous_signature_parser.parser.SignatureParser.TOSEC);
Logging.Log(Logging.LogType.Information, "Signature Import", "Processing MAME Arcade files");
tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME Arcade"), gaseous_signature_parser.parser.SignatureParser.MAMEArcade);
Logging.Log(Logging.LogType.Information, "Signature Import", "Processing MAME MESS files");
tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME MESS"), gaseous_signature_parser.parser.SignatureParser.MAMEMess);
break;
case QueueItemType.TitleIngestor:

5
gaseous-server/Program.cs
View File

@@ -1,6 +1,7 @@
using System.Reflection;
using System.Text.Json.Serialization;
using gaseous_server;
using gaseous_server.SignatureIngestors.XML;
using gaseous_tools;
using Microsoft.AspNetCore.Http.Features;
using Microsoft.AspNetCore.Mvc;
@@ -146,6 +147,10 @@ gaseous_server.Classes.Metadata.Platforms.GetPlatform(0);
// organise library
//gaseous_server.Classes.ImportGame.OrganiseLibrary();
// Migrate signature data if needed
XMLIngestor ingestor = new XMLIngestor();
ingestor.MigrateMetadatVersion();
// add background tasks
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.SignatureIngestor, 60));
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(

4
gaseous-server/Support/PlatformMap.json
View File

@@ -205,7 +205,9 @@
{
"IGDBId": 52,
"IGDBName": "Arcade",
"AlternateNames": [],
"AlternateNames": [
"Arcade"
],
"KnownFileExtensions": [],
"WebEmulator": {
"Type": "EmulatorJS",

2
gaseous-server/gaseous-server.csproj
View File

@@ -18,7 +18,7 @@
<DocumentationFile>bin\Release\net7.0\gaseous-server.xml</DocumentationFile>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="gaseous-signature-parser" Version="1.0.1" />
<PackageReference Include="gaseous-signature-parser" Version="2.0.0" />
<PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.10" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
<PackageReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Design" Version="7.0.9" />

69
gaseous-server/wwwroot/pages/dialogs/rominfo.html
View File

@@ -1,5 +1,6 @@
<div id="properties_toc">
<div id="properties_toc_general" name="properties_toc_item" onclick="SelectTab('general');">General</div>
<div id="properties_toc_attributes" name="properties_toc_item" onclick="SelectTab('attributes');">Attributes</div>
<div id="properties_toc_match" name="properties_toc_item" onclick="SelectTab('match');">Title Match</div>
<!--<div id="properties_toc_manage" name="properties_toc_item" onclick="SelectTab('manage');">Manage</div>-->
</div>
@@ -39,8 +40,8 @@
<td id="rominfo_signaturematch"></td>
</tr>
<tr>
<th>Signature Source Flags</th>
<td id="rominfo_flags"></td>
<th>Signature Game Title</th>
<td id="rominfo_signaturetitle"></td>
</tr>
<tr>
<td colspan="2" style="text-align: right;"><button id="romDelete" class="redbutton">Delete</button></td>
@@ -48,6 +49,10 @@
</table>
</div>
<div id="properties_bodypanel_attributes" name="properties_tab" style="display: none;">
</div>
<div id="properties_bodypanel_match" name="properties_tab" style="display: none;">
<table style="width: 100%;">
<tr>
@@ -113,7 +118,6 @@
romData = result;
document.getElementById('modal-heading').innerHTML = result.name;
document.getElementById('rominfo_platform').innerHTML = result.platform.name;
//document.getElementById('rominfo_filename').innerHTML = result.name;
document.getElementById('rominfo_size').innerHTML = formatBytes(result.size, 2);
document.getElementById('rominfo_type').innerHTML = getRomType(result.romType);
document.getElementById('rominfo_mediatype').innerHTML = result.romTypeMedia;
@@ -121,10 +125,16 @@
document.getElementById('rominfo_md5').innerHTML = result.mD5;
document.getElementById('rominfo_sha1').innerHTML = result.shA1;
document.getElementById('rominfo_signaturematch').innerHTML = result.source;
document.getElementById('rominfo_flags').innerHTML = result.flags;
document.getElementById('rominfo_signaturetitle').innerHTML = result.signatureSourceGameTitle;
document.getElementById('properties_fixplatform').innerHTML = "<option value='" + result.platform.id + "' selected='selected'>" + result.platform.name + "</option>";
document.getElementById('properties_fixgame').innerHTML = "<option value='" + gameData.id + "' selected='selected'>" + gameData.name + "</option>";
if (result.attributes.length > 0) {
document.getElementById('properties_bodypanel_attributes').appendChild(BuildAttributesTable(result.attributes, result.source));
} else {
document.getElementById('properties_toc_attributes').style.display = 'none';
}
});
function SelectTab(TabName) {
@@ -222,6 +232,57 @@
});
}
function BuildAttributesTable(attributes, sourceName) {
var aTable = document.createElement('table');
aTable.style.width = '100%';
for (var i = 0; i < attributes.length; i++) {
var aRow = document.createElement('tr');
var aTitleCell = document.createElement('th');
aTitleCell.width = "25%";
if (sourceName == "TOSEC") {
aTitleCell.innerHTML = ConvertTOSECAttributeName(attributes[i].key);
} else {
aTitleCell.innerHTML = attributes[i].key;
}
aRow.appendChild(aTitleCell);
var aValueCell = document.createElement('td');
aValueCell.width = "75%";
aValueCell.innerHTML = attributes[i].value;
aRow.appendChild(aValueCell);
aTable.appendChild(aRow);
}
return aTable;
}
function ConvertTOSECAttributeName(attributeName) {
var tosecAttributeNames = {
"cr": "Cracked",
"f" : "Fixed",
"h" : "Hacked",
"m" : "Modified",
"p" : "Pirated",
"t" : "Trained",
"tr": "Translated",
"o" : "Over Dump",
"u" : "Under Dump",
"v" : "Virus",
"b" : "Bad Dump",
"a" : "Alternate",
"!" : "Known Verified Dump"
};
if (attributeName in tosecAttributeNames) {
return tosecAttributeNames[attributeName];
} else {
return attributeName;
}
}
SelectTab('general');
document.getElementById('romDelete').setAttribute("onclick", "showSubDialog('romdelete', " + modalVariables + ");");

5
gaseous-tools/Common.cs
View File

@@ -99,6 +99,11 @@ namespace gaseous_tools
}
return size;
}
public static string[] SkippableFiles = {
".DS_STORE",
"desktop.ini"
};
}
}

9
gaseous-tools/Config.cs
View File

@@ -404,14 +404,6 @@ namespace gaseous_tools
}
}
public string LibrarySignatureImportDirectory_TOSEC
{
get
{
return Path.Combine(LibrarySignatureImportDirectory, "TOSEC");
}
}
public void InitLibrary()
{
if (!Directory.Exists(LibraryRootDirectory)) { Directory.CreateDirectory(LibraryRootDirectory); }
@@ -423,7 +415,6 @@ namespace gaseous_tools
if (!Directory.Exists(LibraryTempDirectory)) { Directory.CreateDirectory(LibraryTempDirectory); }
if (!Directory.Exists(LibraryCollectionsDirectory)) { Directory.CreateDirectory(LibraryCollectionsDirectory); }
if (!Directory.Exists(LibrarySignatureImportDirectory)) { Directory.CreateDirectory(LibrarySignatureImportDirectory); }
if (!Directory.Exists(LibrarySignatureImportDirectory_TOSEC)) { Directory.CreateDirectory(LibrarySignatureImportDirectory_TOSEC); }
}
}

254
gaseous-tools/Database/MySQL/gaseous-1000.sql
View File

@@ -1,27 +1,4 @@
-- MySQL dump 10.13 Distrib 8.0.29, for macos12 (x86_64)
--
-- Host: localhost Database: gaseous
-- ------------------------------------------------------
-- Server version 8.0.33
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!50503 SET NameS utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
--
-- Table structure for table `AgeRating`
--
DROP TABLE IF EXISTS `AgeRating`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `AgeRating` (
`Id` bigint NOT NULL,
`Category` int DEFAULT NULL,
@@ -33,16 +10,9 @@ CREATE TABLE `AgeRating` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `AgeRatingContentDescription`
--
);
DROP TABLE IF EXISTS `AgeRatingContentDescription`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `AgeRatingContentDescription` (
`Id` bigint NOT NULL,
`Category` int DEFAULT NULL,
@@ -51,16 +21,9 @@ CREATE TABLE `AgeRatingContentDescription` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `AlternativeName`
--
);
DROP TABLE IF EXISTS `AlternativeName`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `AlternativeName` (
`Id` bigint NOT NULL,
`Checksum` varchar(45) DEFAULT NULL,
@@ -70,16 +33,9 @@ CREATE TABLE `AlternativeName` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Artwork`
--
);
DROP TABLE IF EXISTS `Artwork`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Artwork` (
`Id` bigint NOT NULL,
`AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -93,16 +49,9 @@ CREATE TABLE `Artwork` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Collection`
--
);
DROP TABLE IF EXISTS `Collection`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Collection` (
`Id` bigint NOT NULL,
`Checksum` varchar(45) DEFAULT NULL,
@@ -115,16 +64,9 @@ CREATE TABLE `Collection` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Company`
--
);
DROP TABLE IF EXISTS `Company`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Company` (
`Id` bigint NOT NULL,
`ChangeDate` datetime DEFAULT NULL,
@@ -148,16 +90,9 @@ CREATE TABLE `Company` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `CompanyLogo`
--
);
DROP TABLE IF EXISTS `CompanyLogo`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `CompanyLogo` (
`Id` bigint NOT NULL,
`AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -170,16 +105,9 @@ CREATE TABLE `CompanyLogo` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Cover`
--
);
DROP TABLE IF EXISTS `Cover`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Cover` (
`Id` bigint NOT NULL,
`AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -193,16 +121,9 @@ CREATE TABLE `Cover` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `ExternalGame`
--
);
DROP TABLE IF EXISTS `ExternalGame`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `ExternalGame` (
`Id` bigint NOT NULL,
`Category` int DEFAULT NULL,
@@ -220,16 +141,9 @@ CREATE TABLE `ExternalGame` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Franchise`
--
);
DROP TABLE IF EXISTS `Franchise`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Franchise` (
`Id` bigint NOT NULL,
`Checksum` varchar(45) DEFAULT NULL,
@@ -242,16 +156,9 @@ CREATE TABLE `Franchise` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Game`
--
);
DROP TABLE IF EXISTS `Game`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Game` (
`Id` bigint NOT NULL,
`AgeRatings` json DEFAULT NULL,
@@ -331,16 +238,9 @@ CREATE TABLE `Game` (
KEY `Idx_themes` ((cast(`Themes` as unsigned array))),
KEY `Idx_vIdeos` ((cast(`Videos` as unsigned array))),
KEY `Idx_websites` ((cast(`Websites` as unsigned array)))
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Games_Roms`
--
);
DROP TABLE IF EXISTS `Games_Roms`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Games_Roms` (
`Id` bigint NOT NULL AUTO_INCREMENT,
`PlatformId` bigint DEFAULT NULL,
@@ -361,16 +261,9 @@ CREATE TABLE `Games_Roms` (
UNIQUE KEY `Id_UNIQUE` (`Id`),
INDEX `GameId` (`GameId` ASC) VISIBLE,
INDEX `Id_GameId` (`GameId` ASC, `Id` ASC) VISIBLE
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `GameVideo`
--
);
DROP TABLE IF EXISTS `GameVideo`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `GameVideo` (
`Id` bigint NOT NULL,
`Checksum` varchar(45) DEFAULT NULL,
@@ -380,16 +273,9 @@ CREATE TABLE `GameVideo` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Genre`
--
);
DROP TABLE IF EXISTS `Genre`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Genre` (
`Id` bigint NOT NULL,
`Checksum` varchar(45) DEFAULT NULL,
@@ -401,16 +287,9 @@ CREATE TABLE `Genre` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `InvolvedCompany`
--
);
DROP TABLE IF EXISTS `InvolvedCompany`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `InvolvedCompany` (
`Id` bigint NOT NULL,
`Checksum` varchar(45) DEFAULT NULL,
@@ -425,16 +304,9 @@ CREATE TABLE `InvolvedCompany` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Platform`
--
);
DROP TABLE IF EXISTS `Platform`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Platform` (
`Id` bigint NOT NULL,
`Abbreviation` varchar(45) DEFAULT NULL,
@@ -456,16 +328,9 @@ CREATE TABLE `Platform` (
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`),
UNIQUE KEY `Id_UNIQUE` (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `PlatformLogo`
--
);
DROP TABLE IF EXISTS `PlatformLogo`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `PlatformLogo` (
`Id` bigint NOT NULL,
`AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -478,16 +343,9 @@ CREATE TABLE `PlatformLogo` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Platformversion`
--
);
DROP TABLE IF EXISTS `PlatformVersion`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `PlatformVersion` (
`Id` bigint NOT NULL,
`Checksum` varchar(45) DEFAULT NULL,
@@ -512,16 +370,9 @@ CREATE TABLE `PlatformVersion` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Screenshot`
--
);
DROP TABLE IF EXISTS `Screenshot`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Screenshot` (
`Id` bigint NOT NULL,
`AlphaChannel` tinyint(1) DEFAULT NULL,
@@ -535,31 +386,17 @@ CREATE TABLE `Screenshot` (
`dateAdded` datetime DEFAULT NULL,
`lastUpdated` datetime DEFAULT NULL,
PRIMARY KEY (`Id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Settings`
--
);
DROP TABLE IF EXISTS `Settings`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Settings` (
`Setting` varchar(45) NOT NULL,
`Value` longtext,
PRIMARY KEY (`Setting`),
UNIQUE KEY `Setting_UNIQUE` (`Setting`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Signatures_Games`
--
);
DROP TABLE IF EXISTS `Signatures_Games`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Signatures_Games` (
`Id` int NOT NULL AUTO_INCREMENT,
`Name` varchar(255) DEFAULT NULL,
@@ -578,48 +415,27 @@ CREATE TABLE `Signatures_Games` (
KEY `publisher_Idx` (`PublisherId`),
KEY `system_Idx` (`SystemId`),
KEY `ingest_Idx` (`Name`,`Year`,`PublisherId`,`SystemId`,`Country`,`Language`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Signatures_Platforms`
--
);
DROP TABLE IF EXISTS `Signatures_Platforms`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Signatures_Platforms` (
`Id` int NOT NULL AUTO_INCREMENT,
`Platform` varchar(100) DEFAULT NULL,
PRIMARY KEY (`Id`),
UNIQUE KEY `IdSignatures_Platforms_UNIQUE` (`Id`),
KEY `Platforms_Idx` (`Platform`,`Id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Signatures_Publishers`
--
);
DROP TABLE IF EXISTS `Signatures_Publishers`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Signatures_Publishers` (
`Id` int NOT NULL AUTO_INCREMENT,
`Publisher` varchar(100) DEFAULT NULL,
PRIMARY KEY (`Id`),
UNIQUE KEY `Id_UNIQUE` (`Id`),
KEY `publisher_Idx` (`Publisher`,`Id`)
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Signatures_Roms`
--
);
DROP TABLE IF EXISTS `Signatures_Roms`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Signatures_Roms` (
`Id` int NOT NULL AUTO_INCREMENT,
`GameId` int DEFAULT NULL,
@@ -640,16 +456,9 @@ CREATE TABLE `Signatures_Roms` (
KEY `md5_Idx` (`MD5`) USING BTREE,
KEY `sha1_Idx` (`SHA1`) USING BTREE,
KEY `flags_Idx` ((cast(`Flags` as char(255) array)))
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Table structure for table `Signatures_Sources`
--
);
DROP TABLE IF EXISTS `Signatures_Sources`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `Signatures_Sources` (
`Id` int NOT NULL AUTO_INCREMENT,
`Name` varchar(255) DEFAULT NULL,
@@ -667,20 +476,7 @@ CREATE TABLE `Signatures_Sources` (
UNIQUE KEY `Id_UNIQUE` (`Id`),
KEY `sourcemd5_Idx` (`SourceMD5`,`Id`) USING BTREE,
KEY `sourcesha1_Idx` (`SourceSHA1`,`Id`) USING BTREE
) ENGINE=InnoDB AUTO_INCREMENT=0 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
--
-- Dumping events for database 'gaseous'
--
--
-- Dumping routines for database 'gaseous'
--
--
-- Final view structure for view `view_Signatures_Games`
--
);
DROP VIEW IF EXISTS `view_Signatures_Games`;
CREATE VIEW `view_Signatures_Games` AS

4
gaseous-tools/Database/MySQL/gaseous-1001.sql
View File

@@ -1,5 +1,5 @@
DROP TABLE IF EXISTS `GameMode`;
CREATE TABLE `gaseous`.`GameMode` (
CREATE TABLE `GameMode` (
`Id` BIGINT NOT NULL,
`CreatedAt` DATETIME NULL,
`Checksum` VARCHAR(45) NULL,
@@ -82,5 +82,5 @@ CREATE TABLE `RomCollections` (
`BuiltStatus` INT NULL,
PRIMARY KEY (`Id`));
ALTER TABLE `gaseous`.`Signatures_Sources`
ALTER TABLE `Signatures_Sources`
CHANGE COLUMN `Author` `Author` LONGTEXT NULL DEFAULT NULL ;

11
gaseous-tools/Database/MySQL/gaseous-1002.sql Normal file
View File

@@ -0,0 +1,11 @@
ALTER TABLE `Signatures_Roms`
DROP INDEX `flags_Idx`;
ALTER TABLE `Signatures_Roms`
ADD COLUMN `Attributes` JSON NULL AFTER `Flags`,
ADD COLUMN `IngestorVersion` INT NULL DEFAULT 1;
ALTER TABLE `Games_Roms`
ADD COLUMN `Attributes` JSON NULL AFTER `Flags`,
ADD COLUMN `MetadataGameName` VARCHAR(255) NULL AFTER `MetadataSource`,
ADD COLUMN `MetadataVersion` INT NULL DEFAULT 1;

2
gaseous-tools/gaseous-tools.csproj
View File

@@ -17,6 +17,7 @@
<None Remove="Database\MySQL\" />
<None Remove="Database\MySQL\gaseous-1000.sql" />
<None Remove="Database\MySQL\gaseous-1001.sql" />
<None Remove="Database\MySQL\gaseous-1002.sql" />
</ItemGroup>
<ItemGroup>
<Folder Include="Database\" />
@@ -25,5 +26,6 @@
<ItemGroup>
<EmbeddedResource Include="Database\MySQL\gaseous-1000.sql" />
<EmbeddedResource Include="Database\MySQL\gaseous-1001.sql" />
<EmbeddedResource Include="Database\MySQL\gaseous-1002.sql" />
</ItemGroup>
</Project>