sendnrw/gaseous-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: signature ingestor (mostly) complete

Browse Source
This commit is contained in:
Michael Green
2023-02-26 00:58:38 +11:00
parent be5904f9ad
commit c9e55c03f8
2 changed files with 358 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

363
gaseous-signature-ingestor/Program.cs
View File

@@ -4,6 +4,7 @@ using MySql.Data.MySqlClient;
using gaseous_romsignatureobject;
using gaseous_signature_parser.parsers;
using gaseous_tools;
using MySqlX.XDevAPI;
string configPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".gaseous-server");
@@ -11,6 +12,7 @@ string configPath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFo
string[] commandLineArgs = Environment.GetCommandLineArgs();
string tosecXML = "";
bool showGames = false;
string inArgument = "";
foreach (string commandLineArg in commandLineArgs)
{
@@ -23,6 +25,9 @@ foreach (string commandLineArg in commandLineArgs)
case "-tosecpath":
inArgument = commandLineArg.ToLower();
break;
case "-showgames":
showGames = true;
break;
default:
break;
}
@@ -42,6 +47,8 @@ foreach (string commandLineArg in commandLineArgs)
}
}
tosecXML = "/Users/michaelgreen/Downloads/TOSEC - DAT Pack - Complete (3764) (TOSEC-v2023-01-23)/TOSEC";
// check if configPath is valid and create it if not
if (!Directory.Exists(configPath))
{
@@ -63,62 +70,378 @@ if (Directory.Exists(tosecXML))
tosecXML = Path.GetFullPath(tosecXML);
string[] tosecPathContents = Directory.GetFiles(tosecXML);
int lineFileNameLength = 0;
int lineGameNameLength = 0;
string sql = "";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
System.Data.DataTable sigDB;
for (UInt16 i = 0; i < tosecPathContents.Length; ++i)
{
string tosecXMLFile = tosecPathContents[i];
string statusOutput = i + " / " + tosecPathContents.Length + " : " + Path.GetFileName(tosecXMLFile);
string statusOutput = (i + 1).ToString().PadLeft(7, ' ') + " / " + tosecPathContents.Length.ToString().PadLeft(7, ' ') + " : " + Path.GetFileName(tosecXMLFile);
Console.SetCursorPosition(0, Console.CursorTop - 2);
Console.WriteLine("\r " + statusOutput.PadRight(lineFileNameLength, ' ') + "\r");
lineFileNameLength = statusOutput.Length;
Console.WriteLine("Parsing file");
Console.WriteLine(" ==> Parsing file");
TosecParser tosecParser = new TosecParser();
RomSignatureObject tosecObject = tosecParser.Parse(tosecXMLFile);
// store in database
foreach (RomSignatureObject.Game gameObject in tosecObject.Games)
{
string sql = "";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
System.Data.DataTable sigDB;
// store platform
if (gameObject.System != null)
// store source object
bool processGames = false;
if (tosecObject.SourceMd5 != null)
{
sql = "SELECT * FROM signatures_platforms WHERE platform=@platform";
sql = "SELECT * FROM signatures_sources WHERE sourcemd5=@sourcemd5";
dbDict = new Dictionary<string, object>();
dbDict.Add("platform", gameObject.System);
if (tosecObject.Name != null)
{
dbDict.Add("name", tosecObject.Name);
}
else
{
dbDict.Add("name", "");
}
if (tosecObject.Description != null)
{
dbDict.Add("description", tosecObject.Description);
}
else
{
dbDict.Add("description", "");
}
if (tosecObject.Category != null)
{
dbDict.Add("category", tosecObject.Category);
}
else
{
dbDict.Add("category", "");
}
if (tosecObject.Version != null)
{
dbDict.Add("version", tosecObject.Version);
}
else
{
dbDict.Add("version", "");
}
if (tosecObject.Author != null)
{
dbDict.Add("author", tosecObject.Author);
}
else
{
dbDict.Add("author", "");
}
if (tosecObject.Email != null)
{
dbDict.Add("email", tosecObject.Email);
}
else
{
dbDict.Add("email", "");
}
if (tosecObject.Homepage != null)
{
dbDict.Add("homepage", tosecObject.Homepage);
}
else
{
dbDict.Add("homepage", "");
}
if (tosecObject.Url != null)
{
dbDict.Add("uri", tosecObject.Url.ToString());
}
else
{
dbDict.Add("uri", "");
}
if (tosecObject.SourceType != null)
{
dbDict.Add("sourcetype", tosecObject.SourceType);
}
else
{
dbDict.Add("sourcetype", "");
}
dbDict.Add("sourcemd5", tosecObject.SourceMd5);
dbDict.Add("sourcesha1", tosecObject.SourceSHA1);
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
Console.SetCursorPosition(0, Console.CursorTop - 1);
Console.WriteLine("Saving platform: " + gameObject.System);
sql = "INSERT INTO signatures_sources (name, description, category, version, author, email, homepage, url, sourcetype, sourcemd5, sourcesha1) VALUES (@name, @description, @category, @version, @author, @email, @homepage, @uri, @sourcetype, @sourcemd5, @sourcesha1)";
sql = "INSERT INTO signatures_platforms (platform) VALUES (@platform)";
db.ExecuteCMD(sql, dbDict);
processGames = true;
}
if (processGames == true)
{
for (int x = 0; x < tosecObject.Games.Count; ++x)
{
RomSignatureObject.Game gameObject = tosecObject.Games[x];
// update display
if (showGames == true)
{
Console.SetCursorPosition(0, Console.CursorTop - 1);
string statusGameOutput = " ==> " + (x + 1).ToString().PadLeft(7, ' ') + " / " + tosecObject.Games.Count.ToString().PadLeft(7, ' ') + " : " + gameObject.Name;
Console.WriteLine("\r " + statusGameOutput.PadRight(lineGameNameLength, ' ') + "\r");
lineGameNameLength = statusGameOutput.Length;
}
// set up game dictionary
dbDict = new Dictionary<string, object>();
if (gameObject.Name != null)
{
dbDict.Add("name", gameObject.Name);
} else
{
dbDict.Add("name", "");
}
if (gameObject.Description != null)
{
dbDict.Add("description", gameObject.Description);
}
else
{
dbDict.Add("description", "");
}
if (gameObject.Year != null)
{
dbDict.Add("year", gameObject.Year);
}
else
{
dbDict.Add("year", "");
}
if (gameObject.Publisher != null)
{
dbDict.Add("publisher", gameObject.Publisher);
}
else
{
dbDict.Add("publisher", "");
}
dbDict.Add("demo", (int)gameObject.Demo);
if (gameObject.System != null)
{
dbDict.Add("system", gameObject.System);
dbDict.Add("platform", gameObject.System);
}
else
{
dbDict.Add("system", "");
}
if (gameObject.SystemVariant != null)
{
dbDict.Add("systemvariant", gameObject.SystemVariant);
}
else
{
dbDict.Add("systemvariant", "");
}
if (gameObject.Video != null)
{
dbDict.Add("video", gameObject.Video);
}
else
{
dbDict.Add("video", "");
}
if (gameObject.Country != null)
{
dbDict.Add("country", gameObject.Country);
}
else
{
dbDict.Add("country", "");
}
if (gameObject.Language != null)
{
dbDict.Add("language", gameObject.Language);
}
else
{
dbDict.Add("language", "");
}
if (gameObject.Copyright != null)
{
dbDict.Add("copyright", gameObject.Copyright);
}
else
{
dbDict.Add("copyright", "");
}
// store platform
int gameSystem = 0;
if (gameObject.System != null)
{
sql = "SELECT id FROM signatures_platforms WHERE platform=@platform";
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
sql = "INSERT INTO signatures_platforms (platform) VALUES (@platform); SELECT LAST_INSERT_ID()";
sigDB = db.ExecuteCMD(sql, dbDict);
gameSystem = int.Parse(sigDB.Rows[0][0].ToString());
} else
{
gameSystem = int.Parse(sigDB.Rows[0][0].ToString());
}
}
dbDict.Add("systemid", gameSystem);
// store publisher
int gamePublisher = 0;
if (gameObject.Publisher != null)
{
sql = "SELECT * FROM signatures_publishers WHERE publisher=@publisher";
dbDict = new Dictionary<string, object>();
dbDict.Add("publisher", gameObject.Publisher);
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
Console.SetCursorPosition(0, Console.CursorTop - 1);
Console.WriteLine("Saving publisher: " + gameObject.Publisher);
sql = "INSERT INTO signatures_publishers (publisher) VALUES (@publisher); SELECT LAST_INSERT_ID()";
sigDB = db.ExecuteCMD(sql, dbDict);
gamePublisher = int.Parse(sigDB.Rows[0][0].ToString());
}
else
{
gamePublisher = int.Parse(sigDB.Rows[0][0].ToString());
}
}
dbDict.Add("publisherid", gamePublisher);
sql = "INSERT INTO signatures_publishers (publisher) VALUES (@publisher)";
db.ExecuteCMD(sql, dbDict);
// store game
int gameId = 0;
sql = "SELECT * FROM signatures_games WHERE name=@name AND year=@year AND publisherid=@publisher AND systemid=@systemid AND country=@country AND language=@language";
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
sql = "INSERT INTO signatures_games " +
"(name, description, year, publisherid, demo, systemid, systemvariant, video, country, language, copyright) VALUES " +
"(@name, @description, @year, @publisherid, @demo, @systemid, @systemvariant, @video, @country, @language, @copyright); SELECT LAST_INSERT_ID()";
sigDB = db.ExecuteCMD(sql, dbDict);
gameId = int.Parse(sigDB.Rows[0][0].ToString());
}
else
{
gameId = int.Parse(sigDB.Rows[0][0].ToString());
}
// store rom
foreach (RomSignatureObject.Game.Rom romObject in gameObject.Roms)
{
if (romObject.Md5 != null)
{
int romId = 0;
sql = "SELECT * FROM signatures_roms WHERE gameid=@gameid AND md5=@md5";
dbDict = new Dictionary<string, object>();
dbDict.Add("gameid", gameId);
if (romObject.Name != null)
{
dbDict.Add("name", romObject.Name);
} else
{
dbDict.Add("name", "");
}
if (romObject.Size != null)
{
dbDict.Add("size", romObject.Size);
}
else
{
dbDict.Add("size", 0);
}
if (romObject.Crc != null)
{
dbDict.Add("crc", romObject.Crc);
}
else
{
dbDict.Add("name", "");
}
dbDict.Add("md5", romObject.Md5);
if (romObject.Sha1 != null)
{
dbDict.Add("sha1", romObject.Sha1);
}
else
{
dbDict.Add("sha1", "");
}
if (romObject.DevelopmentStatus != null)
{
dbDict.Add("developmentstatus", romObject.DevelopmentStatus);
}
else
{
dbDict.Add("developmentstatus", "");
}
if (romObject.flags != null)
{
if (romObject.flags.Count > 0)
{
dbDict.Add("flags", Newtonsoft.Json.JsonConvert.SerializeObject(romObject.flags));
} else
{
dbDict.Add("flags", "{ }");
}
}
else
{
dbDict.Add("flags", "{ }");
}
dbDict.Add("romtype", (int)romObject.RomType);
if (romObject.RomTypeMedia != null)
{
dbDict.Add("romtypemedia", romObject.RomTypeMedia);
}
else
{
dbDict.Add("romtypemedia", "");
}
if (romObject.MediaLabel != null)
{
dbDict.Add("medialabel", romObject.MediaLabel);
}
else
{
dbDict.Add("medialabel", "");
}
sigDB = db.ExecuteCMD(sql, dbDict);
if (sigDB.Rows.Count == 0)
{
// entry not present, insert it
sql = "INSERT INTO signatures_roms (gameid, name, size, crc, md5, sha1, developmentstatus, flags, romtype, romtypemedia, medialabel) VALUES (@gameid, @name, @size, @crc, @md5, @sha1, @developmentstatus, @flags, @romtype, @romtypemedia, @medialabel); SELECT LAST_INSERT_ID()";
sigDB = db.ExecuteCMD(sql, dbDict);
romId = int.Parse(sigDB.Rows[0][0].ToString());
}
else
{
romId = int.Parse(sigDB.Rows[0][0].ToString());
}
}
}
}
}
}

1
gaseous-signature-ingestor/gaseous-signature-ingestor.csproj
View File

@@ -15,5 +15,6 @@
</ItemGroup>
<ItemGroup>
<PackageReference Include="MySql.Data" Version="8.0.32" />
<PackageReference Include="Newtonsoft.Json" Version="13.0.2" />
</ItemGroup>
</Project>