sendnrw/gaseous-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Signature ingestor database update is now a background task (#121)

Browse Source 
* Updated background task code to support options and self clearing

* Moved background safe database upgrade code to a background task
This commit is contained in:
Michael Green
2023-09-20 00:35:24 +10:00
committed by GitHub
parent 60fab488a2
commit 7eb418d6a2
7 changed files with 239 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

274
gaseous-server/Classes/Collections.cs
View File

@@ -171,13 +171,10 @@ namespace gaseous_server.Classes
db.ExecuteCMD(sql, dbDict);
// start background task
foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems)
{
if (qi.ItemType == ProcessQueue.QueueItemType.CollectionCompiler) {
qi.ForceExecute();
break;
}
}
ProcessQueue.QueueItem queueItem = new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.CollectionCompiler, 1, false, true);
queueItem.Options = Id;
queueItem.ForceExecute();
ProcessQueue.QueueItems.Add(queueItem);
}
}
@@ -364,176 +361,173 @@ namespace gaseous_server.Classes
return collectionContents;
}
public static void CompileCollections()
public static void CompileCollections(long CollectionId)
{
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
List<CollectionItem> collectionItems = GetCollections();
foreach (CollectionItem collectionItem in collectionItems)
CollectionItem collectionItem = GetCollection(CollectionId);
if (collectionItem.BuildStatus == CollectionItem.CollectionBuildStatus.WaitingForBuild)
{
if (collectionItem.BuildStatus == CollectionItem.CollectionBuildStatus.WaitingForBuild)
Logging.Log(Logging.LogType.Information, "Collections", "Beginning build of collection: " + collectionItem.Name);
// set starting
string sql = "UPDATE RomCollections SET BuiltStatus=@bs WHERE Id=@id";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
dbDict.Add("id", collectionItem.Id);
dbDict.Add("bs", CollectionItem.CollectionBuildStatus.Building);
db.ExecuteCMD(sql, dbDict);
List<CollectionContents.CollectionPlatformItem> collectionPlatformItems = GetCollectionContent(collectionItem).Collection;
string ZipFilePath = Path.Combine(Config.LibraryConfiguration.LibraryCollectionsDirectory, collectionItem.Id + ".zip");
string ZipFileTempPath = Path.Combine(Config.LibraryConfiguration.LibraryTempDirectory, collectionItem.Id.ToString());
try
{
Logging.Log(Logging.LogType.Information, "Collections", "Beginning build of collection: " + collectionItem.Name);
// set starting
string sql = "UPDATE RomCollections SET BuiltStatus=@bs WHERE Id=@id";
Dictionary<string, object> dbDict = new Dictionary<string, object>();
dbDict.Add("id", collectionItem.Id);
dbDict.Add("bs", CollectionItem.CollectionBuildStatus.Building);
db.ExecuteCMD(sql, dbDict);
List<CollectionContents.CollectionPlatformItem> collectionPlatformItems = GetCollectionContent(collectionItem).Collection;
string ZipFilePath = Path.Combine(Config.LibraryConfiguration.LibraryCollectionsDirectory, collectionItem.Id + ".zip");
string ZipFileTempPath = Path.Combine(Config.LibraryConfiguration.LibraryTempDirectory, collectionItem.Id.ToString());
try
// clean up if needed
if (File.Exists(ZipFilePath))
{
Logging.Log(Logging.LogType.Warning, "Collections", "Deleting existing build of collection: " + collectionItem.Name);
File.Delete(ZipFilePath);
}
// clean up if needed
if (File.Exists(ZipFilePath))
if (Directory.Exists(ZipFileTempPath))
{
Directory.Delete(ZipFileTempPath, true);
}
// gather collection files
Directory.CreateDirectory(ZipFileTempPath);
string ZipBiosPath = Path.Combine(ZipFileTempPath, "BIOS");
// get the games
foreach (CollectionContents.CollectionPlatformItem collectionPlatformItem in collectionPlatformItems)
{
// get platform bios files if present
if (collectionItem.IncludeBIOSFiles == true)
{
Logging.Log(Logging.LogType.Warning, "Collections", "Deleting existing build of collection: " + collectionItem.Name);
File.Delete(ZipFilePath);
}
List<Bios.BiosItem> bios = Bios.GetBios(collectionPlatformItem.Id, true);
if (!Directory.Exists(ZipBiosPath)) {
Directory.CreateDirectory(ZipBiosPath);
}
if (Directory.Exists(ZipFileTempPath))
{
Directory.Delete(ZipFileTempPath, true);
}
// gather collection files
Directory.CreateDirectory(ZipFileTempPath);
string ZipBiosPath = Path.Combine(ZipFileTempPath, "BIOS");
// get the games
foreach (CollectionContents.CollectionPlatformItem collectionPlatformItem in collectionPlatformItems)
{
// get platform bios files if present
if (collectionItem.IncludeBIOSFiles == true)
foreach (Bios.BiosItem biosItem in bios)
{
List<Bios.BiosItem> bios = Bios.GetBios(collectionPlatformItem.Id, true);
if (!Directory.Exists(ZipBiosPath)) {
Directory.CreateDirectory(ZipBiosPath);
}
foreach (Bios.BiosItem biosItem in bios)
if (File.Exists(biosItem.biosPath))
{
if (File.Exists(biosItem.biosPath))
{
Logging.Log(Logging.LogType.Information, "Collections", "Copying BIOS file: " + biosItem.filename);
File.Copy(biosItem.biosPath, Path.Combine(ZipBiosPath, biosItem.filename));
}
Logging.Log(Logging.LogType.Information, "Collections", "Copying BIOS file: " + biosItem.filename);
File.Copy(biosItem.biosPath, Path.Combine(ZipBiosPath, biosItem.filename));
}
}
}
// create platform directory
string ZipPlatformPath = "";
switch (collectionItem.FolderStructure)
{
case CollectionItem.FolderStructures.Gaseous:
// create platform directory
string ZipPlatformPath = "";
switch (collectionItem.FolderStructure)
{
case CollectionItem.FolderStructures.Gaseous:
ZipPlatformPath = Path.Combine(ZipFileTempPath, collectionPlatformItem.Slug);
break;
case CollectionItem.FolderStructures.RetroPie:
try
{
PlatformMapping.PlatformMapItem platformMapItem = PlatformMapping.GetPlatformMap(collectionPlatformItem.Id);
ZipPlatformPath = Path.Combine(ZipFileTempPath, "roms", platformMapItem.RetroPieDirectoryName);
}
catch
{
ZipPlatformPath = Path.Combine(ZipFileTempPath, collectionPlatformItem.Slug);
break;
}
case CollectionItem.FolderStructures.RetroPie:
try
{
PlatformMapping.PlatformMapItem platformMapItem = PlatformMapping.GetPlatformMap(collectionPlatformItem.Id);
ZipPlatformPath = Path.Combine(ZipFileTempPath, "roms", platformMapItem.RetroPieDirectoryName);
}
catch
{
ZipPlatformPath = Path.Combine(ZipFileTempPath, collectionPlatformItem.Slug);
}
break;
break;
}
if (!Directory.Exists(ZipPlatformPath))
{
Directory.CreateDirectory(ZipPlatformPath);
}
}
if (!Directory.Exists(ZipPlatformPath))
foreach (CollectionContents.CollectionPlatformItem.CollectionGameItem collectionGameItem in collectionPlatformItem.Games)
{
bool includeGame = false;
if (collectionGameItem.InclusionStatus == null)
{
Directory.CreateDirectory(ZipPlatformPath);
includeGame = true;
}
foreach (CollectionContents.CollectionPlatformItem.CollectionGameItem collectionGameItem in collectionPlatformItem.Games)
else
{
bool includeGame = false;
if (collectionGameItem.InclusionStatus == null)
if (collectionGameItem.InclusionStatus.InclusionState == CollectionItem.AlwaysIncludeStatus.AlwaysInclude)
{
includeGame = true;
}
else
}
if (includeGame == true)
{
string ZipGamePath = "";
switch (collectionItem.FolderStructure)
{
if (collectionGameItem.InclusionStatus.InclusionState == CollectionItem.AlwaysIncludeStatus.AlwaysInclude)
{
includeGame = true;
}
case CollectionItem.FolderStructures.Gaseous:
// create game directory
ZipGamePath = Path.Combine(ZipPlatformPath, collectionGameItem.Slug);
if (!Directory.Exists(ZipGamePath))
{
Directory.CreateDirectory(ZipGamePath);
}
break;
case CollectionItem.FolderStructures.RetroPie:
ZipGamePath = ZipPlatformPath;
break;
}
if (includeGame == true)
// copy in roms
foreach (Roms.GameRomItem gameRomItem in collectionGameItem.Roms)
{
string ZipGamePath = "";
switch (collectionItem.FolderStructure)
if (File.Exists(gameRomItem.Path))
{
case CollectionItem.FolderStructures.Gaseous:
// create game directory
ZipGamePath = Path.Combine(ZipPlatformPath, collectionGameItem.Slug);
if (!Directory.Exists(ZipGamePath))
{
Directory.CreateDirectory(ZipGamePath);
}
break;
case CollectionItem.FolderStructures.RetroPie:
ZipGamePath = ZipPlatformPath;
break;
}
// copy in roms
foreach (Roms.GameRomItem gameRomItem in collectionGameItem.Roms)
{
if (File.Exists(gameRomItem.Path))
{
Logging.Log(Logging.LogType.Information, "Collections", "Copying ROM: " + gameRomItem.Name);
File.Copy(gameRomItem.Path, Path.Combine(ZipGamePath, gameRomItem.Name));
}
Logging.Log(Logging.LogType.Information, "Collections", "Copying ROM: " + gameRomItem.Name);
File.Copy(gameRomItem.Path, Path.Combine(ZipGamePath, gameRomItem.Name));
}
}
}
}
// compress to zip
Logging.Log(Logging.LogType.Information, "Collections", "Compressing collection");
ZipFile.CreateFromDirectory(ZipFileTempPath, ZipFilePath, CompressionLevel.SmallestSize, false);
// clean up
if (Directory.Exists(ZipFileTempPath))
{
Logging.Log(Logging.LogType.Information, "Collections", "Cleaning up");
Directory.Delete(ZipFileTempPath, true);
}
// set completed
dbDict["bs"] = CollectionItem.CollectionBuildStatus.Completed;
db.ExecuteCMD(sql, dbDict);
}
catch (Exception ex)
// compress to zip
Logging.Log(Logging.LogType.Information, "Collections", "Compressing collection");
ZipFile.CreateFromDirectory(ZipFileTempPath, ZipFilePath, CompressionLevel.SmallestSize, false);
// clean up
if (Directory.Exists(ZipFileTempPath))
{
// clean up
if (Directory.Exists(ZipFileTempPath))
{
Directory.Delete(ZipFileTempPath, true);
}
if (File.Exists(ZipFilePath))
{
File.Delete(ZipFilePath);
}
// set failed
dbDict["bs"] = CollectionItem.CollectionBuildStatus.Failed;
db.ExecuteCMD(sql, dbDict);
Logging.Log(Logging.LogType.Critical, "Collection Builder", "Collection building has failed", ex);
Logging.Log(Logging.LogType.Information, "Collections", "Cleaning up");
Directory.Delete(ZipFileTempPath, true);
}
// set completed
dbDict["bs"] = CollectionItem.CollectionBuildStatus.Completed;
db.ExecuteCMD(sql, dbDict);
}
catch (Exception ex)
{
// clean up
if (Directory.Exists(ZipFileTempPath))
{
Directory.Delete(ZipFileTempPath, true);
}
if (File.Exists(ZipFilePath))
{
File.Delete(ZipFilePath);
}
// set failed
dbDict["bs"] = CollectionItem.CollectionBuildStatus.Failed;
db.ExecuteCMD(sql, dbDict);
Logging.Log(Logging.LogType.Critical, "Collection Builder", "Collection building has failed", ex);
}
}
}

54
gaseous-server/ProcessQueue.cs
View File

@@ -9,34 +9,51 @@ namespace gaseous_server
public class QueueItem
{
public QueueItem(QueueItemType ItemType, int ExecutionInterval, bool AllowManualStart = true)
public QueueItem(QueueItemType ItemType, int ExecutionInterval, bool AllowManualStart = true, bool RemoveWhenStopped = false)
{
_ItemType = ItemType;
_ItemState = QueueItemState.NeverStarted;
_LastRunTime = DateTime.UtcNow.AddMinutes(ExecutionInterval);
_LastRunTime = DateTime.Parse(Config.ReadSetting("LastRun_" + _ItemType.ToString(), DateTime.UtcNow.ToString("yyyy-MM-ddThh:mm:ssZ")));
_Interval = ExecutionInterval;
_AllowManualStart = AllowManualStart;
_RemoveWhenStopped = RemoveWhenStopped;
}
public QueueItem(QueueItemType ItemType, int ExecutionInterval, List<QueueItemType> Blocks, bool AllowManualStart = true)
public QueueItem(QueueItemType ItemType, int ExecutionInterval, List<QueueItemType> Blocks, bool AllowManualStart = true, bool RemoveWhenStopped = false)
{
_ItemType = ItemType;
_ItemState = QueueItemState.NeverStarted;
_LastRunTime = DateTime.UtcNow.AddMinutes(ExecutionInterval);
_LastRunTime = DateTime.Parse(Config.ReadSetting("LastRun_" + _ItemType.ToString(), DateTime.UtcNow.ToString("yyyy-MM-ddThh:mm:ssZ")));
_Interval = ExecutionInterval;
_AllowManualStart = AllowManualStart;
_RemoveWhenStopped = RemoveWhenStopped;
_Blocks = Blocks;
}
private QueueItemType _ItemType = QueueItemType.NotConfigured;
private QueueItemState _ItemState = QueueItemState.NeverStarted;
private DateTime _LastRunTime = DateTime.UtcNow;
private DateTime _LastFinishTime = DateTime.UtcNow;
private DateTime _LastFinishTime
{
get
{
return DateTime.Parse(Config.ReadSetting("LastRun_" + _ItemType.ToString(), DateTime.UtcNow.ToString("yyyy-MM-ddThh:mm:ssZ")));
}
set
{
if (_SaveLastRunTime == true)
{
Config.SetSetting("LastRun_" + _ItemType.ToString(), value.ToString("yyyy-MM-ddThh:mm:ssZ"));
}
}
}
private bool _SaveLastRunTime = false;
private int _Interval = 0;
private string _LastResult = "";
private string? _LastError = null;
private bool _ForceExecute = false;
private bool _AllowManualStart = true;
private bool _RemoveWhenStopped = false;
private List<QueueItemType> _Blocks = new List<QueueItemType>();
public QueueItemType ItemType => _ItemType;
@@ -54,6 +71,8 @@ namespace gaseous_server
public string? LastError => _LastError;
public bool Force => _ForceExecute;
public bool AllowManualStart => _AllowManualStart;
public bool RemoveWhenStopped => _RemoveWhenStopped;
public object? Options { get; set; } = null;
public List<QueueItemType> Blocks => _Blocks;
public void Execute()
@@ -87,32 +106,52 @@ namespace gaseous_server
Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing MAME MESS files");
tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME MESS"), gaseous_signature_parser.parser.SignatureParser.MAMEMess);
_SaveLastRunTime = true;
break;
case QueueItemType.TitleIngestor:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Title Ingestor");
Classes.ImportGames importGames = new Classes.ImportGames(Config.LibraryConfiguration.LibraryImportDirectory);
_SaveLastRunTime = true;
break;
case QueueItemType.MetadataRefresh:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Metadata Refresher");
Classes.MetadataManagement.RefreshMetadata(true);
_SaveLastRunTime = true;
break;
case QueueItemType.OrganiseLibrary:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Organiser");
Classes.ImportGame.OrganiseLibrary();
_SaveLastRunTime = true;
break;
case QueueItemType.LibraryScan:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Scanner");
Classes.ImportGame.LibraryScan();
_SaveLastRunTime = true;
break;
case QueueItemType.CollectionCompiler:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Collection Compiler");
Classes.Collections.CompileCollections();
Classes.Collections.CompileCollections((long)Options);
break;
case QueueItemType.BackgroundDatabaseUpgrade:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Background Upgrade");
gaseous_tools.DatabaseMigration.UpgradeScriptBackgroundTasks();
break;
}
}
catch (Exception ex)
@@ -143,7 +182,8 @@ namespace gaseous_server
MetadataRefresh,
OrganiseLibrary,
LibraryScan,
CollectionCompiler
CollectionCompiler,
BackgroundDatabaseUpgrade
}
public enum QueueItemState

9
gaseous-server/Program.cs
View File

@@ -170,7 +170,14 @@ ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.OrganiseLibrary
})
);
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.CollectionCompiler, 5, false));
// kick off any delayed upgrade tasks
// run 1002 background updates in the background on every start
DatabaseMigration.BackgroundUpgradeTargetSchemaVersions.Add(1002);
// start the task
ProcessQueue.QueueItem queueItem = new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.BackgroundDatabaseUpgrade, 1, false, true);
queueItem.ForceExecute();
ProcessQueue.QueueItems.Add(queueItem);
// start the app
app.Run();

8
gaseous-server/Timer.cs
View File

@@ -33,9 +33,15 @@ namespace gaseous_server
//_logger.LogInformation(
// "Timed Hosted Service is working. Count: {Count}", count);
foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems) {
List<ProcessQueue.QueueItem> ActiveList = new List<ProcessQueue.QueueItem>();
ActiveList.AddRange(ProcessQueue.QueueItems);
foreach (ProcessQueue.QueueItem qi in ActiveList) {
if ((DateTime.UtcNow > qi.NextRunTime || qi.Force == true) && CheckProcessBlockList(qi) == true) {
qi.Execute();
if (qi.RemoveWhenStopped == true && qi.ItemState == ProcessQueue.QueueItemState.Stopped)
{
ProcessQueue.QueueItems.Remove(qi);
}
}
}
}

7
gaseous-server/wwwroot/pages/dialogs/collectionedit.html
View File

@@ -394,7 +394,10 @@
}
modalAlwaysInclude = alwaysInclude;
console.log(JSON.stringify(modalAlwaysInclude));
if (!alwaysInclude) {
alwaysInclude = [];
}
var item = {
"name": document.getElementById('collection_name').value,
@@ -414,6 +417,8 @@
"alwaysInclude": alwaysInclude
}
console.log("Item: " + JSON.stringify(item));
return item;
}

16
gaseous-server/wwwroot/pages/settings/system.html
View File

@@ -52,7 +52,10 @@
itemTypeName = "Library scan";
break;
case 'CollectionCompiler':
itemTypeName = "Compress collections";
itemTypeName = "Compress collection id: " + result[i].options;
break;
case 'BackgroundDatabaseUpgrade':
itemTypeName = "Background database upgrade";
break;
default:
itemTypeName = result[i].itemType;
@@ -80,17 +83,24 @@
break;
}
var itemInterval = result[i].interval;
var nextRunTime = moment(result[i].nextRunTime).fromNow();
var startButton = '';
if (result[i].allowManualStart == true && result[i].itemState != "Running") {
startButton = "<span id='startProcess' class='romstart' onclick='StartProcess(\"" + result[i].itemType + "\");'>Start</span>";
}
if (result[i].allowManualStart == false && result[i].removeWhenStopped == true) {
itemInterval = '';
nextRunTime = '';
}
var newRow = [
itemTypeName,
itemStateName,
result[i].interval,
itemInterval,
itemLastStart,
moment(result[i].nextRunTime).fromNow(),
nextRunTime,
startButton
];
newTable.appendChild(createTableRow(false, newRow, 'romrow', 'romcell'));

29
gaseous-tools/DatabaseMigration.cs
View File

@@ -5,26 +5,43 @@ namespace gaseous_tools
{
public static class DatabaseMigration
{
public static void PreUpgradeScript(int TargetSchemaVersion, gaseous_tools.Database.databaseType? DatabaseType) {
public static List<int> BackgroundUpgradeTargetSchemaVersions = new List<int>();
public static void PreUpgradeScript(int TargetSchemaVersion, gaseous_tools.Database.databaseType? DatabaseType)
{
}
public static void PostUpgradeScript(int TargetSchemaVersion, gaseous_tools.Database.databaseType? DatabaseType) {
public static void PostUpgradeScript(int TargetSchemaVersion, gaseous_tools.Database.databaseType? DatabaseType)
{
switch(DatabaseType)
{
case gaseous_tools.Database.databaseType.MySql:
case Database.databaseType.MySql:
switch (TargetSchemaVersion)
{
case 1002:
MySql_1002_MigrateMetadataVersion();
// this is a safe background task
BackgroundUpgradeTargetSchemaVersions.Add(1002);
break;
}
break;
}
}
private static void MySql_1002_MigrateMetadataVersion() {
public static void UpgradeScriptBackgroundTasks()
{
foreach (int TargetSchemaVersion in BackgroundUpgradeTargetSchemaVersions)
{
switch (TargetSchemaVersion)
{
case 1002:
MySql_1002_MigrateMetadataVersion();
break;
}
}
}
public static void MySql_1002_MigrateMetadataVersion() {
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = "";
Dictionary<string, object> dbDict = new Dictionary<string, object>();