sendnrw/gaseous-server
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Signature ingestor database update is now a background task (#121)

Browse Source 
* Updated background task code to support options and self clearing

* Moved background safe database upgrade code to a background task
This commit is contained in:
Michael Green
2023-09-20 00:35:24 +10:00
committed by GitHub
parent 60fab488a2
commit 7eb418d6a2
7 changed files with 239 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
gaseous-server/Classes/Collections.cs
View File

@@ -171,13 +171,10 @@ namespace gaseous_server.Classes
db.ExecuteCMD(sql, dbDict); db.ExecuteCMD(sql, dbDict);
// start background task // start background task
foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems) ProcessQueue.QueueItem queueItem = new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.CollectionCompiler, 1, false, true);
{ queueItem.Options = Id;
if (qi.ItemType == ProcessQueue.QueueItemType.CollectionCompiler) { queueItem.ForceExecute();
qi.ForceExecute(); ProcessQueue.QueueItems.Add(queueItem);
break;
}
}
} }
} }
@@ -364,13 +361,11 @@ namespace gaseous_server.Classes
return collectionContents; return collectionContents;
} }
public static void CompileCollections() public static void CompileCollections(long CollectionId)
{ {
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
List<CollectionItem> collectionItems = GetCollections(); CollectionItem collectionItem = GetCollection(CollectionId);
foreach (CollectionItem collectionItem in collectionItems)
{
if (collectionItem.BuildStatus == CollectionItem.CollectionBuildStatus.WaitingForBuild) if (collectionItem.BuildStatus == CollectionItem.CollectionBuildStatus.WaitingForBuild)
{ {
Logging.Log(Logging.LogType.Information, "Collections", "Beginning build of collection: " + collectionItem.Name); Logging.Log(Logging.LogType.Information, "Collections", "Beginning build of collection: " + collectionItem.Name);
@@ -536,7 +531,6 @@ namespace gaseous_server.Classes
} }
} }
} }
}
private static CollectionItem BuildCollectionItem(DataRow row) { private static CollectionItem BuildCollectionItem(DataRow row) {
string strPlatforms = (string)Common.ReturnValueIfNull(row["Platforms"], "[ ]"); string strPlatforms = (string)Common.ReturnValueIfNull(row["Platforms"], "[ ]");

54
gaseous-server/ProcessQueue.cs
View File

@@ -9,34 +9,51 @@ namespace gaseous_server
public class QueueItem public class QueueItem
{ {
public QueueItem(QueueItemType ItemType, int ExecutionInterval, bool AllowManualStart = true) public QueueItem(QueueItemType ItemType, int ExecutionInterval, bool AllowManualStart = true, bool RemoveWhenStopped = false)
{ {
_ItemType = ItemType; _ItemType = ItemType;
_ItemState = QueueItemState.NeverStarted; _ItemState = QueueItemState.NeverStarted;
_LastRunTime = DateTime.UtcNow.AddMinutes(ExecutionInterval); _LastRunTime = DateTime.Parse(Config.ReadSetting("LastRun_" + _ItemType.ToString(), DateTime.UtcNow.ToString("yyyy-MM-ddThh:mm:ssZ")));
_Interval = ExecutionInterval; _Interval = ExecutionInterval;
_AllowManualStart = AllowManualStart; _AllowManualStart = AllowManualStart;
_RemoveWhenStopped = RemoveWhenStopped;
} }
public QueueItem(QueueItemType ItemType, int ExecutionInterval, List<QueueItemType> Blocks, bool AllowManualStart = true) public QueueItem(QueueItemType ItemType, int ExecutionInterval, List<QueueItemType> Blocks, bool AllowManualStart = true, bool RemoveWhenStopped = false)
{ {
_ItemType = ItemType; _ItemType = ItemType;
_ItemState = QueueItemState.NeverStarted; _ItemState = QueueItemState.NeverStarted;
_LastRunTime = DateTime.UtcNow.AddMinutes(ExecutionInterval); _LastRunTime = DateTime.Parse(Config.ReadSetting("LastRun_" + _ItemType.ToString(), DateTime.UtcNow.ToString("yyyy-MM-ddThh:mm:ssZ")));
_Interval = ExecutionInterval; _Interval = ExecutionInterval;
_AllowManualStart = AllowManualStart; _AllowManualStart = AllowManualStart;
_RemoveWhenStopped = RemoveWhenStopped;
_Blocks = Blocks; _Blocks = Blocks;
} }
private QueueItemType _ItemType = QueueItemType.NotConfigured; private QueueItemType _ItemType = QueueItemType.NotConfigured;
private QueueItemState _ItemState = QueueItemState.NeverStarted; private QueueItemState _ItemState = QueueItemState.NeverStarted;
private DateTime _LastRunTime = DateTime.UtcNow; private DateTime _LastRunTime = DateTime.UtcNow;
private DateTime _LastFinishTime = DateTime.UtcNow; private DateTime _LastFinishTime
{
get
{
return DateTime.Parse(Config.ReadSetting("LastRun_" + _ItemType.ToString(), DateTime.UtcNow.ToString("yyyy-MM-ddThh:mm:ssZ")));
}
set
{
if (_SaveLastRunTime == true)
{
Config.SetSetting("LastRun_" + _ItemType.ToString(), value.ToString("yyyy-MM-ddThh:mm:ssZ"));
}
}
}
private bool _SaveLastRunTime = false;
private int _Interval = 0; private int _Interval = 0;
private string _LastResult = ""; private string _LastResult = "";
private string? _LastError = null; private string? _LastError = null;
private bool _ForceExecute = false; private bool _ForceExecute = false;
private bool _AllowManualStart = true; private bool _AllowManualStart = true;
private bool _RemoveWhenStopped = false;
private List<QueueItemType> _Blocks = new List<QueueItemType>(); private List<QueueItemType> _Blocks = new List<QueueItemType>();
public QueueItemType ItemType => _ItemType; public QueueItemType ItemType => _ItemType;
@@ -54,6 +71,8 @@ namespace gaseous_server
public string? LastError => _LastError; public string? LastError => _LastError;
public bool Force => _ForceExecute; public bool Force => _ForceExecute;
public bool AllowManualStart => _AllowManualStart; public bool AllowManualStart => _AllowManualStart;
public bool RemoveWhenStopped => _RemoveWhenStopped;
public object? Options { get; set; } = null;
public List<QueueItemType> Blocks => _Blocks; public List<QueueItemType> Blocks => _Blocks;
public void Execute() public void Execute()
@@ -87,32 +106,52 @@ namespace gaseous_server
Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing MAME MESS files"); Logging.Log(Logging.LogType.Debug, "Signature Import", "Processing MAME MESS files");
tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME MESS"), gaseous_signature_parser.parser.SignatureParser.MAMEMess); tIngest.Import(Path.Combine(Config.LibraryConfiguration.LibrarySignatureImportDirectory, "MAME MESS"), gaseous_signature_parser.parser.SignatureParser.MAMEMess);
_SaveLastRunTime = true;
break; break;
case QueueItemType.TitleIngestor: case QueueItemType.TitleIngestor:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Title Ingestor"); Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Title Ingestor");
Classes.ImportGames importGames = new Classes.ImportGames(Config.LibraryConfiguration.LibraryImportDirectory); Classes.ImportGames importGames = new Classes.ImportGames(Config.LibraryConfiguration.LibraryImportDirectory);
_SaveLastRunTime = true;
break; break;
case QueueItemType.MetadataRefresh: case QueueItemType.MetadataRefresh:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Metadata Refresher"); Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Metadata Refresher");
Classes.MetadataManagement.RefreshMetadata(true); Classes.MetadataManagement.RefreshMetadata(true);
_SaveLastRunTime = true;
break; break;
case QueueItemType.OrganiseLibrary: case QueueItemType.OrganiseLibrary:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Organiser"); Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Organiser");
Classes.ImportGame.OrganiseLibrary(); Classes.ImportGame.OrganiseLibrary();
_SaveLastRunTime = true;
break; break;
case QueueItemType.LibraryScan: case QueueItemType.LibraryScan:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Scanner"); Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Library Scanner");
Classes.ImportGame.LibraryScan(); Classes.ImportGame.LibraryScan();
_SaveLastRunTime = true;
break; break;
case QueueItemType.CollectionCompiler: case QueueItemType.CollectionCompiler:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Collection Compiler"); Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Collection Compiler");
Classes.Collections.CompileCollections(); Classes.Collections.CompileCollections((long)Options);
break; break;
case QueueItemType.BackgroundDatabaseUpgrade:
Logging.Log(Logging.LogType.Debug, "Timered Event", "Starting Background Upgrade");
gaseous_tools.DatabaseMigration.UpgradeScriptBackgroundTasks();
break;
} }
} }
catch (Exception ex) catch (Exception ex)
@@ -143,7 +182,8 @@ namespace gaseous_server
MetadataRefresh, MetadataRefresh,
OrganiseLibrary, OrganiseLibrary,
LibraryScan, LibraryScan,
CollectionCompiler CollectionCompiler,
BackgroundDatabaseUpgrade
} }
public enum QueueItemState public enum QueueItemState

9
gaseous-server/Program.cs
View File

@@ -170,7 +170,14 @@ ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(
ProcessQueue.QueueItemType.OrganiseLibrary ProcessQueue.QueueItemType.OrganiseLibrary
}) })
); );
ProcessQueue.QueueItems.Add(new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.CollectionCompiler, 5, false));
// kick off any delayed upgrade tasks
// run 1002 background updates in the background on every start
DatabaseMigration.BackgroundUpgradeTargetSchemaVersions.Add(1002);
// start the task
ProcessQueue.QueueItem queueItem = new ProcessQueue.QueueItem(ProcessQueue.QueueItemType.BackgroundDatabaseUpgrade, 1, false, true);
queueItem.ForceExecute();
ProcessQueue.QueueItems.Add(queueItem);
// start the app // start the app
app.Run(); app.Run();

8
gaseous-server/Timer.cs
View File

@@ -33,9 +33,15 @@ namespace gaseous_server
//_logger.LogInformation( //_logger.LogInformation(
// "Timed Hosted Service is working. Count: {Count}", count); // "Timed Hosted Service is working. Count: {Count}", count);
foreach (ProcessQueue.QueueItem qi in ProcessQueue.QueueItems) { List<ProcessQueue.QueueItem> ActiveList = new List<ProcessQueue.QueueItem>();
ActiveList.AddRange(ProcessQueue.QueueItems);
foreach (ProcessQueue.QueueItem qi in ActiveList) {
if ((DateTime.UtcNow > qi.NextRunTime || qi.Force == true) && CheckProcessBlockList(qi) == true) { if ((DateTime.UtcNow > qi.NextRunTime || qi.Force == true) && CheckProcessBlockList(qi) == true) {
qi.Execute(); qi.Execute();
if (qi.RemoveWhenStopped == true && qi.ItemState == ProcessQueue.QueueItemState.Stopped)
{
ProcessQueue.QueueItems.Remove(qi);
}
} }
} }
} }

7
gaseous-server/wwwroot/pages/dialogs/collectionedit.html
View File

@@ -394,7 +394,10 @@
} }
modalAlwaysInclude = alwaysInclude; modalAlwaysInclude = alwaysInclude;
console.log(JSON.stringify(modalAlwaysInclude));
if (!alwaysInclude) {
alwaysInclude = [];
}
var item = { var item = {
"name": document.getElementById('collection_name').value, "name": document.getElementById('collection_name').value,
@@ -414,6 +417,8 @@
"alwaysInclude": alwaysInclude "alwaysInclude": alwaysInclude
} }
console.log("Item: " + JSON.stringify(item));
return item; return item;
} }

16
gaseous-server/wwwroot/pages/settings/system.html
View File

@@ -52,7 +52,10 @@
itemTypeName = "Library scan"; itemTypeName = "Library scan";
break; break;
case 'CollectionCompiler': case 'CollectionCompiler':
itemTypeName = "Compress collections"; itemTypeName = "Compress collection id: " + result[i].options;
break;
case 'BackgroundDatabaseUpgrade':
itemTypeName = "Background database upgrade";
break; break;
default: default:
itemTypeName = result[i].itemType; itemTypeName = result[i].itemType;
@@ -80,17 +83,24 @@
break; break;
} }
var itemInterval = result[i].interval;
var nextRunTime = moment(result[i].nextRunTime).fromNow();
var startButton = ''; var startButton = '';
if (result[i].allowManualStart == true && result[i].itemState != "Running") { if (result[i].allowManualStart == true && result[i].itemState != "Running") {
startButton = "<span id='startProcess' class='romstart' onclick='StartProcess(\"" + result[i].itemType + "\");'>Start</span>"; startButton = "<span id='startProcess' class='romstart' onclick='StartProcess(\"" + result[i].itemType + "\");'>Start</span>";
} }
if (result[i].allowManualStart == false && result[i].removeWhenStopped == true) {
itemInterval = '';
nextRunTime = '';
}
var newRow = [ var newRow = [
itemTypeName, itemTypeName,
itemStateName, itemStateName,
result[i].interval, itemInterval,
itemLastStart, itemLastStart,
moment(result[i].nextRunTime).fromNow(), nextRunTime,
startButton startButton
]; ];
newTable.appendChild(createTableRow(false, newRow, 'romrow', 'romcell')); newTable.appendChild(createTableRow(false, newRow, 'romrow', 'romcell'));

29
gaseous-tools/DatabaseMigration.cs
View File

@@ -5,26 +5,43 @@ namespace gaseous_tools
{ {
public static class DatabaseMigration public static class DatabaseMigration
{ {
public static void PreUpgradeScript(int TargetSchemaVersion, gaseous_tools.Database.databaseType? DatabaseType) { public static List<int> BackgroundUpgradeTargetSchemaVersions = new List<int>();
public static void PreUpgradeScript(int TargetSchemaVersion, gaseous_tools.Database.databaseType? DatabaseType)
{
} }
public static void PostUpgradeScript(int TargetSchemaVersion, gaseous_tools.Database.databaseType? DatabaseType) { public static void PostUpgradeScript(int TargetSchemaVersion, gaseous_tools.Database.databaseType? DatabaseType)
{
switch(DatabaseType) switch(DatabaseType)
{ {
case gaseous_tools.Database.databaseType.MySql: case Database.databaseType.MySql:
switch (TargetSchemaVersion)
{
case 1002:
// this is a safe background task
BackgroundUpgradeTargetSchemaVersions.Add(1002);
break;
}
break;
}
}
public static void UpgradeScriptBackgroundTasks()
{
foreach (int TargetSchemaVersion in BackgroundUpgradeTargetSchemaVersions)
{
switch (TargetSchemaVersion) switch (TargetSchemaVersion)
{ {
case 1002: case 1002:
MySql_1002_MigrateMetadataVersion(); MySql_1002_MigrateMetadataVersion();
break; break;
} }
break;
} }
} }
private static void MySql_1002_MigrateMetadataVersion() { public static void MySql_1002_MigrateMetadataVersion() {
Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString); Database db = new gaseous_tools.Database(Database.databaseType.MySql, Config.DatabaseConfiguration.ConnectionString);
string sql = ""; string sql = "";
Dictionary<string, object> dbDict = new Dictionary<string, object>(); Dictionary<string, object> dbDict = new Dictionary<string, object>();